This site uses cookies. Visit our cookies policy page or click the link in any footer for more information and to change your preferences.

Accept all cookies Accept only essential cookies

Code development platform for open source projects from the European Union institutions :large_blue_circle: EU Login authentication by SMS has been phased out. To see alternatives please check here

Skip to content
Snippets Groups Projects
Commit e95b05e3 authored by Thomas Vliagkoftis's avatar Thomas Vliagkoftis
Browse files

enh: README.md, environment.yml, db.py and config.py

parent 4431776f
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
GIT_SETUP.md
+ 6
− 7
View file @ e95b05e3
......@@ -27,8 +27,6 @@ You will be using the following programs the jeodpp provides:
Create a new access token:
* Log into Gitlab [https://gitlab.jrc.nl/](https://gitlab.jrc.nl/)and click your Profile -> Settings -> Access Tokens
![](https://t2189764.p.clickup-attachments.com/t2189764/934070d9-de92-4f6a-bc29-c979f7cc7909/image.png)
......@@ -38,13 +36,14 @@ Create a new access token:
* tick `read_repository` and `write_repository` scopes
* click `Create personal access token` button
* Copy the token into your clipboard - it will be only visible until you close/refresh the page; afterwards you would have to re-create it.
* Try to clone:
* Try to clone this repo inside your **home dir** (`/home/$USER`):
```plain
git clone https://jeodpp.jrc.ec.europa.eu/apps/gitlab/use_cases/legent/uds-scripts.git
```bash
cd ~
git clone https://jeodpp.jrc.ec.europa.eu/apps/gitlab/use_cases/legent/uds-scripts.git/
```
OR add a new remote into your _existing_ local git-repo and then try to fetch:
**DO WE NEED THIS LINE?** OR add a new remote into your _existing_ local git-repo and then try to fetch:
2.2. SSH keys
......
README.md
+ 2
− 4
View file @ e95b05e3
......@@ -31,12 +31,10 @@ preloaded in *eos-dirs* (read section below about Nextcloud mapping).
1. Read and apply [first-time `git@BDAP` setup instructions](./GIT_SETUP.md).
2. Clone this repo inside your **home dir** (`/home/$USER`):
2. Navigate inside your previously cloned **repo folder** (`/home/$USER/uds-scripts`):
```bash
cd ~
git clone https://jeodpp.jrc.ec.europa.eu/apps/gitlab/use_cases/legent/uds-scripts.git/ \
&& cd uds-scripts
cd ~/uds-scripts
```
3. Run the `setup-account.sh` script to setup your BDAP linux account for the first time,
......
src/uds4jrc/config.py
+ 235
− 15
View file @ e95b05e3
......@@ -12,7 +12,7 @@ class Config(mysecrets.Secrets):
UEOS = f"/eos/jeodpp/home/users/{USER}"
UTRANS = f"/eos/jeodpp/home/users/{USER}/transfer"
PEOS = "/eos/jeodpp/data/projects/LEGENT"
PTRANS = "/eos/jeodpp/data/projects/LEGENT/transfer"
PTRANS = "/eos/jeodpp/data/projects/LEGENT/internal/transfer"
EEA_2013 = {
'delimiter': '\t',
......@@ -344,7 +344,7 @@ class Config(mysecrets.Secrets):
GECO_2020 = {
'year': 2020,
'file_path': '/eos/jeodpp/data/projects/LEGENT/realworld/Geco_air_2020.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/realworld/Geco_air_2020.xlsx',
'encoding': 'utf-8',
'column_properties': pd.DataFrame([
['id', '', np.int32],
......@@ -370,7 +370,7 @@ class Config(mysecrets.Secrets):
GECO_2021 = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/realworld/Geco_air_2021.xls',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/realworld/Geco_air_2021.xls',
'encoding': 'utf-8',
'column_properties': pd.DataFrame([
['id', '', np.int32],
......@@ -398,7 +398,7 @@ class Config(mysecrets.Secrets):
TRAVELCARD_PETROL = {
'year': '-',
'file_path': '/eos/jeodpp/data/projects/LEGENT/realworld/TravelcardPetrol4JRC.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/realworld/TravelcardPetrol4JRC.xlsx',
'file_data_category': 'petrol',
'encoding': 'utf-8',
'column_properties': pd.DataFrame([
......@@ -423,7 +423,7 @@ class Config(mysecrets.Secrets):
TRAVELCARD_DIESEL = {
'year': '-',
'file_path': '/eos/jeodpp/data/projects/LEGENT/realworld/TravelcardDiesel4JRC.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/realworld/TravelcardDiesel4JRC.xlsx',
'file_data_category': 'diesel',
'encoding': 'utf-8',
'column_properties': pd.DataFrame([
......@@ -448,7 +448,7 @@ class Config(mysecrets.Secrets):
SPRITMONITOR_2020 = {
'year': 2020,
'file_path': '/eos/jeodpp/data/projects/LEGENT/realworld/Spritmonitor_2020.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/realworld/Spritmonitor_2020.xlsx',
'encoding': 'utf-8',
'column_properties': pd.DataFrame([
['id', 'VehicleID', np.int32],
......@@ -493,7 +493,7 @@ class Config(mysecrets.Secrets):
SPRITMONITOR_2021 = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/realworld/Spritmonitor_2021.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/realworld/Spritmonitor_2021.xlsx',
'encoding': 'utf-8',
'column_properties': pd.DataFrame([
['id', 'VehicleID', np.int32],
......@@ -538,7 +538,7 @@ class Config(mysecrets.Secrets):
ATCT_LUXEMBURG = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/ATCT_SNCH_20210426_Luxembourg.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/ATCT_SNCH_20210426_Luxembourg.xlsx',
'data': 'Luxemburg',
'encoding': 'utf-8',
'skip_rows': 0,
......@@ -557,7 +557,7 @@ class Config(mysecrets.Secrets):
ATCT_IRISH = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/20210504_Export_ATCT_data_Irish.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/20210504_Export_ATCT_data_Irish.xlsx',
'data': 'Irish',
'encoding': 'utf-8',
'skip_rows': 0,
......@@ -572,7 +572,7 @@ class Config(mysecrets.Secrets):
ATCT_IDIADA = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/e9_ATCT_IDIADA.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/e9_ATCT_IDIADA.xlsx',
'data': 'Idiada',
'encoding': 'utf-8',
'skip_rows': 3,
......@@ -596,7 +596,7 @@ class Config(mysecrets.Secrets):
ATCT_INTA_1 = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/e9_ATCT_INTA_1.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/e9_ATCT_INTA_1.xlsx',
'data': 'Inta_1',
'encoding': 'utf-8',
'skip_rows': 1,
......@@ -605,7 +605,7 @@ class Config(mysecrets.Secrets):
ATCT_INTA_2 = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/e9_ATCT_INTA_2.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/e9_ATCT_INTA_2.xlsx',
'data': 'Inta_2',
'encoding': 'utf-8',
'skip_rows': 1,
......@@ -614,7 +614,7 @@ class Config(mysecrets.Secrets):
ATCT_INTA_3 = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/e9_ATCT_INTA_3.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/e9_ATCT_INTA_3.xlsx',
'data': 'Inta_3',
'encoding': 'utf-8',
'skip_rows': 1,
......@@ -623,7 +623,7 @@ class Config(mysecrets.Secrets):
ATCT_FCA = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/FCA_ATCT_FCF.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/FCA_ATCT_FCF.xlsx',
'data': 'fca',
'encoding': 'utf-8',
'skip_rows': 0,
......@@ -639,7 +639,7 @@ class Config(mysecrets.Secrets):
ATCT_FERRARI = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/atct/Ferrari_Gamma_FCF.xlsx',
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/atct/Ferrari_Gamma_FCF.xlsx',
'data': 'Ferrari',
'encoding': 'utf-8',
'skip_rows': 0,
......@@ -652,3 +652,223 @@ class Config(mysecrets.Secrets):
['emission_character', 'Emission Character', np.object],
], columns=['db_names', 'names', 'coltype']),
}
COP_FR = {
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/cop/CoP data format_FR.xlsx',
'data': 'FR',
'encoding': 'utf-8',
'skip_rows': 1,
'column_properties': pd.DataFrame([
['cop_id', 'CoP_ID', np.int32],
['cop_family', 'CoP_Family_Code', np.object],
['cop_family_2', 'Vehicle model code', np.int32],
['fuel_mode', 'Vehicle Type', np.object],
['fuel_type', 'Fuel', np.object],
['co2_measured_declared_ratio', 'Normalized_CO2 or electric consumption', np.float16],
['mox_measured', 'measured NOx', np.float16],
['nmhc_measured', 'measured NMHC', np.float16],
['hc_nox_measured', 'measured HC+NOx', np.float16],
['co_measured', 'measured CO', np.float16],
['pm_measured', 'measured PM', np.float16],
['pn_measured', 'measured PN', np.float16],
['hc_measured', 'measured HC', np.float16],
], columns=['db_names', 'names', 'coltype']),
}
COP_DE = {
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/cop/CoP data format_DE.xlsx',
'data': 'DE',
'encoding': 'utf-8',
'skip_rows': 1,
'column_properties': pd.DataFrame([
['cop_id', 'CoP_ID', np.int32],
['cop_family', 'CoP_Family_Code', np.int32],
['cop_family_2', 'vehicles with same declared CO2 within an IP-family', np.object],
['fuel_mode', 'Vehicle Type', np.object],
['fuel_type', 'Fuel', np.object],
['co2_measured_declared_ratio', 'Normalized_CO2', np.float16],
['mox_measured', 'measured NOx', np.float16],
['nmhc_measured', 'measured NMHC', np.float16],
['hc_nox_measured', 'measured HC+NOx', np.float16],
['co_measured', 'measured CO', np.float16],
['pm_measured', 'measured PM', np.float16],
['pn_measured', 'measured PN', np.float16],
['comments', 'Comments', np.object],
], columns=['db_names', 'names', 'coltype']),
}
COP_NED = {
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/cop/CoP data format_NED.xlsx',
'data': 'NED',
'encoding': 'utf-8',
'skip_rows': 1,
'column_properties': pd.DataFrame([
['cop_id', 'CoP_ID', np.object],
['cop_family', 'CoP_Family_Code', np.int32],
['cop_family_2', 'Vehicle model code', np.float16],
['fuel_mode', 'Vehicle Type', np.object],
['fuel_type', 'Fuel', np.object],
['co2_measured_declared_ratio', 'Normalized_CO2', np.float16],
['mox_measured', 'measured NOx', np.float16],
['nmhc_measured', 'measured NMHC', np.float16],
['hc_nox_measured', 'measured HC+NOx', np.float16],
['co_measured', 'measured CO', np.float16],
['pm_measured', 'measured PM', np.float16],
['pn_measured', 'measured PN', np.float16],
], columns=['db_names', 'names', 'coltype']),
}
LDV_PEMS = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/obfcm/LDV-PEMS.xlsx',
'data': 'LDV PEMS',
'encoding': 'utf-8',
'skip_rows': 0,
'column_properties': pd.DataFrame([
['id', '', np.int32],
['test_id', 'Test.ID', np.object],
['cycle_phase', 'Phase', np.object],
['vehicle', 'Vehicle', np.object],
['lab', 'LAB', np.int32],
['fuel', 'Fuel', np.object],
['cycle', 'Cycle', np.object],
['rde_mileage', 'RDE.distance..km.', np.float16],
['rde_fc', 'RDE.FC..g.', np.float16],
['time', 'Time..s.', np.int32],
['lab_speed', 'Lab.Speed', np.float16],
['obd_speed', 'OBD.speed', np.float16],
['obd_engine_speed', 'OBD.rpm', np.float16],
['obd_coolant_temperature', 'OBD.coolant.temp', np.float16],
['obd_mileage', 'OBD.distance..km.', np.float16],
['ambient_temperature', 'Amb.Temp', np.float16],
['average_relative_positive_acceleration', 'RPA', np.float16],
['va_95', 'V.a_95.', np.float16],
['obd_fc', 'OBD.FC..g.', np.float16],
['mileage_difference', 'Diff_km_percent', np.float16],
['fc_difference', 'Diff_g_percent', np.float16],
['route', 'Route', np.object],
['compliant', 'Compliant', np.object],
['oem', 'OEM', np.object],
['fuel_type', 'Fuel', np.object],
['engine_displacement', 'Engine Size (cc) ', np.int32],
['engine_power', 'Engine Power (kW)', np.int32],
['gearbox_type', 'TR', np.object],
['mass_in_running_order', 'MRO** (kg)', np.int32],
], columns=['db_names', 'names', 'coltype']),
}
LDV_LABS = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/obfcm/LDV-LAB.xlsx',
'data': 'LDV LAB',
'encoding': 'utf-8',
'skip_rows': 0,
'column_properties': pd.DataFrame([
['id', '', np.int32],
['test_id', 'Test.ID', np.object],
['cycle_phase', 'Phase', np.object],
['vehicle', 'Vehicle', np.object],
['lab', 'LAB', np.int32],
['fuel', 'Fuel', np.object],
['cycle', 'Cycle', np.object],
['fuel_density', 'Density', np.float16],
['lab_mileage', 'Lab.distance..km.', np.float16],
['lab_fc_l_100km', 'Lab.FC..l.100km.', np.float16],
['lab_fc_g', 'Lab.FC..g.', np.float16],
['lab_fc_l', 'Lab.FC..l.', np.float16],
['time', 'Time..s.', np.int32],
['lab_speed', 'Lab.Speed', np.float16],
['average_relative_positive_acceleration', 'Acceleration', np.float16],
['va_95', 'Va', np.float16],
['obd_speed', 'OBD.speed', np.float16],
['obd_engine_speed', 'OBD.rpm', np.float16],
['obd_coolant_temperature', 'OBD.coolant.temp', np.float16],
['obd_load', 'OBD.load', np.float16],
['obd_mileage', 'OBD.distance..km.', np.float16],
['obd_fc_l', 'OBD.FC..l.', np.float16],
['obd_fc_g', 'OBD.FC..g.', np.float16],
['cold_hot_start', 'Cold', np.object],
['mileage_difference', 'Diff_km_percent', np.float16],
['fc_difference_g', 'Difference', np.float16],
['obd_fc_l_100km', 'obd l/100km', np.float16],
['fc_difference_l_100km', 'diff km l/100km', np.float16],
['fc_difference_l', 'diff fc l', np.float16],
['oem', 'OEM', np.object],
['fuel_type', 'Fuel', np.object],
['engine_displacement', 'Engine Size (cc) ', np.int32],
['engine_power', 'Engine Power (kW)', np.int32],
['gearbox_type', 'TR', np.object],
['mass_in_running_order', 'MRO** (kg)', np.int32],
], columns=['db_names', 'names', 'coltype']),
}
HDV_PEMS = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/obfcm/HDV-PEMS.xlsx',
'data': 'HDV PEMS',
'encoding': 'utf-8',
'skip_rows': 0,
'column_properties': pd.DataFrame([
['id', '', np.int32],
['test_id', 'Test_ID', np.object],
['cycle_phase', 'Phase', np.object],
['vehicle', 'Vehicle', np.int32],
['time', 'Time', np.int32],
['pems_speed', 'PEMS_speed_avg', np.float16],
['pems_engine_speed', 'RPA', np.float16],
['va_95', 'V_a_95', np.float16],
['obd_speed', 'OBD_speed_avg', np.float16],
['obd_engine_speed', 'OBD_rpm', np.float16],
['obd_coolant_temperature', 'OBD_cool_temp', np.float16],
['obd_power', 'OBD_power', np.float16],
['ambient_temperature', 'Amb_Temp', np.float16],
['pems_mileage', 'PEMS_distance', np.float16],
['obd_mileage', 'OBD_distance', np.float16],
['pems_fc', 'PEMS_FC_g', np.float16],
['obd_fc', 'OBD_FC_g', np.float16],
['can_fc', 'CAN_FC_g', np.float16],
['ffm_fc', 'FFM_FC_g', np.float16],
['mileage', 'Distance_GPSvsOBD', np.float16],
['fc_difference_obd_vs_pems', 'FC_OBDvsPEMS', np.float16],
['fc_difference_obd_vs_ffm', 'FC_OBDvsFFM', np.float16],
['fc_difference_can_vs_pems', 'FC_CANvsPEMS', np.float16],
['fc_difference_can_vs_ffm', 'FC_CANvsFFM', np.float16],
['fc_difference_ffm_vs_pems', 'FC_FFMvsPEMS', np.float16],
['oem', 'OEM', np.object],
['engine_displacement', 'Engine displacement (L)', np.float16],
['engine_power', 'Engine Power (kW)', np.int32],
['gearbox_type', 'TR', np.object],
['technically_permissible_maximum_laden_mass.', 'TPMLM (t)', np.float16],
['test_mass', 'Test mass (t)', np.float16],
], columns=['db_names', 'names', 'coltype']),
}
HDV_LABS = {
'year': 2021,
'file_path': '/eos/jeodpp/data/projects/LEGENT/internal/obfcm/HDV-LAB.xlsx',
'data': 'HDV LAB',
'encoding': 'utf-8',
'skip_rows': 0,
'column_properties': pd.DataFrame([
['id', '', np.int32],
['test_id', 'Test ID', np.object],
['cycle_phase', 'Phase', np.object],
['time', 'Time (s)', np.int32],
['lab_speed', 'Lab Speed', np.float16],
['lab_engine_speed', 'RPA', np.float16],
['va_95', 'V*a_95%', np.float16],
['obd_speed', 'OBD speed', np.float16],
['obd_engine_speed', 'OBD rpm', np.float16],
['obd_coolant_temperature', 'OBD coolant temp', np.float16],
['obd_engine_power', 'OBD engine power', np.float16],
['ambient_temp', 'Amb Temp', np.float16],
['lab_mileage', 'LAB distance [km]', np.float16],
['obd_mileage', 'OBD distance [km]', np.float16],
['lab_fc', 'LAB FC [g]', np.float16],
['obd_fc', 'OBD FC [g]', np.float16],
['fmm_fc', 'FFM FC [g]', np.float16],
['ftir_Fc', 'FTIR FC [g]', np.float16],
['cycle', 'Test', np.object],
['fc_difference', 'accuracy', np.float16],
], columns=['db_names', 'names', 'coltype']),
}
src/uds4jrc/db.py
+ 6
− 0
View file @ e95b05e3
......@@ -19,6 +19,9 @@ geco_data = db["geco_air_data"]
spritmonitor_data = db["spritmonitor_data"]
travelcard_data = db["travelcard_data"]
atct_data = db["atct_data"]
cop_data = db["cop_data"]
obfcm_data = db["obfcm_data"]
jrcmatics_data = db["jrcmatics_data"]
# reference collections definitions
eea_reference = db["eea_properties_reference"]
......@@ -27,6 +30,9 @@ geco_reference = db["geco_air_properties_reference"]
travelcard_reference = db["travelcard_properties_reference"]
spritmonitor_reference = db["spritmonitor_properties_reference"]
atct_reference = db["atct_properties_reference"]
cop_reference = db["cop_properties_reference"]
obfcm_reference = db["obfcm_properties_reference"]
jrcmatics_reference = db["jrcmatics_properties_reference"]
# EEA views definitions
eea_2013_flattened = db["eea_2013_flattened"]
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

For further information about the platform and support, please see https://code.europa.eu/info/about