Code development platform for open source projects from the European Union institutions

Skip to content
Snippets Groups Projects
Commit 1b239ddb authored by Thomas Vliagkoftis's avatar Thomas Vliagkoftis
Browse files

add realworld data impoerters

parent e1c8dcc7
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/env/ python
from collections import defaultdict
import pandas as pd
from munch import Munch
from src.uds4jrc.config import Config
from src.uds4jrc.db import geco_data, geco_reference
file_config = Munch(Config.GECO_2021)
def _import_data():
df = pd.read_excel(
file_config.file_path,
header=0,
dtype=defaultdict(
lambda: object, file_config.column_properties.iloc[:, 1:3].values),
names=file_config.column_properties.iloc[:, 0].values,
index_col=False
)
del df['id']
df['file_year'] = file_config.year
df['version'] = 1.0
# Need to calculate the year for Geco Air 2021 data only
df['year'] = df.apply (lambda row: str(row['date'])[0:4], axis=1)
geco_data.insert_many(df.to_dict('records'))
def _import_reference_data():
df = file_config.column_properties
reference_docs = []
for i in range(len(df)):
reference_docs.append({
'db_property_name': df.iloc[i, 0],
'file_property_name': df.iloc[i, 1],
'file_year': file_config.year
})
geco_reference.insert_many(reference_docs)
_import_data()
# _import_reference_data()
#!/usr/bin/env/ python
from collections import defaultdict
import pandas as pd
from munch import Munch
from src.uds4jrc.config import Config
from src.uds4jrc.db import spritmonitor_data, spritmonitor_reference
file_config = Munch(Config.SPRITMONITOR_2021)
def _import_data():
df = pd.read_excel(
file_config.file_path,
header=0,
dtype=defaultdict(
lambda: object, file_config.column_properties.iloc[:, 1:3].values),
names=file_config.column_properties.iloc[:, 0].values,
index_col=False
)
del df['id']
df['file_year'] = file_config.year
df['version'] = 1.0
spritmonitor_data.insert_many(df.to_dict('records'))
def _import_reference_data():
df = file_config.column_properties
reference_docs = []
for i in range(len(df)):
reference_docs.append({
'db_property_name': df.iloc[i, 0],
'file_property_name': df.iloc[i, 1],
'file_year': file_config.year
})
spritmonitor_reference.insert_many(reference_docs)
_import_data()
_import_reference_data()
#!/usr/bin/env/ python
from collections import defaultdict
import pandas as pd
from munch import Munch
from src.uds4jrc.config import Config
from src.uds4jrc.db import travelcard_data, travelcard_reference
file_config = Munch(Config.TRAVELCARD_DIESEL)
def _import_data():
df = pd.read_excel(
file_config.file_path,
header=0,
dtype=defaultdict(
lambda: object, file_config.column_properties.iloc[:, 1:3].values),
names=file_config.column_properties.iloc[:, 0].values,
index_col=False
)
df['file_data_category'] = file_config.file_data_category
df['file_year'] = file_config.year
df['version'] = 1.0
travelcard_data.insert_many(df.to_dict('records'))
def _import_reference_data():
df = file_config.column_properties
reference_docs = []
for i in range(len(df)):
reference_docs.append({
'db_property_name': df.iloc[i, 0],
'file_property_name': df.iloc[i, 1],
'file_data_category': file_config.file_data_category
})
travelcard_reference.insert_many(reference_docs)
_import_data()
_import_reference_data()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment