add realworld data impoerters

importers/gecoair_importer.py

+#!/usr/bin/env/ python
+
+from collections import defaultdict
+
+import pandas as pd
+from munch import Munch
+
+from src.uds4jrc.config import Config
+from src.uds4jrc.db import geco_data, geco_reference
+
+file_config = Munch(Config.GECO_2021)
+
+
+def _import_data():
+    df = pd.read_excel(
+        file_config.file_path,
+        header=0,
+        dtype=defaultdict(
+            lambda: object, file_config.column_properties.iloc[:, 1:3].values),
+        names=file_config.column_properties.iloc[:, 0].values,
+        index_col=False
+    )
+
+    del df['id']
+    df['file_year'] = file_config.year
+    df['version'] = 1.0
+
+    #  Need to calculate the year for Geco Air 2021 data only
+    df['year'] = df.apply (lambda row: str(row['date'])[0:4], axis=1)
+
+    geco_data.insert_many(df.to_dict('records'))
+
+
+def _import_reference_data():
+    df = file_config.column_properties
+    reference_docs = []
+
+    for i in range(len(df)):
+        reference_docs.append({
+            'db_property_name': df.iloc[i, 0],
+            'file_property_name': df.iloc[i, 1],
+            'file_year': file_config.year
+        })
+
+    geco_reference.insert_many(reference_docs)
+
+
+_import_data()
+# _import_reference_data()

importers/spritmonitor_importer.py

+#!/usr/bin/env/ python
+
+from collections import defaultdict
+
+import pandas as pd
+from munch import Munch
+
+from src.uds4jrc.config import Config
+from src.uds4jrc.db import spritmonitor_data, spritmonitor_reference
+
+file_config = Munch(Config.SPRITMONITOR_2021)
+
+
+def _import_data():
+    df = pd.read_excel(
+        file_config.file_path,
+        header=0,
+        dtype=defaultdict(
+            lambda: object, file_config.column_properties.iloc[:, 1:3].values),
+        names=file_config.column_properties.iloc[:, 0].values,
+        index_col=False
+    )
+
+    del df['id']
+    df['file_year'] = file_config.year
+    df['version'] = 1.0
+
+    spritmonitor_data.insert_many(df.to_dict('records'))
+
+
+def _import_reference_data():
+    df = file_config.column_properties
+    reference_docs = []
+
+    for i in range(len(df)):
+        reference_docs.append({
+            'db_property_name': df.iloc[i, 0],
+            'file_property_name': df.iloc[i, 1],
+            'file_year': file_config.year
+        })
+
+    spritmonitor_reference.insert_many(reference_docs)
+
+
+_import_data()
+_import_reference_data()

importers/travelcard_importer.py

+#!/usr/bin/env/ python
+
+from collections import defaultdict
+
+import pandas as pd
+from munch import Munch
+
+from src.uds4jrc.config import Config
+from src.uds4jrc.db import travelcard_data, travelcard_reference
+
+file_config = Munch(Config.TRAVELCARD_DIESEL)
+
+
+def _import_data():
+    df = pd.read_excel(
+        file_config.file_path,
+        header=0,
+        dtype=defaultdict(
+            lambda: object, file_config.column_properties.iloc[:, 1:3].values),
+        names=file_config.column_properties.iloc[:, 0].values,
+        index_col=False
+    )
+
+    df['file_data_category'] = file_config.file_data_category
+    df['file_year'] = file_config.year
+    df['version'] = 1.0
+
+    travelcard_data.insert_many(df.to_dict('records'))
+
+
+def _import_reference_data():
+    df = file_config.column_properties
+    reference_docs = []
+
+    for i in range(len(df)):
+        reference_docs.append({
+            'db_property_name': df.iloc[i, 0],
+            'file_property_name': df.iloc[i, 1],
+            'file_data_category': file_config.file_data_category
+        })
+
+    travelcard_reference.insert_many(reference_docs)
+
+
+_import_data()
+_import_reference_data()