From 4431776f43221ed4088c4bfdc3bdb737b7b0df69 Mon Sep 17 00:00:00 2001 From: Thomas Vliagkoftis <thomas.vliagkoftis@gmail.com> Date: Wed, 7 Sep 2022 11:24:42 +0200 Subject: [PATCH] add obfcm importers --- importers/obfcm_importer.py | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 importers/obfcm_importer.py diff --git a/importers/obfcm_importer.py b/importers/obfcm_importer.py new file mode 100755 index 0000000..8eacba8 --- /dev/null +++ b/importers/obfcm_importer.py @@ -0,0 +1,47 @@ +#!/usr/bin/env/ python + +from collections import defaultdict + +import pandas as pd +from munch import Munch + +from src.uds4jrc.config import Config +from src.uds4jrc.db import obfcm_data, obfcm_reference + +file_config = Munch(Config.HDV_LABS) + + +def _import_data(): + df = pd.read_excel( + file_config.file_path, + header=0, + dtype=defaultdict( + lambda: object, file_config.column_properties.iloc[:, 1:3].values), + names=file_config.column_properties.iloc[:, 0].values, + skiprows=file_config.skip_rows + ) + + df['file_year'] = file_config.year + df['data'] = file_config.data + df['version'] = 1.0 + + obfcm_data.insert_many(df.to_dict('records')) + + +def _import_reference_data(): + df = file_config.column_properties + reference_docs = [] + + for i in range(len(df)): + reference_docs.append({ + 'db_property_name': df.iloc[i, 0], + 'file_property_name': df.iloc[i, 1], + 'file_year': file_config.year, + 'data': file_config.data + }) + + obfcm_reference.insert_many(reference_docs) + + +_import_data() +_import_reference_data() -- GitLab