Code development platform for open source projects from the European Union institutions

Skip to content
Snippets Groups Projects
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
main.py 786 B
#!/usr/bin/env python

from collections import defaultdict

import pandas as pd
from munch import Munch

from uds4jrc.config import Config
from uds4jrc.db import eea_raw_data

file_config = Munch(Config.EEA_2019)

chunks = pd.read_csv(
    '/eos/jeodpp/data/projects/LEGENT/transfer/EEA_passenger_cars_2019_final_AMI.csv',
    delimiter=file_config.delimiter,
    header=0,
    dtype=defaultdict(
        lambda: object, file_config.column_properties.iloc[:, 1:3].values),
    names=file_config.column_properties.iloc[:, 0].values,
    chunksize=100000,
    encoding=file_config.encoding,
    index_col=False,
    low_memory=False
)

for df in chunks:

    del df['id']
    df['file_year'] = file_config.year
    df['version'] = 1.0

    eea_raw_data.insert_many(df.to_dict('records'))