main.py

#!/usr/bin/env python

from collections import defaultdict

import pandas as pd
from munch import Munch

from uds4jrc.config import Config
from uds4jrc.db import eea_raw_data

file_config = Munch(Config.EEA_2019)

chunks = pd.read_csv(
    '/eos/jeodpp/data/projects/LEGENT/transfer/EEA_passenger_cars_2019_final_AMI.csv',
    delimiter=file_config.delimiter,
    header=0,
    dtype=defaultdict(
        lambda: object, file_config.column_properties.iloc[:, 1:3].values),
    names=file_config.column_properties.iloc[:, 0].values,
    chunksize=100000,
    encoding=file_config.encoding,
    index_col=False,
    low_memory=False
)

for df in chunks:

    del df['id']
    df['file_year'] = file_config.year
    df['version'] = 1.0

    eea_raw_data.insert_many(df.to_dict('records'))