-
Kostis ANAGNOSTOPOULOS authoredKostis ANAGNOSTOPOULOS authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
main.py 786 B
#!/usr/bin/env python
from collections import defaultdict
import pandas as pd
from munch import Munch
from uds4jrc.config import Config
from uds4jrc.db import eea_raw_data
file_config = Munch(Config.EEA_2019)
chunks = pd.read_csv(
'/eos/jeodpp/data/projects/LEGENT/transfer/EEA_passenger_cars_2019_final_AMI.csv',
delimiter=file_config.delimiter,
header=0,
dtype=defaultdict(
lambda: object, file_config.column_properties.iloc[:, 1:3].values),
names=file_config.column_properties.iloc[:, 0].values,
chunksize=100000,
encoding=file_config.encoding,
index_col=False,
low_memory=False
)
for df in chunks:
del df['id']
df['file_year'] = file_config.year
df['version'] = 1.0
eea_raw_data.insert_many(df.to_dict('records'))