"""Import eligible root-level CSV files into their form tables. Usage: python -m scripts.import_csv_data [--only-empty] Without a flag every form table is REPLACED by its CSV. ``--only-empty`` is the safe mode for a database that already holds work: a table that has any rows is left untouched, and only empty tables are loaded (e.g. the product lists). """ import argparse import asyncio import csv import uuid from pathlib import Path from sqlalchemy import text from app.database.session import AsyncSessionLocal ROOT = Path(__file__).resolve().parent.parent EXCLUDED_FILES = {"stock.csv"} CSV_TABLES = { "Weekly_Report.csv": ("weekly_report", ["employee_name", "report_date", "headquarter", "month", "no_of_days_worked", "no_of_doctor_called", "no_of_chemist_called", "pob_booked", "secondary_sales", "pob_objective", "secondary_objective", "product_offer_scheme_booked", "product_offer_scheme_objective"]), "FIELD_ACTIVITY_REPORT.csv": ("field_activity_report", ["name", "headquarter", "month", "report_date", "station_worked", "no_of_doctors_called", "no_of_chemists_pds_called", "doctor_call_avg", "chemist_call_avg"]), "Fare_Chart.csv": ("fare_chart", ["name", "headquarter", "serial_no", "report_date", "from_location", "to_location", "kilometers_km"]), "Daily_Call_Report.csv": ("daily_call_report_csv", ["name", "headquarter", "report_date", "month", "doctor_sr_no", "doctor_name", "qualification", "product_promote", "product_sample", "remarks", "chemist_sr_no", "chemist_name", "pob", "stockist_visit", "order_value"]), "CORE_Dr._CALL_CONVERSION.csv": ("core_dr_call_conversion", ["name", "headquarter", "month", "doctor_name", "date_of_visit", "product_targeted", "status"]), "AREA_COVERED.csv": ("area_covered", ["name", "headquarter", "month", "sale_status", "area_to_cover", "sale_objective", "status"]), "Sales_and_Stock_Statement.csv": ("sales_and_stock_statement", ["name", "headquarter", "report_date", "month", "serial_no", "product_name", "opening", "receipt", "sale", "closing", "value_in_pts"]), "TOUR_PLAN.csv": ("tour_plan", ["name", "headquarter", "serial_no", "report_date", "from_location", "to_location", "pob", "stockiest_visit"]), "STANDARD_TOUR_PLAN.csv": ("standard_tour_plan", ["name", "headquarter", "serial_no", "place"]), "sales_projection.csv": ("sales_projection", ["name", "headquarter", "report_date", "month", "serial_no", "product_name", "pts", "secondary_qty", "value", "closing", "secondary_value", "aug_sales", "value_2"]), } SERIAL_COLUMNS = {"serial_no", "doctor_sr_no", "chemist_sr_no"} def read_rows(path: Path, columns: list[str]) -> list[tuple[int, dict[str, str]]]: with path.open(newline="", encoding="utf-8-sig") as file: reader = csv.reader(file) next(reader, None) rows: list[tuple[int, dict[str, str]]] = [] for row in reader: data = { column: (row[index] if index < len(row) else "").strip() for index, column in enumerate(columns) } if any(value for column, value in data.items() if column not in SERIAL_COLUMNS): rows.append((len(rows) + 1, data)) return rows async def apply_sales_projection_account(db) -> None: await db.execute( text( '''UPDATE "sales_projection" AS projection SET name = account.full_name, headquarter = headquarters.name FROM "user" AS account LEFT JOIN headquarters ON headquarters.id = account.headquarters_id WHERE account.role = 'MR' ''' ) ) async def import_csv_data(only_empty: bool = False) -> None: csv_files = sorted(path for path in ROOT.glob("*.csv") if path.name not in EXCLUDED_FILES) async with AsyncSessionLocal() as db: imported_rows = 0 for path in csv_files: table_name, columns = CSV_TABLES[path.name] if only_empty: count = await db.execute(text(f'SELECT COUNT(*) FROM "{table_name}"')) existing = count.scalar_one() if existing: print(f"{table_name}: skipped, already has {existing} rows") continue else: await db.execute(text(f'DELETE FROM "{table_name}"')) source_columns = columns[4:] if table_name == "sales_projection" else columns rows = read_rows(path, source_columns) if table_name == "sales_projection": rows = [ (row_number, {"name": "", "headquarter": "", "report_date": "", "month": "", **data}) for row_number, data in rows ] if rows: column_names = ", ".join(f'"{column}"' for column in columns) parameters = ", ".join(f":{column}" for column in columns) statement = text( f'INSERT INTO "{table_name}" ' f'(id, source_row_number, {column_names}) ' f'VALUES (:id, :source_row_number, {parameters})' ) await db.execute( statement, [ {"id": uuid.uuid4(), "source_row_number": row_number, **data} for row_number, data in rows ], ) imported_rows += len(rows) print(f"{table_name}: {len(rows)} rows") if table_name == "sales_projection": await apply_sales_projection_account(db) await db.commit() print(f"Imported {imported_rows} rows from {len(csv_files)} CSV files.") if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--only-empty", action="store_true", help="load only tables that have no rows; never delete existing data", ) asyncio.run(import_csv_data(parser.parse_args().only_empty))