Production
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
"""Import eligible root-level CSV files into their form tables.
|
||||
|
||||
Usage: python -m scripts.import_csv_data [--only-empty]
|
||||
|
||||
Without a flag every form table is REPLACED by its CSV. ``--only-empty`` is the safe
|
||||
mode for a database that already holds work: a table that has any rows is left
|
||||
untouched, and only empty tables are loaded (e.g. the product lists).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.database.session import AsyncSessionLocal
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXCLUDED_FILES = {"stock.csv"}
|
||||
CSV_TABLES = {
|
||||
"Weekly_Report.csv": ("weekly_report", ["employee_name", "report_date", "headquarter", "month", "no_of_days_worked", "no_of_doctor_called", "no_of_chemist_called", "pob_booked", "secondary_sales", "pob_objective", "secondary_objective", "product_offer_scheme_booked", "product_offer_scheme_objective"]),
|
||||
"FIELD_ACTIVITY_REPORT.csv": ("field_activity_report", ["name", "headquarter", "month", "report_date", "station_worked", "no_of_doctors_called", "no_of_chemists_pds_called", "doctor_call_avg", "chemist_call_avg"]),
|
||||
"Fare_Chart.csv": ("fare_chart", ["name", "headquarter", "serial_no", "report_date", "from_location", "to_location", "kilometers_km"]),
|
||||
"Daily_Call_Report.csv": ("daily_call_report_csv", ["name", "headquarter", "report_date", "month", "doctor_sr_no", "doctor_name", "qualification", "product_promote", "product_sample", "remarks", "chemist_sr_no", "chemist_name", "pob", "stockist_visit", "order_value"]),
|
||||
"CORE_Dr._CALL_CONVERSION.csv": ("core_dr_call_conversion", ["name", "headquarter", "month", "doctor_name", "date_of_visit", "product_targeted", "status"]),
|
||||
"AREA_COVERED.csv": ("area_covered", ["name", "headquarter", "month", "sale_status", "area_to_cover", "sale_objective", "status"]),
|
||||
"Sales_and_Stock_Statement.csv": ("sales_and_stock_statement", ["name", "headquarter", "report_date", "month", "serial_no", "product_name", "opening", "receipt", "sale", "closing", "value_in_pts"]),
|
||||
"TOUR_PLAN.csv": ("tour_plan", ["name", "headquarter", "serial_no", "report_date", "from_location", "to_location", "pob", "stockiest_visit"]),
|
||||
"STANDARD_TOUR_PLAN.csv": ("standard_tour_plan", ["name", "headquarter", "serial_no", "place"]),
|
||||
"sales_projection.csv": ("sales_projection", ["name", "headquarter", "report_date", "month", "serial_no", "product_name", "pts", "secondary_qty", "value", "closing", "secondary_value", "aug_sales", "value_2"]),
|
||||
}
|
||||
SERIAL_COLUMNS = {"serial_no", "doctor_sr_no", "chemist_sr_no"}
|
||||
|
||||
|
||||
def read_rows(path: Path, columns: list[str]) -> list[tuple[int, dict[str, str]]]:
|
||||
with path.open(newline="", encoding="utf-8-sig") as file:
|
||||
reader = csv.reader(file)
|
||||
next(reader, None)
|
||||
rows: list[tuple[int, dict[str, str]]] = []
|
||||
for row in reader:
|
||||
data = {
|
||||
column: (row[index] if index < len(row) else "").strip()
|
||||
for index, column in enumerate(columns)
|
||||
}
|
||||
if any(value for column, value in data.items() if column not in SERIAL_COLUMNS):
|
||||
rows.append((len(rows) + 1, data))
|
||||
return rows
|
||||
|
||||
|
||||
async def apply_sales_projection_account(db) -> None:
|
||||
await db.execute(
|
||||
text(
|
||||
'''UPDATE "sales_projection" AS projection
|
||||
SET name = account.full_name,
|
||||
headquarter = headquarters.name
|
||||
FROM "user" AS account
|
||||
LEFT JOIN headquarters ON headquarters.id = account.headquarters_id
|
||||
WHERE account.role = 'MR' '''
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def import_csv_data(only_empty: bool = False) -> None:
|
||||
csv_files = sorted(path for path in ROOT.glob("*.csv") if path.name not in EXCLUDED_FILES)
|
||||
async with AsyncSessionLocal() as db:
|
||||
imported_rows = 0
|
||||
for path in csv_files:
|
||||
table_name, columns = CSV_TABLES[path.name]
|
||||
if only_empty:
|
||||
count = await db.execute(text(f'SELECT COUNT(*) FROM "{table_name}"'))
|
||||
existing = count.scalar_one()
|
||||
if existing:
|
||||
print(f"{table_name}: skipped, already has {existing} rows")
|
||||
continue
|
||||
else:
|
||||
await db.execute(text(f'DELETE FROM "{table_name}"'))
|
||||
source_columns = columns[4:] if table_name == "sales_projection" else columns
|
||||
rows = read_rows(path, source_columns)
|
||||
if table_name == "sales_projection":
|
||||
rows = [
|
||||
(row_number, {"name": "", "headquarter": "", "report_date": "", "month": "", **data})
|
||||
for row_number, data in rows
|
||||
]
|
||||
if rows:
|
||||
column_names = ", ".join(f'"{column}"' for column in columns)
|
||||
parameters = ", ".join(f":{column}" for column in columns)
|
||||
statement = text(
|
||||
f'INSERT INTO "{table_name}" '
|
||||
f'(id, source_row_number, {column_names}) '
|
||||
f'VALUES (:id, :source_row_number, {parameters})'
|
||||
)
|
||||
await db.execute(
|
||||
statement,
|
||||
[
|
||||
{"id": uuid.uuid4(), "source_row_number": row_number, **data}
|
||||
for row_number, data in rows
|
||||
],
|
||||
)
|
||||
imported_rows += len(rows)
|
||||
print(f"{table_name}: {len(rows)} rows")
|
||||
if table_name == "sales_projection":
|
||||
await apply_sales_projection_account(db)
|
||||
await db.commit()
|
||||
print(f"Imported {imported_rows} rows from {len(csv_files)} CSV files.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument(
|
||||
"--only-empty",
|
||||
action="store_true",
|
||||
help="load only tables that have no rows; never delete existing data",
|
||||
)
|
||||
asyncio.run(import_csv_data(parser.parse_args().only_empty))
|
||||
Reference in New Issue
Block a user