diff --git a/pyproject.toml b/pyproject.toml index fdd927ae8..6d687f42b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "aiohttp>=3.10.3", "asyncpg>=0.29.0", "coloredlogs>=15.0.1", - "csv-detective==0.12.0", + "csv-detective", "dateparser>=1.1.7", "humanfriendly>=10.0", "json-stream>=2.3.3", @@ -94,6 +94,11 @@ local_scheme = "no-local-version" [tool.uv] constraint-dependencies = ["urllib3>=2.7.0"] +# `cast()` only takes the inferred date format from that branch on. Pin it back to a released +# version once datagouv/csv-detective#245 is merged and published. +[tool.uv.sources] +csv-detective = { git = "https://github.com/datagouv/csv-detective", branch = "improve_date_parsing" } + [tool.uv.build-backend] module-name = "udata_hydra" module-root = "" diff --git a/tests/test_analysis/test_analysis_csv.py b/tests/test_analysis/test_analysis_csv.py index 221d54853..1121e755a 100644 --- a/tests/test_analysis/test_analysis_csv.py +++ b/tests/test_analysis/test_analysis_csv.py @@ -382,7 +382,12 @@ def create_analysis(scan: dict) -> dict: "rows": [["1", "2022-11-03"], ["5", "2025-11-02"]], "columns": { "a": {"score": 1.0, "format": "int", "python_type": "int"}, - "b": {"score": 1.0, "format": "date", "python_type": "date"}, + "b": { + "score": 1.0, + "format": "date", + "python_type": "date", + "date_format": "%Y-%m-%d", + }, }, "formats": {"int": ["a"], "date": ["b"]}, }, diff --git a/tests/test_conversion/test_csv_to_db.py b/tests/test_conversion/test_csv_to_db.py index 5aa044fcc..719b4d6a8 100644 --- a/tests/test_conversion/test_csv_to_db.py +++ b/tests/test_conversion/test_csv_to_db.py @@ -81,6 +81,31 @@ async def test_csv_to_db_complex_type_casting(db, line_expected, clean_db, fake_ assert dict(res[0]) == {k: v for k, v in zip(cols, expected)} +@pytest.mark.parametrize( + "values_expected", + ( + # a value no other value of the column disambiguates is read day-first + (["05/03/2022"], [date(2022, 3, 5)]), + # a single day-only value settles the format of the whole column... + (["03/04/2022", "25/04/2022"], [date(2022, 4, 3), date(2022, 4, 25)]), + # ...and the same value is read the other way around in a month-first column + (["03/04/2022", "04/25/2022"], [date(2022, 3, 4), date(2022, 4, 25)]), + ), +) +async def test_csv_to_db_ambiguous_date_column(db, values_expected, clean_db, fake_check): + check = await fake_check() + values, expected = values_expected + rows = "\n".join(f"{index};{value}" for index, value in enumerate(values, start=1)) + with NamedTemporaryFile() as fp: + fp.write(f"int;date\n{rows}".encode("utf-8")) + fp.seek(0) + file = Csv(file_name=os.path.basename(fp.name), resource_id=RESOURCE_ID) + await file.inspect() + table = await file.to_db(check=check) + res = await db.fetch(f'SELECT date FROM "{table.table_name}" ORDER BY __id') + assert [row["date"] for row in res] == expected + + async def test_basic_sql_injection(db, clean_db, fake_check): check = await fake_check() # tries to execute diff --git a/udata_hydra/utils/casting.py b/udata_hydra/utils/casting.py index 159ff2919..5a1ddd31a 100644 --- a/udata_hydra/utils/casting.py +++ b/udata_hydra/utils/casting.py @@ -9,14 +9,16 @@ log = logging.getLogger("udata-hydra") -def _smart_cast(_type: str, value, cast_json: bool = True, failsafe: bool = False) -> Any: +def _smart_cast( + _type: str, date_format: str | None, value, cast_json: bool = True, failsafe: bool = False +) -> Any: try: if value is None or value == "": return None if _type == "json" and not cast_json: # handing JSON as string to postgres, which casts it itself return value - return cast(value, _type) + return cast(value, _type, date_format) except ValueError as e: if not failsafe: raise e @@ -29,27 +31,38 @@ def iter_tabular_rows( ) -> Iterator[list | dict]: # because we need the iterator multiple times, not possible to # handle db, parquet and geojson through the same iteration - columns = {col: v["python_type"] for col, v in inspection["columns"].items()} + column_names: list[str] = [] + python_types: list[str] = [] + date_formats: list[str | None] = [] + for col, spec in inspection["columns"].items(): + column_names.append(col) + python_types.append(spec["python_type"]) + date_formats.append(spec.get("date_format")) + with Reader(file_path, inspection) as reader: for line in reader: if line: if not as_dict: yield [ _smart_cast( - _type, + python_type, + date_format, value if isinstance(value, str) or value is None else str(value), cast_json=cast_json, failsafe=False, ) - for _type, value in zip(columns.values(), line) + for python_type, date_format, value in zip(python_types, date_formats, line) ] else: yield { col: _smart_cast( - _type, + python_type, + date_format, value if isinstance(value, str) or value is None else str(value), cast_json=cast_json, failsafe=False, ) - for (col, _type), value in zip(columns.items(), line) + for col, python_type, date_format, value in zip( + column_names, python_types, date_formats, line + ) } diff --git a/uv.lock b/uv.lock index 018b2aaf6..abcdf5e0b 100644 --- a/uv.lock +++ b/uv.lock @@ -458,8 +458,8 @@ wheels = [ [[package]] name = "csv-detective" -version = "0.12.0" -source = { registry = "https://pypi.org/simple" } +version = "0.12.1.dev20" +source = { git = "https://github.com/datagouv/csv-detective?branch=improve_date_parsing#79e3685f342aaa0ecfa7d2f9fdbd559c2e48124e" } dependencies = [ { name = "charset-normalizer" }, { name = "dateparser" }, @@ -477,9 +477,6 @@ dependencies = [ { name = "unidecode" }, { name = "xlrd" }, ] -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a6/53a360a34f4d42d9cbe9fc03a8c42bb0db535e74d49b0c4a72df29d77bbe/csv_detective-0.12.0-py3-none-any.whl", hash = "sha256:9c5f6c9715061b1912cadb04de6f2ef3e49b1210ed176660b5674b0b30e1ec50", size = 222567, upload-time = "2026-07-08T13:26:13.488Z" }, -] [[package]] name = "dateparser" @@ -2089,7 +2086,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.29.0" }, { name = "boto3", specifier = ">=1.35.0" }, { name = "coloredlogs", specifier = ">=15.0.1" }, - { name = "csv-detective", specifier = "==0.12.0" }, + { name = "csv-detective", git = "https://github.com/datagouv/csv-detective?branch=improve_date_parsing" }, { name = "dateparser", specifier = ">=1.1.7" }, { name = "humanfriendly", specifier = ">=10.0" }, { name = "json-stream", specifier = ">=2.3.3" },