perf: cast dates with the format inferred by csv-detective

pyproject.toml

@@ -12,7 +12,7 @@ dependencies = [
     "aiohttp>=3.10.3",
     "asyncpg>=0.29.0",
     "coloredlogs>=15.0.1",
-    "csv-detective==0.12.0",
+    "csv-detective",
     "dateparser>=1.1.7",
     "humanfriendly>=10.0",
     "json-stream>=2.3.3",
@@ -94,6 +94,11 @@ local_scheme = "no-local-version"
 [tool.uv]
 constraint-dependencies = ["urllib3>=2.7.0"]
 
+# `cast()` only takes the inferred date format from that branch on. Pin it back to a released
+# version once datagouv/csv-detective#245 is merged and published.
+[tool.uv.sources]
+csv-detective = { git = "https://github.com/datagouv/csv-detective", branch = "improve_date_parsing" }
+
 [tool.uv.build-backend]
 module-name = "udata_hydra"
 module-root = ""

tests/test_analysis/test_analysis_csv.py

@@ -382,7 +382,12 @@ def create_analysis(scan: dict) -> dict:
                 "rows": [["1", "2022-11-03"], ["5", "2025-11-02"]],
                 "columns": {
                     "a": {"score": 1.0, "format": "int", "python_type": "int"},
-                    "b": {"score": 1.0, "format": "date", "python_type": "date"},
+                    "b": {
+                        "score": 1.0,
+                        "format": "date",
+                        "python_type": "date",
+                        "date_format": "%Y-%m-%d",
+                    },
                 },
                 "formats": {"int": ["a"], "date": ["b"]},
             },

tests/test_conversion/test_csv_to_db.py

@@ -81,6 +81,31 @@ async def test_csv_to_db_complex_type_casting(db, line_expected, clean_db, fake_
     assert dict(res[0]) == {k: v for k, v in zip(cols, expected)}
 
 
+@pytest.mark.parametrize(
+    "values_expected",
+    (
+        # a value no other value of the column disambiguates is read day-first
+        (["05/03/2022"], [date(2022, 3, 5)]),
+        # a single day-only value settles the format of the whole column...
+        (["03/04/2022", "25/04/2022"], [date(2022, 4, 3), date(2022, 4, 25)]),
+        # ...and the same value is read the other way around in a month-first column
+        (["03/04/2022", "04/25/2022"], [date(2022, 3, 4), date(2022, 4, 25)]),
+    ),
+)
+async def test_csv_to_db_ambiguous_date_column(db, values_expected, clean_db, fake_check):
+    check = await fake_check()
+    values, expected = values_expected
+    rows = "\n".join(f"{index};{value}" for index, value in enumerate(values, start=1))
+    with NamedTemporaryFile() as fp:
+        fp.write(f"int;date\n{rows}".encode("utf-8"))
+        fp.seek(0)
+        file = Csv(file_name=os.path.basename(fp.name), resource_id=RESOURCE_ID)
+        await file.inspect()
+        table = await file.to_db(check=check)
+    res = await db.fetch(f'SELECT date FROM "{table.table_name}" ORDER BY __id')
+    assert [row["date"] for row in res] == expected
+
+
 async def test_basic_sql_injection(db, clean_db, fake_check):
     check = await fake_check()
     # tries to execute

udata_hydra/utils/casting.py

@@ -9,14 +9,16 @@
 log = logging.getLogger("udata-hydra")
 
 
-def _smart_cast(_type: str, value, cast_json: bool = True, failsafe: bool = False) -> Any:
+def _smart_cast(
+    _type: str, date_format: str | None, value, cast_json: bool = True, failsafe: bool = False
+) -> Any:
     try:
         if value is None or value == "":
             return None
         if _type == "json" and not cast_json:
             # handing JSON as string to postgres, which casts it itself
             return value
-        return cast(value, _type)
+        return cast(value, _type, date_format)
     except ValueError as e:
         if not failsafe:
             raise e
@@ -29,27 +31,38 @@ def iter_tabular_rows(
 ) -> Iterator[list | dict]:
     # because we need the iterator multiple times, not possible to
     # handle db, parquet and geojson through the same iteration
-    columns = {col: v["python_type"] for col, v in inspection["columns"].items()}
+    column_names: list[str] = []
+    python_types: list[str] = []
+    date_formats: list[str | None] = []
+    for col, spec in inspection["columns"].items():
+        column_names.append(col)
+        python_types.append(spec["python_type"])
+        date_formats.append(spec.get("date_format"))
+
     with Reader(file_path, inspection) as reader:
         for line in reader:
             if line:
                 if not as_dict:
                     yield [
                         _smart_cast(
-                            _type,
+                            python_type,
+                            date_format,
                             value if isinstance(value, str) or value is None else str(value),
                             cast_json=cast_json,
                             failsafe=False,
                         )
-                        for _type, value in zip(columns.values(), line)
+                        for python_type, date_format, value in zip(python_types, date_formats, line)
                     ]
                 else:
                     yield {
                         col: _smart_cast(
-                            _type,
+                            python_type,
+                            date_format,
                             value if isinstance(value, str) or value is None else str(value),
                             cast_json=cast_json,
                             failsafe=False,
                         )
-                        for (col, _type), value in zip(columns.items(), line)
+                        for col, python_type, date_format, value in zip(
+                            column_names, python_types, date_formats, line
+                        )
                     }