feat: improve date parsing

README.md

@@ -64,6 +64,20 @@ The program creates a `python` dictionary with the following information :
             "format": "code_commune",
             "score": 1.0
         },
+        "Date de création": {
+            "python_type": "date",
+            "format": "date",
+            "score": 1.0,
+            # How to read every value of that column: a column that no single format reads
+            # is not detected as a date at all. Present on every `date` / `datetime_naive` /
+            # `datetime_aware` column, except when a custom proportion made the format
+            # tolerant; `datetime_rfc822` has a single shape and carries no format.
+            # A format prefixed with "csvd:" is not readable by datetime.strptime and has to
+            # go through csv-detective: text months (strptime only knows the ones of the
+            # process locale) and optional parts, written "[.%f]" for a column whose source
+            # only prints fractional seconds when they are non-zero.
+            "date_format": "%d/%m/%Y"
+        },
     },
     "columns_labels": {                     # Property that return detection from header columns
         "Code commune": {

csv_detective/detection/formats.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from typing import Any, Callable, Iterable
 
 import numpy as np
 import pandas as pd
@@ -23,6 +24,49 @@
 )
 
 
+def _winning_format(detections: dict | list[dict], limited_output: bool) -> str:
+    """The format that won for a column, whatever shape prepare_output_dict produced."""
+    if limited_output:
+        return detections["format"]
+    return max(detections, key=lambda d: d["score"], default={"format": "string"})["format"]
+
+
+def _infer_column_formats(
+    formats: dict[str, Format],
+    scores_table_fields: pd.DataFrame,
+    values_of: Callable[[str], Iterable[Any]],
+) -> dict[str, dict[str, str]]:
+    """Runs the column-wide inference of the formats that have one, and zeroes those that fail.
+
+    A format that cannot say how to read the whole column has not detected it: the value-by-value
+    test only says that each value looks valid on its own, not that a single format reads them all.
+    """
+    inferable = [
+        (label, fmt)
+        for label, fmt in formats.items()
+        if fmt.infer is not None
+        # a format the user made tolerant cannot be pinned down: asking for a single format that
+        # reads every value contradicts the proportion of failures they just allowed
+        and fmt.proportion == 1
+        and label in scores_table_fields.index
+    ]
+    inferred: dict[str, dict[str, str]] = {}
+    for col in scores_table_fields.columns:
+        candidates = [
+            (label, fmt) for label, fmt in inferable if scores_table_fields.loc[label, col]
+        ]
+        if not candidates:
+            continue
+        values = list(values_of(col))
+        for label, fmt in candidates:
+            column_format = fmt.infer(values)
+            if column_format is None:
+                scores_table_fields.loc[label, col] = 0.0
+            else:
+                inferred.setdefault(col, {})[label] = column_format
+    return inferred
+
+
 def detect_formats(
     table: pd.DataFrame | pq.ParquetFile,
     analysis: dict,
@@ -84,11 +128,23 @@ def detect_formats(
             na_values=na_values,
             verbose=verbose,
         )
+    if analysis.get("engine") == "parquet":
+        # parquet types columns itself, so there is nothing to infer and nothing to read the
+        # format with: cast_df_chunks hands the file over to pandas untouched
+        inferred_formats: dict[str, dict[str, str]] = {}
+    else:
+        inferred_formats = _infer_column_formats(
+            formats,
+            scores_table_fields,
+            (lambda col: table[col].dropna().unique())
+            if col_values is None
+            else (lambda col: col_values[col].index.dropna()),
+        )
     analysis["columns_fields"] = prepare_output_dict(scores_table_fields, limited_output)
     analysis["unique_values"] = {}
     if col_values is None:
         for col in table.columns:
-            if analysis["columns_fields"][col]["format"] == "json" and all(
+            if _winning_format(analysis["columns_fields"][col], limited_output) == "json" and all(
                 value.startswith("[") for value in table[col]
             ):
                 unique = extract_unique_from_multicat(table[col])
@@ -98,7 +154,7 @@ def detect_formats(
                 analysis["unique_values"][col] = list(table[col].dropna().unique())
     else:
         for col in col_values.keys():
-            if analysis["columns_fields"][col]["format"] == "json" and all(
+            if _winning_format(analysis["columns_fields"][col], limited_output) == "json" and all(
                 value.startswith("[") for value in col_values[col].index
             ):
                 unique = extract_unique_from_multicat(col_values[col].index.to_series())
@@ -165,4 +221,10 @@ def detect_formats(
         for header, col_metadata in analysis["columns"].items():
             analysis["formats"][col_metadata["format"]].append(header)
 
+    for col_name, detections in analysis["columns"].items():
+        for detection in [detections] if limited_output else detections:
+            column_format = inferred_formats.get(col_name, {}).get(detection["format"])
+            if column_format is not None:
+                detection["date_format"] = column_format
+
     return analysis, col_values

csv_detective/format.py

@@ -1,4 +1,4 @@
-from typing import Any, Callable
+from typing import Any, Callable, Iterable
 
 from csv_detective.parsing.text import header_score
 
@@ -10,6 +10,7 @@ def __init__(
         description: str,
         func: Callable[[Any], bool],
         _test_values: dict[bool, list[str]],
+        infer: Callable[[Iterable[Any]], str | None] | None = None,
         labels: dict[str, float] = {},
         proportion: float | int = 1,
         tags: list[str] = [],
@@ -24,6 +25,10 @@ def __init__(
             descrption: a short description of the format.
             func: the value test for the format (returns whether a string is valid).
             _test_values: lists of valid and invalid values, used in the tests
+            infer: for formats whose detection needs the whole column, returns the single way to
+                read every value of it, or None if there is none (the column is then not of this
+                format). Only used when proportion is 1: a format the user made tolerant cannot
+                be pinned down to a single way of reading every value.
             labels: the dict of hint headers and their credibilty for the header score (NB: credibility is relative witin a single format, should be used to rank the valid labels)
             proportion: the tolerance (between 0 and 1) to say a column is valid for a format. (1 => 100% of the column has to pass the func check for the column to be considered valid)
             tags: to allow users to submit a file to only a subset of formats
@@ -34,6 +39,7 @@ def __init__(
         self.description: str = description
         self.func: Callable[[Any], bool] = func
         self._test_values: dict[bool, list[str]] = _test_values
+        self.infer: Callable[[Iterable[Any]], str | None] | None = infer
         self.labels: dict[str, float] = labels
         self.proportion: float = self.check_proportion(proportion)
         self.tags: list[str] = tags
@@ -80,6 +86,7 @@ def __init__(
                 name=label,
                 func=(module := getattr(formats, label))._is,
                 _test_values=module._test_values,
+                infer=getattr(module, "_infer", None),
                 **{
                     attr: val
                     for attr in ["labels", "description", "tags", "mandatory_label", "python_type"]