perf: add linked format checks to skip redundant field tests

csv_detective/format.py

@@ -15,6 +15,7 @@ def __init__(
         tags: list[str] = [],
         mandatory_label: bool = False,
         python_type: str = "string",
+        parent: str | None = None,
     ) -> None:
         """
         Instanciates a Format object.
@@ -29,6 +30,7 @@ def __init__(
             tags: to allow users to submit a file to only a subset of formats
             mandatory_label: whether the format can only be considered if the column passes both field and label tests
             python_type: the python type related to the format (less specific, used for downstream casting)
+            parent: optional name of a less specific format to test when this one fails or to skip when it matches
         """
         self.name: str = name
         self.description: str = description
@@ -39,6 +41,7 @@ def __init__(
         self.tags: list[str] = tags
         self.mandatory_label: bool = mandatory_label
         self.python_type: str = python_type
+        self.parent: str | None = parent
 
     def is_valid_label(self, val: str) -> float:
         return header_score(val, self.labels)
@@ -50,6 +53,12 @@ def check_proportion(cls, proportion: float | int) -> float | int:
         return proportion
 
 
+def get_leaf_formats(formats: dict[str, Format]) -> dict[str, Format]:
+    """Return formats that are not parent of any other format (most specific first)."""
+    parents = {fmt.parent for fmt in formats.values() if fmt.parent}
+    return {name: fmt for name, fmt in formats.items() if name not in parents}
+
+
 class FormatsManager:
     formats: dict[str, Format]
 
@@ -97,10 +106,14 @@ def __init__(
                         else getattr(module, "proportion", 1)
                     )
                 },
+                parent=getattr(module, "parent", None),
             )
             for label in format_labels
         }
 
+    def get_leaf_formats(self) -> dict[str, Format]:
+        return get_leaf_formats(self.formats)
+
     def get_formats_from_tags(self, tags: list[str]) -> dict[str, Format]:
         return {
             label: fmt

csv_detective/formats/date_fr.py

@@ -3,6 +3,7 @@
 from csv_detective.parsing.text import _process_text
 
 proportion = 1
+parent = "date"
 description = "Full text date in French"
 tags = ["fr", "temp"]
 labels = {"date": 1}

csv_detective/formats/datetime_rfc822.py

@@ -3,6 +3,7 @@
 from csv_detective.formats.datetime_aware import labels  # noqa
 
 proportion = 1
+parent = "datetime_aware"
 description = "Datetime in the RFC822 format"
 tags = ["temp", "type"]
 python_type = "datetime"

csv_detective/formats/geojson.py

@@ -1,6 +1,7 @@
 import json
 
 proportion = 1
+parent = "json"
 description = "JSON object in the [GeoJSON](https://fr.wikipedia.org/wiki/GeoJSON) format"
 tags = ["geo"]
 python_type = "json"

csv_detective/formats/latitude_l93.py

@@ -5,6 +5,7 @@
 from csv_detective.formats.latitude_wgs import SHARED_LATITUDE_LABELS
 
 proportion = 1
+parent = "float"
 description = "Latitude in the Lambert 93 format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/latitude_wgs.py

@@ -3,6 +3,7 @@
 from csv_detective.formats.int import _is as is_int
 
 proportion = 1
+parent = "float"
 description = "Latitude in the WGS format"
 tags = ["geo"]
 mandatory_label = True

csv_detective/formats/latitude_wgs_fr_metropole.py

@@ -1,6 +1,7 @@
 from csv_detective.formats.latitude_wgs import _is as is_latitude, labels  # noqa
 
 proportion = 1
+parent = "latitude_wgs"
 description = "Latitude within the French metropole bounds in the WGS format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/longitude_l93.py

@@ -5,6 +5,7 @@
 from csv_detective.formats.longitude_wgs import SHARED_LONGITUDE_LABELS
 
 proportion = 1
+parent = "float"
 description = "Longitude in the Lambert 93 format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/longitude_wgs.py

@@ -3,6 +3,7 @@
 from csv_detective.formats.int import _is as is_int
 
 proportion = 1
+parent = "float"
 description = "Longitude in the WGS format"
 tags = ["geo"]
 mandatory_label = True

csv_detective/formats/longitude_wgs_fr_metropole.py

@@ -1,6 +1,7 @@
 from csv_detective.formats.longitude_wgs import _is as is_longitude, labels  # noqa
 
 proportion = 1
+parent = "longitude_wgs"
 description = "Longitude within the French metropole bounds in the WGS format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/year.py

@@ -1,4 +1,5 @@
 proportion = 1
+parent = "int"
 description = "Year"
 tags = ["temp"]
 python_type = "int"

csv_detective/parsing/columns.py

@@ -8,7 +8,7 @@
 import pyarrow.parquet as pq
 from more_itertools import peekable
 
-from csv_detective.format import Format
+from csv_detective.format import Format, get_leaf_formats
 from csv_detective.parsing.csv import CHUNK_SIZE
 from csv_detective.utils import display_logs_depending_process_time
 
@@ -75,6 +75,51 @@ def apply_test_func(serie: pd.Series, test_func: Callable, _range: int):
             )
 
 
+def _test_column_with_linked_checks(
+    serie: pd.Series,
+    formats: dict[str, Format],
+    leaf_formats: dict[str, Format],
+    *,
+    skipna: bool,
+    limited_output: bool,
+    zero_if_too_low: bool,
+    verbose: bool,
+) -> dict[str, float]:
+    results: dict[str, float] = {}
+
+    for format_name, format in leaf_formats.items():
+        results[format_name] = test_col_val(
+            serie,
+            format,
+            skipna=skipna,
+            limited_output=limited_output,
+            zero_if_too_low=zero_if_too_low,
+            verbose=verbose,
+        )
+
+    for format_name, _score in reversed(sorted(results.items(), key=lambda item: item[1])):
+        current = format_name
+        parent_name = formats[current].parent
+        while parent_name is not None and parent_name in formats:
+            if parent_name in results:
+                break
+            if results[current] > 0:
+                results[parent_name] = results[current]
+            else:
+                results[parent_name] = test_col_val(
+                    serie,
+                    formats[parent_name],
+                    skipna=skipna,
+                    limited_output=limited_output,
+                    zero_if_too_low=zero_if_too_low,
+                    verbose=verbose,
+                )
+            current = parent_name
+            parent_name = formats[current].parent
+
+    return {name: results.get(name, 0.0) for name in formats}
+
+
 def test_col(
     table: pd.DataFrame,
     formats: dict[str, Format],
@@ -87,32 +132,32 @@ def test_col(
     if verbose:
         start = time()
         logging.info("Testing columns to get formats")
-    return_table = pd.DataFrame(columns=table.columns)
-    for idx, (label, format) in enumerate(formats.items()):
+    leaf_formats = get_leaf_formats(formats)
+    column_results: dict[str, dict[str, float]] = {}
+    nb_cols = len(table.columns)
+    for idx, column in enumerate(table.columns):
         if verbose:
-            start_type = time()
-            logging.info(f"\t- Starting with format '{label}'")
-        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
-        # => the following needs to change, "apply" means all columns are tested for one type at once
-        for col in table.columns:
-            return_table.loc[label, col] = test_col_val(
-                table[col],
-                format,
-                skipna=skipna,
-                zero_if_too_low=zero_if_too_low,
-                limited_output=limited_output,
-                verbose=verbose,
-            )
+            start_col = time()
+            logging.info(f"\t- Starting with column '{column}' ({idx + 1}/{nb_cols})")
+        column_results[column] = _test_column_with_linked_checks(
+            table[column],
+            formats,
+            leaf_formats,
+            skipna=skipna,
+            limited_output=limited_output,
+            zero_if_too_low=zero_if_too_low,
+            verbose=verbose,
+        )
         if verbose:
             display_logs_depending_process_time(
-                f'\t> Done with format "{label}" in {round(time() - start_type, 3)}s ({idx + 1}/{len(formats)})',
-                time() - start_type,
+                f'\t> Done with column "{column}" in {round(time() - start_col, 3)}s',
+                time() - start_col,
             )
     if verbose:
         display_logs_depending_process_time(
             f"Done testing columns in {round(time() - start, 3)}s", time() - start
         )
-    return return_table
+    return pd.DataFrame(column_results)
 
 
 def test_label(columns: list[str], formats: dict[str, Format], verbose: bool = False):

tests/test_linked_checks.py

@@ -0,0 +1,119 @@
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+
+from csv_detective.format import Format, FormatsManager, get_leaf_formats
+from csv_detective.parsing.columns import test_col as col_test
+
+
+def _make_format(name: str, *, parent: str | None = None) -> Format:
+    return Format(
+        name=name,
+        description=name,
+        func=MagicMock(return_value=True),
+        _test_values={True: ["a"], False: ["b"]},
+        parent=parent,
+    )
+
+
+def test_get_leaf_formats_excludes_parents():
+    formats = {
+        "float": _make_format("float"),
+        "latitude_wgs": _make_format("latitude_wgs", parent="float"),
+        "latitude_wgs_fr_metropole": _make_format(
+            "latitude_wgs_fr_metropole", parent="latitude_wgs"
+        ),
+        "email": _make_format("email"),
+    }
+    leaves = get_leaf_formats(formats)
+    assert set(leaves) == {"latitude_wgs_fr_metropole", "email"}
+
+
+def test_parent_score_propagated_when_child_matches():
+    child_func = MagicMock(return_value=True)
+    parent_func = MagicMock(return_value=True)
+    formats = {
+        "float": Format(
+            name="float",
+            description="float",
+            func=parent_func,
+            _test_values={True: ["1.0"], False: ["x"]},
+        ),
+        "latitude_wgs": Format(
+            name="latitude_wgs",
+            description="latitude_wgs",
+            func=MagicMock(return_value=True),
+            _test_values={True: ["45.0"], False: ["x"]},
+            parent="float",
+        ),
+        "latitude_wgs_fr_metropole": Format(
+            name="latitude_wgs_fr_metropole",
+            description="latitude_wgs_fr_metropole",
+            func=child_func,
+            _test_values={True: ["45.0"], False: ["x"]},
+            parent="latitude_wgs",
+        ),
+    }
+    table = pd.DataFrame({"col": ["45.0"] * 10})
+
+    with patch(
+        "csv_detective.parsing.columns.test_col_val",
+        side_effect=lambda serie, fmt, **kwargs: 1.0
+        if fmt.name == "latitude_wgs_fr_metropole"
+        else 0.0,
+    ) as mock_test_col_val:
+        result = col_test(table, formats, limited_output=True)
+
+    assert result.loc["latitude_wgs_fr_metropole", "col"] == 1.0
+    assert result.loc["latitude_wgs", "col"] == 1.0
+    assert result.loc["float", "col"] == 1.0
+    tested_formats = {call.args[1].name for call in mock_test_col_val.call_args_list}
+    assert tested_formats == {"latitude_wgs_fr_metropole"}
+
+
+def test_parent_tested_when_child_scores_zero():
+    formats = {
+        "float": Format(
+            name="float",
+            description="float",
+            func=lambda v: str(v).replace(".", "", 1).isdigit(),
+            _test_values={True: ["1.0"], False: ["x"]},
+        ),
+        "latitude_wgs": Format(
+            name="latitude_wgs",
+            description="latitude_wgs",
+            func=lambda v: False,
+            _test_values={True: ["45.0"], False: ["x"]},
+            parent="float",
+        ),
+        "latitude_wgs_fr_metropole": Format(
+            name="latitude_wgs_fr_metropole",
+            description="latitude_wgs_fr_metropole",
+            func=lambda v: False,
+            _test_values={True: ["45.0"], False: ["x"]},
+            parent="latitude_wgs",
+        ),
+    }
+    table = pd.DataFrame({"col": ["1.0"] * 10})
+    result = col_test(table, formats, limited_output=True)
+
+    assert result.loc["latitude_wgs_fr_metropole", "col"] == 0.0
+    assert result.loc["latitude_wgs", "col"] == 0.0
+    assert result.loc["float", "col"] == 1.0
+
+
+def test_output_dataframe_includes_all_formats():
+    fmtm = FormatsManager()
+    table = pd.DataFrame({"a": ["1"], "b": ["test@example.com"]})
+    result = col_test(table, fmtm.formats, limited_output=True)
+
+    assert set(result.index) == set(fmtm.formats)
+    assert list(result.columns) == ["a", "b"]
+    assert all(pd.api.types.is_float_dtype(dtype) for dtype in result.dtypes)
+
+
+def test_formats_manager_loads_parent_from_module():
+    fmtm = FormatsManager()
+    assert fmtm.formats["latitude_wgs_fr_metropole"].parent == "latitude_wgs"
+    assert fmtm.formats["geojson"].parent == "json"
+    assert fmtm.formats["float"].parent is None