perf: add linked format checks to skip redundant field tests

csv_detective/format.py

@@ -15,6 +15,7 @@ def __init__(
         tags: list[str] = [],
         mandatory_label: bool = False,
         python_type: str = "string",
+        parent: str | None = None,
     ) -> None:
         """
         Instanciates a Format object.
@@ -29,6 +30,7 @@ def __init__(
             tags: to allow users to submit a file to only a subset of formats
             mandatory_label: whether the format can only be considered if the column passes both field and label tests
             python_type: the python type related to the format (less specific, used for downstream casting)
+            parent: optional name of a less specific format to test when this one fails or to skip when it matches
         """
         self.name: str = name
         self.description: str = description
@@ -39,6 +41,7 @@ def __init__(
         self.tags: list[str] = tags
         self.mandatory_label: bool = mandatory_label
         self.python_type: str = python_type
+        self.parent: str | None = parent
 
     def is_valid_label(self, val: str) -> float:
         return header_score(val, self.labels)
@@ -50,6 +53,12 @@ def check_proportion(cls, proportion: float | int) -> float | int:
         return proportion
 
 
+def get_leaf_formats(formats: dict[str, Format]) -> dict[str, Format]:
+    """Return formats that are not parent of any other format (most specific first)."""
+    parents = {fmt.parent for fmt in formats.values() if fmt.parent}
+    return {name: fmt for name, fmt in formats.items() if name not in parents}
+
+
 class FormatsManager:
     formats: dict[str, Format]
 
@@ -97,10 +106,14 @@ def __init__(
                         else getattr(module, "proportion", 1)
                     )
                 },
+                parent=getattr(module, "parent", None),
             )
             for label in format_labels
         }
 
+    def get_leaf_formats(self) -> dict[str, Format]:
+        return get_leaf_formats(self.formats)
+
     def get_formats_from_tags(self, tags: list[str]) -> dict[str, Format]:
         return {
             label: fmt

csv_detective/formats/date_fr.py

@@ -3,6 +3,7 @@
 from csv_detective.parsing.text import _process_text
 
 proportion = 1
+parent = "date"
 description = "Full text date in French"
 tags = ["fr", "temp"]
 labels = {"date": 1}

csv_detective/formats/datetime_rfc822.py

@@ -3,6 +3,7 @@
 from csv_detective.formats.datetime_aware import labels  # noqa
 
 proportion = 1
+parent = "datetime_aware"
 description = "Datetime in the RFC822 format"
 tags = ["temp", "type"]
 python_type = "datetime"

csv_detective/formats/geojson.py

@@ -1,6 +1,7 @@
 import json
 
 proportion = 1
+parent = "json"
 description = "JSON object in the [GeoJSON](https://fr.wikipedia.org/wiki/GeoJSON) format"
 tags = ["geo"]
 python_type = "json"

csv_detective/formats/latitude_l93.py

@@ -5,6 +5,7 @@
 from csv_detective.formats.latitude_wgs import SHARED_LATITUDE_LABELS
 
 proportion = 1
+parent = "float"
 description = "Latitude in the Lambert 93 format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/latitude_wgs.py

@@ -3,6 +3,7 @@
 from csv_detective.formats.int import _is as is_int
 
 proportion = 1
+parent = "float"
 description = "Latitude in the WGS format"
 tags = ["geo"]
 mandatory_label = True

csv_detective/formats/latitude_wgs_fr_metropole.py

@@ -1,6 +1,7 @@
 from csv_detective.formats.latitude_wgs import _is as is_latitude, labels  # noqa
 
 proportion = 1
+parent = "latitude_wgs"
 description = "Latitude within the French metropole bounds in the WGS format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/longitude_l93.py

@@ -5,6 +5,7 @@
 from csv_detective.formats.longitude_wgs import SHARED_LONGITUDE_LABELS
 
 proportion = 1
+parent = "float"
 description = "Longitude in the Lambert 93 format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/longitude_wgs.py

@@ -3,6 +3,7 @@
 from csv_detective.formats.int import _is as is_int
 
 proportion = 1
+parent = "float"
 description = "Longitude in the WGS format"
 tags = ["geo"]
 mandatory_label = True

csv_detective/formats/longitude_wgs_fr_metropole.py

@@ -1,6 +1,7 @@
 from csv_detective.formats.longitude_wgs import _is as is_longitude, labels  # noqa
 
 proportion = 1
+parent = "longitude_wgs"
 description = "Longitude within the French metropole bounds in the WGS format"
 tags = ["fr", "geo"]
 mandatory_label = True

csv_detective/formats/year.py

@@ -1,4 +1,5 @@
 proportion = 1
+parent = "int"
 description = "Year"
 tags = ["temp"]
 python_type = "int"

csv_detective/parsing/columns.py

@@ -8,7 +8,7 @@
 import pyarrow.parquet as pq
 from more_itertools import peekable
 
-from csv_detective.format import Format
+from csv_detective.format import Format, get_leaf_formats
 from csv_detective.parsing.csv import CHUNK_SIZE
 from csv_detective.utils import display_logs_depending_process_time
 
@@ -75,6 +75,80 @@ def apply_test_func(serie: pd.Series, test_func: Callable, _range: int):
             )
 
 
+def _test_column_with_linked_checks(
+    serie: pd.Series,
+    formats: dict[str, Format],
+    leaf_formats: dict[str, Format],
+    *,
+    skipna: bool,
+    limited_output: bool,
+    zero_if_too_low: bool,
+    verbose: bool,
+) -> dict[str, float]:
+    """Score every format for one column while avoiding redundant checks.
+
+    Some formats are more specific versions of others (e.g.
+    ``float`` → ``latitude_wgs`` → ``latitude_wgs_fr_metropole``), linked
+    through each format's ``parent`` attribute. Instead of running every
+    format's value test independently, we:
+
+    1. Test only *leaf* formats first (most specific end of each chain, e.g.
+       ``latitude_wgs_fr_metropole`` — not ``float``, which is above it).
+    2. Walk up each leaf's parent chain. For each parent not yet scored:
+       - reuse the child's score for the parent if it is >= the parent's
+         ``proportion`` (skip the parent test);
+       - otherwise retest the parent (child failed, or score too low for the
+         parent's ``proportion``).
+
+    This saves work when a column matches a specialized format: one float
+    check instead of three, for example. Copying the child score to the parent
+    is an approximation (many floats are not valid latitudes) but is much
+    faster than testing every format on every column.
+    """
+    results: dict[str, float] = {}
+
+    # Step 1: run the expensive column test on every leaf format.
+    for format_name, format in leaf_formats.items():
+        results[format_name] = test_col_val(
+            serie,
+            format,
+            skipna=skipna,
+            limited_output=limited_output,
+            zero_if_too_low=zero_if_too_low,
+            verbose=verbose,
+        )
+
+    # Step 2: walk up parent chains, best-matching leaves first (highest
+    # match rate). Shared parents (e.g. float) get scored once, not retested.
+    for format_name, _score in reversed(sorted(results.items(), key=lambda item: item[1])):
+        current = format_name
+        parent_name = formats[current].parent
+        while parent_name is not None and parent_name in formats:
+            if parent_name in results:
+                break  # another leaf already scored this parent
+            # Reuse the child's score for the parent if it is >= the parent's
+            # proportion; otherwise retest the parent.
+            if results[current] >= formats[parent_name].proportion:
+                results[parent_name] = results[current]
+            else:
+                # Child failed, or score too low for parent's stricter threshold
+                # (e.g. child 85% with proportion 0.8, parent needs 100%).
+                results[parent_name] = test_col_val(
+                    serie,
+                    formats[parent_name],
+                    skipna=skipna,
+                    limited_output=limited_output,
+                    zero_if_too_low=zero_if_too_low,
+                    verbose=verbose,
+                )
+            current = parent_name
+            parent_name = formats[current].parent
+
+    missing = set(formats) - set(results)
+    assert not missing, f"Formats not scored: {missing}"
+    return {name: results[name] for name in formats}
+
+
 def test_col(
     table: pd.DataFrame,
     formats: dict[str, Format],
@@ -87,32 +161,32 @@ def test_col(
     if verbose:
         start = time()
         logging.info("Testing columns to get formats")
-    return_table = pd.DataFrame(columns=table.columns)
-    for idx, (label, format) in enumerate(formats.items()):
+    leaf_formats = get_leaf_formats(formats)
+    column_results: dict[str, dict[str, float]] = {}
+    nb_cols = len(table.columns)
+    for idx, column in enumerate(table.columns):
         if verbose:
-            start_type = time()
-            logging.info(f"\t- Starting with format '{label}'")
-        # improvement lead : put the longest tests behind and make them only if previous tests not satisfactory
-        # => the following needs to change, "apply" means all columns are tested for one type at once
-        for col in table.columns:
-            return_table.loc[label, col] = test_col_val(
-                table[col],
-                format,
-                skipna=skipna,
-                zero_if_too_low=zero_if_too_low,
-                limited_output=limited_output,
-                verbose=verbose,
-            )
+            start_col = time()
+            logging.info(f"\t- Starting with column '{column}' ({idx + 1}/{nb_cols})")
+        column_results[column] = _test_column_with_linked_checks(
+            table[column],
+            formats,
+            leaf_formats,
+            skipna=skipna,
+            limited_output=limited_output,
+            zero_if_too_low=zero_if_too_low,
+            verbose=verbose,
+        )
         if verbose:
             display_logs_depending_process_time(
-                f'\t> Done with format "{label}" in {round(time() - start_type, 3)}s ({idx + 1}/{len(formats)})',
-                time() - start_type,
+                f'\t> Done with column "{column}" in {round(time() - start_col, 3)}s',
+                time() - start_col,
             )
     if verbose:
         display_logs_depending_process_time(
             f"Done testing columns in {round(time() - start, 3)}s", time() - start
         )
-    return return_table
+    return pd.DataFrame(column_results)
 
 
 def test_label(columns: list[str], formats: dict[str, Format], verbose: bool = False):