diff --git a/README.md b/README.md index c2b0c14..8671eb9 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,26 @@ The program creates a `python` dictionary with the following information : "format": "code_commune", "score": 1.0 }, + "Date de création": { + "python_type": "date", + "format": "date", + "score": 1.0, + # How to read every value of that column: a column that no single format reads + # is not detected as a date at all. Present on every `date` / `datetime_naive` / + # `datetime_aware` column the inference ran on, and absent everywhere else, so a + # consumer always needs a path without it: parquet columns come typed from the + # file itself, `datetime_rfc822` has a single shape, a custom proportion + # deliberately makes the format tolerant, and an analysis produced before this + # key existed is replayed as is. + # A format without the "csvd:" prefix can be handed to datetime.strptime as it is, + # and reads the same there as through csv-detective. A prefixed one cannot, and has + # to go through `csv_detective.formats.date.parse`: text months (strptime only knows + # the abbreviated ones of the process locale, we know the full and abbreviated ones + # of several languages), optional parts written "[.%f]" for a column whose source + # only prints fractional seconds when they are non-zero, and named time zones, which + # strptime reads then drops, leaving a naive datetime where we return an aware one. + "date_format": "%d/%m/%Y" + }, }, "columns_labels": { # Property that return detection from header columns "Code commune": { diff --git a/csv_detective/detection/formats.py b/csv_detective/detection/formats.py index 0621d74..7e1f295 100755 --- a/csv_detective/detection/formats.py +++ b/csv_detective/detection/formats.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import Any, Callable, Iterable import numpy as np import pandas as pd @@ -23,6 +24,49 @@ ) +def _winning_format(detections: dict | list[dict], limited_output: bool) -> str: + """The format that won for a column, whatever shape prepare_output_dict produced.""" + if limited_output: + return detections["format"] + return max(detections, key=lambda d: d["score"], default={"format": "string"})["format"] + + +def _infer_column_formats( + formats: dict[str, Format], + scores_table_fields: pd.DataFrame, + values_of: Callable[[str], Iterable[Any]], +) -> dict[str, dict[str, str]]: + """Runs the column-wide inference of the formats that have one, and zeroes those that fail. + + A format that cannot say how to read the whole column has not detected it: the value-by-value + test only says that each value looks valid on its own, not that a single format reads them all. + """ + inferable = [ + (label, fmt) + for label, fmt in formats.items() + if fmt.infer is not None + # a format the user made tolerant cannot be pinned down: asking for a single format that + # reads every value contradicts the proportion of failures they just allowed + and fmt.proportion == 1 + and label in scores_table_fields.index + ] + inferred: dict[str, dict[str, str]] = {} + for col in scores_table_fields.columns: + candidates = [ + (label, fmt) for label, fmt in inferable if scores_table_fields.loc[label, col] + ] + if not candidates: + continue + values = list(values_of(col)) + for label, fmt in candidates: + column_format = fmt.infer(values) + if column_format is None: + scores_table_fields.loc[label, col] = 0.0 + else: + inferred.setdefault(col, {})[label] = column_format + return inferred + + def detect_formats( table: pd.DataFrame | pq.ParquetFile, analysis: dict, @@ -84,11 +128,23 @@ def detect_formats( na_values=na_values, verbose=verbose, ) + if analysis.get("engine") == "parquet": + # parquet types columns itself, so there is nothing to infer and nothing to read the + # format with: cast_df_chunks hands the file over to pandas untouched + inferred_formats: dict[str, dict[str, str]] = {} + else: + inferred_formats = _infer_column_formats( + formats, + scores_table_fields, + (lambda col: table[col].dropna().unique()) + if col_values is None + else (lambda col: col_values[col].index.dropna()), + ) analysis["columns_fields"] = prepare_output_dict(scores_table_fields, limited_output) analysis["unique_values"] = {} if col_values is None: for col in table.columns: - if analysis["columns_fields"][col]["format"] == "json" and all( + if _winning_format(analysis["columns_fields"][col], limited_output) == "json" and all( value.startswith("[") for value in table[col] ): unique = extract_unique_from_multicat(table[col]) @@ -98,7 +154,7 @@ def detect_formats( analysis["unique_values"][col] = list(table[col].dropna().unique()) else: for col in col_values.keys(): - if analysis["columns_fields"][col]["format"] == "json" and all( + if _winning_format(analysis["columns_fields"][col], limited_output) == "json" and all( value.startswith("[") for value in col_values[col].index ): unique = extract_unique_from_multicat(col_values[col].index.to_series()) @@ -165,4 +221,10 @@ def detect_formats( for header, col_metadata in analysis["columns"].items(): analysis["formats"][col_metadata["format"]].append(header) + for col_name, detections in analysis["columns"].items(): + for detection in [detections] if limited_output else detections: + column_format = inferred_formats.get(col_name, {}).get(detection["format"]) + if column_format is not None: + detection["date_format"] = column_format + return analysis, col_values diff --git a/csv_detective/format.py b/csv_detective/format.py index 8080757..d19a820 100755 --- a/csv_detective/format.py +++ b/csv_detective/format.py @@ -1,4 +1,4 @@ -from typing import Any, Callable +from typing import Any, Callable, Iterable from csv_detective.parsing.text import header_score @@ -10,6 +10,7 @@ def __init__( description: str, func: Callable[[Any], bool], _test_values: dict[bool, list[str]], + infer: Callable[[Iterable[Any]], str | None] | None = None, labels: dict[str, float] = {}, proportion: float | int = 1, tags: list[str] = [], @@ -24,6 +25,10 @@ def __init__( descrption: a short description of the format. func: the value test for the format (returns whether a string is valid). _test_values: lists of valid and invalid values, used in the tests + infer: for formats whose detection needs the whole column, returns the single way to + read every value of it, or None if there is none (the column is then not of this + format). Only used when proportion is 1: a format the user made tolerant cannot + be pinned down to a single way of reading every value. labels: the dict of hint headers and their credibilty for the header score (NB: credibility is relative witin a single format, should be used to rank the valid labels) proportion: the tolerance (between 0 and 1) to say a column is valid for a format. (1 => 100% of the column has to pass the func check for the column to be considered valid) tags: to allow users to submit a file to only a subset of formats @@ -34,6 +39,7 @@ def __init__( self.description: str = description self.func: Callable[[Any], bool] = func self._test_values: dict[bool, list[str]] = _test_values + self.infer: Callable[[Iterable[Any]], str | None] | None = infer self.labels: dict[str, float] = labels self.proportion: float = self.check_proportion(proportion) self.tags: list[str] = tags @@ -80,6 +86,7 @@ def __init__( name=label, func=(module := getattr(formats, label))._is, _test_values=module._test_values, + infer=getattr(module, "_infer", None), **{ attr: val for attr in ["labels", "description", "tags", "mandatory_label", "python_type"] diff --git a/csv_detective/formats/date.py b/csv_detective/formats/date.py index 774f23b..74cc60f 100755 --- a/csv_detective/formats/date.py +++ b/csv_detective/formats/date.py @@ -1,108 +1,442 @@ -import re -from datetime import datetime - -from dateparser import parse as date_parser -from dateutil.parser import ParserError -from dateutil.parser import parse as dateutil_parser - -proportion = 1 -description = "Date (flexible formats)" -tags = ["temp", "type"] -python_type = "date" -SHARED_DATE_LABELS = { - "date": 1, - "mise à jour": 1, - "modifie": 1, - "maj": 0.75, - "datemaj": 1, - "update": 1, - "created": 1, - "modified": 1, -} -labels = SHARED_DATE_LABELS | { - "jour": 0.75, - "periode": 0.75, - "dpc": 0.5, - "yyyymmdd": 1, - "aaaammjj": 1, -} - - -def date_casting(val: str) -> datetime | None: - """For performance reasons, we try first with dateutil and fallback on dateparser""" - try: - return dateutil_parser(val) - except ParserError: - return date_parser(val) - except Exception: - return None - - -threshold = 0.3 -seps = r"[\s/\-\*_\|;.,]" -# matches JJ-MM-AAAA with any of the listed separators -jjmmaaaa_pattern = r"^(0[1-9]|[12][0-9]|3[01])SEP(0[1-9]|1[0-2])SEP((19|20)\d{2})$".replace( - "SEP", seps -) -# matches AAAA-MM-JJ with any of the listed separators OR NO SEPARATOR -aaaammjj_pattern = r"^((19|20)\d{2})SEP(0[1-9]|1[0-2])SEP(0[1-9]|[12][0-9]|3[01])$".replace( - "SEP", seps + "?" -) -# matches JJ-mmm-AAAA and JJ-mmm...mm-AAAA with any of the listed separators OR NO SEPARATOR -string_month_pattern = ( - r"^(0[1-9]|[12][0-9]|3[01])SEP(jan|fev|feb|mar|avr|apr" - r"|mai|may|jun|jui|jul|aou|aug|sep|oct|nov|dec|janvier|fevrier|mars|avril|" - r"mai|juin|juillet|aout|septembre|octobre|novembre|decembre)SEP" - r"([0-9]{2}$|(19|20)[0-9]{2}$)" -).replace("SEP", seps + "?") - - -def _is(val) -> bool: - # many early stops, to cut processing time - # and avoid the costly use of date_casting as much as possible - # /!\ timestamps are considered ints, not dates - if not isinstance(val, str) or len(val) > 20 or len(val) < 8: - return False - # if it's a usual date pattern - if ( - # with this syntax, if any of the first value is True, the next ones are not computed - bool(re.match(jjmmaaaa_pattern, val)) - or bool(re.match(aaaammjj_pattern, val)) - or bool(re.match(string_month_pattern, val, re.IGNORECASE)) - ): - return True - if re.match(r"^-?\d+[\.|,]\d+$", val): - # regular floats are excluded - return False - # not enough digits => not a date (slightly arbitrary) - if sum([char.isdigit() for char in val]) / len(val) < threshold: - return False - # last resort - res = date_casting(val) - if not res or res.hour or res.minute or res.second: - return False - return True - - -_test_values = { - True: [ - "1960-08-07", - "12/02/2007", - "15 jan 1985", - "15 décembre 1985", - "02 05 2003", - "20030502", - "2003.05.02", - "1993-12/02", - ], - False: [ - "1993-1993-1993", - "39-10-1993", - "19-15-1993", - "15 tambour 1985", - "12152003", - "20031512", - "02052003", - "6.27367393749392839", - ], -} +import re +from datetime import datetime, timedelta, timezone +from functools import lru_cache +from typing import Any, Callable, Iterable + +from dateparser import parse as date_parser +from dateutil.parser import ParserError +from dateutil.parser import parse as dateutil_parser +from unidecode import unidecode + +proportion = 1 +description = "Date (flexible formats)" +tags = ["temp", "type"] +python_type = "date" +SHARED_DATE_LABELS = { + "date": 1, + "mise à jour": 1, + "modifie": 1, + "maj": 0.75, + "datemaj": 1, + "update": 1, + "created": 1, + "modified": 1, +} +labels = SHARED_DATE_LABELS | { + "jour": 0.75, + "periode": 0.75, + "dpc": 0.5, + "yyyymmdd": 1, + "aaaammjj": 1, +} + + +def date_casting(val: str) -> datetime | None: + """For performance reasons, we try first with dateutil and fallback on dateparser""" + try: + return dateutil_parser(val) + except ParserError: + return date_parser(val) + except Exception: + return None + + +# Formats that strptime cannot express are prefixed with this marker and read by parse() itself. +# So far only text months: strptime only knows the ones of the process locale. +CUSTOM_PREFIX = "csvd:" + +SEPARATORS = " /-*_|;.," +MIN_LENGTH = 6 # "1/2/85", the shortest shape we read: no padding and a two-digit year +MAX_LENGTH = 20 + +# The only shape made of nothing but digits, and hence the only one a year window has to guard: +# eight bare digits are just a number, of which about one in thirty reads as a valid YYYYMMDD. +# A value that carries separators is specific enough on its own, so no year bounds it — museum +# records and civil registers are routinely dated well before 1900. +_NO_SEPARATOR_DATE = "%Y%m%d" +MIN_YEAR = 1900 +MAX_YEAR = 2099 + +# Standard forms come from the system locale tables (the CLDR data that PHP and Java also use), +# tolerated variants are what published files actually contain. Adding a language is one entry. +MONTH_NAMES: dict[str, list[list[str]]] = { + "fr": [ + ["janvier", "janv", "jan"], + ["fevrier", "fevr", "fev"], + ["mars", "mar"], + ["avril", "avr"], + ["mai"], + ["juin"], + ["juillet", "juil"], + ["aout", "aou"], + ["septembre", "sept", "sep"], + ["octobre", "oct"], + ["novembre", "nov"], + ["decembre", "dec"], + ], + "en": [ + ["january", "jan"], + ["february", "feb"], + ["march", "mar"], + ["april", "apr"], + ["may"], + ["june", "jun"], + ["july", "jul"], + ["august", "aug"], + ["september", "sept", "sep"], + ["october", "oct"], + ["november", "nov"], + ["december", "dec"], + ], +} + + +def build_month_index(month_names: dict[str, list[list[str]]]) -> dict[str, int]: + index: dict[str, int] = {} + ambiguous: set[str] = set() + for months in month_names.values(): + for number, names in enumerate(months, start=1): + for name in names: + if index.setdefault(name, number) != number: + # a spelling that means two different months depending on the language cannot + # be read; no current entry does, this guards the languages added later + ambiguous.add(name) + for name in ambiguous: + del index[name] + return index + + +MONTHS = build_month_index(MONTH_NAMES) + +_DIRECTIVES = { + # the ordinal suffix is part of how English writes a day ("31st december 2022") + "%d": r"(?P\d{1,2})(?:st|nd|rd|th)?", + "%b": r"(?P[^\W\d_]+)\.?", + "%Y": r"(?P\d{4})", + "%y": r"(?P\d{2})", +} +_TOKENS = re.compile(r"%.|.", re.DOTALL) + + +@lru_cache(maxsize=None) +def _compiled(fmt: str) -> re.Pattern: + return re.compile( + "".join(_DIRECTIVES.get(token, re.escape(token)) for token in _TOKENS.findall(fmt)) + ) + + +def _parse_custom(val: str, fmt: str) -> datetime | None: + match = _compiled(fmt).fullmatch(val) + if match is None: + return None + month = MONTHS.get(unidecode(match["month"]).lower()) + if month is None: + return None + groups = match.groupdict() + if groups.get("year"): + year = int(groups["year"]) + else: + # same two-digit window as strptime's %y + short_year = int(groups["short_year"]) + year = 2000 + short_year if short_year < 69 else 1900 + short_year + try: + return datetime(year, month, int(groups["day"])) + except ValueError: + return None + + +_OPTIONAL_PART = re.compile(r"\[([^\]]*)\]") + + +@lru_cache(maxsize=None) +def _variants(fmt: str) -> tuple[str, ...]: + """Expands the optional parts of a format, the most complete one first. + + A column whose source only prints fractional seconds when they are non-zero uses one format + with an optional part, not two competing ones. + """ + match = _OPTIONAL_PART.search(fmt) + if match is None: + return (fmt,) + head, tail = fmt[: match.start()], fmt[match.end() :] + return _variants(head + match.group(1) + tail) + _variants(head + tail) + + +# Zone names we resolve ourselves rather than leaving to strptime, which reads %Z against the +# machine's own zone (time.tzname) and drops it anyway: the same file would not read the same on +# a laptop in Paris as on a server in UTC. Abbreviations are not standardised and many of them +# name two zones — CST is -6 in the US and +8 in China, IST is +5:30, +2 or +1, EST is -5 but +# also +10 in Australia — so only the ones with a single meaning are listed here. +NAMED_ZONES = { + "UTC": 0, + "GMT": 0, + "WET": 0, + "WEST": 1, + "CET": 1, + "CEST": 2, + "EET": 2, + "EEST": 3, +} +_ZONE_NAME = re.compile(r"\b([A-Za-z]+)$") + + +def _without_prefix(fmt: str) -> str: + return fmt[len(CUSTOM_PREFIX) :] if fmt.startswith(CUSTOM_PREFIX) else fmt + + +def _read(val: str, fmt: str) -> datetime | None: + fmt = _without_prefix(fmt) + if "%b" in fmt: + return _parse_custom(val, fmt) + if "%Z" in fmt: + name = _ZONE_NAME.search(val) + offset = NAMED_ZONES.get(name.group(1).upper()) if name else None + if offset is None: + return None + # read what precedes the name, then apply the offset the name stands for + naive = _read(val[: name.start()].rstrip(), fmt[: fmt.index("%Z")].rstrip()) + return naive and naive.replace(tzinfo=timezone(timedelta(hours=offset))) + try: + return datetime.strptime(val, fmt) + except (ValueError, TypeError): + return None + + +def parse(val: str, fmt: str) -> datetime | None: + """Reads a value with one of our formats, the custom ones included.""" + for variant in _variants(fmt): + parsed = _read(val, variant) + if parsed is None: + continue + if _without_prefix(variant).startswith(_NO_SEPARATOR_DATE) and not ( + MIN_YEAR <= parsed.year <= MAX_YEAR + ): + continue + return parsed + return None + + +# the order is the preference: an ambiguous value is read day-first, as French files are the +# overwhelming majority of what csv-detective is fed +_DAY_OR_MONTH_FIRST = ( + "%d{sep}%m{sep}%Y", + "%m{sep}%d{sep}%Y", +) +_DAY_OR_MONTH_FIRST_SHORT = ( + "%d{sep}%m{sep}%y", + "%m{sep}%d{sep}%y", +) +# ISO first, but the year can also be followed by the day ("2022-31-12") +_YEAR_FIRST = ( + "%Y{sep}%m{sep}%d", + "%Y{sep}%d{sep}%m", +) +_TEXT_MONTH_TEMPLATES = ( + "%d{sep}%b{sep}%Y", + "%d{sep}%b{sep}%y", +) + + +# the full stop of an abbreviated month is not a separator ("15 janv. 1985") +_ABBREVIATION_DOT = re.compile(r"(?<=[^\W\d_])\.") +_SEPARATOR_RUN = re.compile(f"[{re.escape(SEPARATORS)}]+") + + +def separator_of(val: str) -> str | None: + """The single separator the value uses, "" if it uses none, None if it mixes several. + + A separator is a run of characters rather than a single one, so a value that spaces its + separators out ("1789 / 07 / 14") uses one separator, not two mixed together. + """ + found = set(_SEPARATOR_RUN.findall(_ABBREVIATION_DOT.sub("", val))) + if len(found) > 1: + return None + return found.pop() if found else "" + + +# What makes strptime read a template differently from us, and hence what CUSTOM_PREFIX marks. +# Each entry is something _read has to handle itself, so a consumer that hands the format to +# strptime would get a wrong answer rather than an error — which is the whole point of marking: +# "%b" strptime only knows the abbreviated months of the process locale, we know the +# abbreviated and the full ones of every language in MONTH_NAMES +# "[" an optional part, which strptime has no syntax for; parse() expands it first +# "%Z" strptime accepts the zone name then drops it, leaving a naive datetime where _read +# returns an aware one +_UNREADABLE_BY_STRPTIME = ("%b", "[", "%Z") + + +def _marked(template: str) -> str: + """Marks the formats strptime cannot read as-is, so that consumers can tell them apart.""" + if any(marker in template for marker in _UNREADABLE_BY_STRPTIME): + return CUSTOM_PREFIX + template + return template + + +# every template starts with a digit, so a value that does not cannot be read by any of them. +# Ruling those out with one match is what keeps the hot path off strptime, which costs an order +# of magnitude more than a regex. +_STARTS_LIKE_DATE = re.compile(r"\d") +_HAS_LETTER = re.compile(r"[^\W\d_]") + + +@lru_cache(maxsize=None) +def _numeric_templates(sep: str, year_first: bool, short_year: bool) -> tuple[str, ...]: + if not sep: + # without a separator, only the year-first order is unambiguous enough to be trusted + return (_NO_SEPARATOR_DATE,) + if year_first: + templates = _YEAR_FIRST + elif short_year: + templates = _DAY_OR_MONTH_FIRST_SHORT + else: + templates = _DAY_OR_MONTH_FIRST + return tuple(template.format(sep=sep) for template in templates) + + +@lru_cache(maxsize=None) +def _text_month_templates(sep: str) -> tuple[str, ...]: + return tuple(_marked(template.format(sep=sep)) for template in _TEXT_MONTH_TEMPLATES) + + +def date_templates(val: str, *, text_month: bool = True) -> tuple[str, ...]: + """Every format the value could plausibly be read with, most preferred first. + + Only the shapes that can possibly read the value are returned: one with a letter can only + have a text month, and %Y wants four digits where %d wants one or two, so the position of + the first separator settles the order. A two-digit last component is %y, not %Y. + Trying the others would be as many failed strptime. + """ + if not _STARTS_LIKE_DATE.match(val): + return () + sep = separator_of(val) + if sep is None: + return () + if _HAS_LETTER.search(val): + return _text_month_templates(sep) if text_month and sep else () + # both hold for a value with no separator at all: "".index("") is 0, and rindex("") is the + # whole length, so neither a four-digit head nor a two-digit tail is ever found + year_first = val.index(sep) == 4 + short_year = not year_first and len(val) - val.rindex(sep) - len(sep) == 2 + return _numeric_templates(sep, year_first, short_year) + + +# the time is either written with colons, minutes and seconds not necessarily padded, or packed +# into six bare digits next to a packed date ("20210622T102010") +_DATETIME_SPLIT = re.compile(r"(?P.+?)(?P[T ])(?P