|
| 1 | +"""Lightweight tabular-data context for thesis claim verification.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import csv |
| 6 | +import io |
| 7 | +import math |
| 8 | +from typing import Any |
| 9 | + |
| 10 | + |
| 11 | +class CsvAnalyzer: |
| 12 | + """Summarize raw CSV or TSV text for repository-claim verification.""" |
| 13 | + |
| 14 | + def __init__(self, content: str, filename: str = "") -> None: |
| 15 | + self._content = content |
| 16 | + self._filename = filename |
| 17 | + |
| 18 | + def analyze(self) -> dict[str, Any]: |
| 19 | + """Return schema, missing-value, sample, and numeric statistics.""" |
| 20 | + result: dict[str, Any] = { |
| 21 | + "filename": self._filename, |
| 22 | + "row_count": 0, |
| 23 | + "column_count": 0, |
| 24 | + "columns": [], |
| 25 | + "column_stats": {}, |
| 26 | + "error": None, |
| 27 | + } |
| 28 | + try: |
| 29 | + sample = self._content[:4096] |
| 30 | + try: |
| 31 | + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") |
| 32 | + except csv.Error: |
| 33 | + dialect = csv.excel # type: ignore[assignment] |
| 34 | + rows = list(csv.reader(io.StringIO(self._content), dialect)) |
| 35 | + except Exception as exc: |
| 36 | + result["error"] = str(exc) |
| 37 | + return result |
| 38 | + |
| 39 | + if not rows: |
| 40 | + return result |
| 41 | + |
| 42 | + columns = [column.strip() for column in rows[0]] |
| 43 | + data_rows = rows[1:] |
| 44 | + result["columns"] = columns |
| 45 | + result["column_count"] = len(columns) |
| 46 | + result["row_count"] = len(data_rows) |
| 47 | + for column_index, column_name in enumerate(columns): |
| 48 | + values = [row[column_index].strip() if column_index < len(row) else "" for row in data_rows] |
| 49 | + missing_count = sum(1 for value in values if not value) |
| 50 | + non_empty = [value for value in values if value] |
| 51 | + dtype = self._infer_type(values) |
| 52 | + stats: dict[str, Any] = { |
| 53 | + "dtype": dtype, |
| 54 | + "missing_count": missing_count, |
| 55 | + "missing_pct": round(missing_count / len(values) * 100, 1) if values else 0.0, |
| 56 | + "unique_count": len(set(non_empty)), |
| 57 | + "sample_values": list(dict.fromkeys(non_empty))[:5], |
| 58 | + } |
| 59 | + if dtype == "numeric": |
| 60 | + stats.update(self._numeric_stats(values)) |
| 61 | + result["column_stats"][column_name] = stats |
| 62 | + return result |
| 63 | + |
| 64 | + @staticmethod |
| 65 | + def _infer_type(values: list[str]) -> str: |
| 66 | + non_empty = [value for value in values if value.strip()] |
| 67 | + if not non_empty: |
| 68 | + return "empty" |
| 69 | + numeric_count = 0 |
| 70 | + for value in non_empty: |
| 71 | + try: |
| 72 | + float(value.replace(",", ".")) |
| 73 | + numeric_count += 1 |
| 74 | + except ValueError: |
| 75 | + pass |
| 76 | + return "numeric" if numeric_count / len(non_empty) >= 0.8 else "categorical" |
| 77 | + |
| 78 | + @staticmethod |
| 79 | + def _numeric_stats(values: list[str]) -> dict[str, Any]: |
| 80 | + numbers: list[float] = [] |
| 81 | + for value in values: |
| 82 | + try: |
| 83 | + if value.strip(): |
| 84 | + numbers.append(float(value.replace(",", "."))) |
| 85 | + except ValueError: |
| 86 | + pass |
| 87 | + if not numbers: |
| 88 | + return {} |
| 89 | + numbers.sort() |
| 90 | + count = len(numbers) |
| 91 | + mean = sum(numbers) / count |
| 92 | + variance = sum((number - mean) ** 2 for number in numbers) / count |
| 93 | + return { |
| 94 | + "min": numbers[0], |
| 95 | + "max": numbers[-1], |
| 96 | + "mean": round(mean, 4), |
| 97 | + "std": round(math.sqrt(variance), 4), |
| 98 | + "median": numbers[count // 2] if count % 2 else (numbers[count // 2 - 1] + numbers[count // 2]) / 2, |
| 99 | + } |
| 100 | + |
| 101 | + @staticmethod |
| 102 | + def format_for_prompt(stats: dict[str, Any]) -> str: |
| 103 | + """Render statistics as a compact LLM prompt block.""" |
| 104 | + lines = [ |
| 105 | + f"File: {stats['filename']}", |
| 106 | + f" Rows (data): {stats['row_count']}", |
| 107 | + f" Columns ({stats['column_count']}): {', '.join(stats['columns'])}", |
| 108 | + ] |
| 109 | + if stats.get("error"): |
| 110 | + lines.append(f" ERROR: {stats['error']}") |
| 111 | + return "\n".join(lines) |
| 112 | + |
| 113 | + lines.append(" Column details:") |
| 114 | + for column, column_stats in stats.get("column_stats", {}).items(): |
| 115 | + missing = f"missing={column_stats['missing_pct']}%" if column_stats["missing_count"] else "complete" |
| 116 | + unique = f"unique={column_stats['unique_count']}" |
| 117 | + if column_stats["dtype"] == "numeric": |
| 118 | + numeric = ( |
| 119 | + f"min={column_stats.get('min')}, max={column_stats.get('max')}, " |
| 120 | + f"mean={column_stats.get('mean')}, std={column_stats.get('std')}" |
| 121 | + ) |
| 122 | + lines.append(f" {column}: numeric, {missing}, {unique}, {numeric}") |
| 123 | + else: |
| 124 | + samples = ", ".join(repr(value) for value in column_stats.get("sample_values", [])[:3]) |
| 125 | + lines.append(f" {column}: {column_stats['dtype']}, {missing}, {unique}, samples=[{samples}]") |
| 126 | + return "\n".join(lines) |
0 commit comments