From 96282dd57f5d9a892268eb1bfe2c229c35ab38fb Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 17 Jul 2026 19:13:38 +0200 Subject: [PATCH 1/2] refactor(parquet): use parquet and duckdb instead of postgresql --- README.md | 33 ++- api_tabular/__init__.py | 15 + api_tabular/config_default.toml | 10 +- api_tabular/core/query.py | 8 +- api_tabular/core/swagger.py | 6 - api_tabular/core/utils.py | 4 - api_tabular/tabular/app.py | 22 +- api_tabular/tabular/duckdb_exec.py | 202 +++++++++++++ api_tabular/tabular/duckdb_query.py | 276 ++++++++++++++++++ api_tabular/tabular/utils.py | 54 +--- ...aaaaaa-1111-bbbb-2222-cccccccccccc.parquet | Bin 0 -> 31720 bytes ...aaaaaa-5555-bbbb-6666-cccccccccccc.parquet | Bin 0 -> 1545 bytes ...aaaaaa-9999-bbbb-1010-cccccccccccc.parquet | Bin 0 -> 1545 bytes ...dddddd-1111-eeee-1212-ffffffffffff.parquet | Bin 0 -> 1517 bytes ...dddddd-3333-eeee-4444-ffffffffffff.parquet | Bin 0 -> 1402 bytes ...dddddd-7777-eeee-8888-ffffffffffff.parquet | Bin 0 -> 1545 bytes docker-compose.yml | 5 + pyproject.toml | 1 + scripts/generate_test_parquets.py | 118 ++++++++ tabular_swagger.yaml | 25 +- tests/conftest.py | 87 +++++- tests/test_api.py | 115 ++------ tests/test_config.py | 9 +- tests/test_duckdb_query.py | 51 ++++ tests/test_query.py | 6 - tests/test_swagger.py | 58 +--- uv.lock | 38 +++ 27 files changed, 859 insertions(+), 284 deletions(-) create mode 100644 api_tabular/tabular/duckdb_exec.py create mode 100644 api_tabular/tabular/duckdb_query.py create mode 100644 db/parquet/aaaaaaaa-1111-bbbb-2222-cccccccccccc.parquet create mode 100644 db/parquet/aaaaaaaa-5555-bbbb-6666-cccccccccccc.parquet create mode 100644 db/parquet/aaaaaaaa-9999-bbbb-1010-cccccccccccc.parquet create mode 100644 db/parquet/dddddddd-1111-eeee-1212-ffffffffffff.parquet create mode 100644 db/parquet/dddddddd-3333-eeee-4444-ffffffffffff.parquet create mode 100644 db/parquet/dddddddd-7777-eeee-8888-ffffffffffff.parquet create mode 100644 scripts/generate_test_parquets.py create mode 100644 tests/test_duckdb_query.py diff --git a/README.md b/README.md index 126f026..9755f50 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![CircleCI](https://circleci.com/gh/datagouv/api-tabular.svg?style=svg)](https://app.circleci.com/pipelines/github/datagouv/api-tabular) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -An API service that provides RESTful access to CSV or tabular data converted by [Hydra](https://github.com/datagouv/hydra). This service provides a REST API to access PostgreSQL database tables containing CSV data, offering HTTP querying capabilities, pagination, and data streaming for CSV or tabular resources. +An API service that provides RESTful access to CSV or tabular data converted by [Hydra](https://github.com/datagouv/hydra). The **Tabular API** queries public Parquet files with [DuckDB](https://duckdb.org/) (filters, pagination, aggregations, CSV/JSON export), while resource metadata still comes from PostgreSQL via PostgREST (`tables_index`). The **Metrics API** continues to use PostgREST against PostgreSQL. This service is mainly used, developed and maintained by [data.gouv.fr](https://data.gouv.fr) - the France Open Data platform. The production API is deployed on data.gouv.fr infrastructure at [`https://tabular-api.data.gouv.fr/api`](https://tabular-api.data.gouv.fr/api). See the [product documentation](https://www.data.gouv.fr/dataservices/api-tabulaire-data-gouv-fr-beta/) (in French) for usage details and the [technical documentation](https://tabular-api.data.gouv.fr/api/doc) for API reference. @@ -74,9 +74,11 @@ The production API is deployed on data.gouv.fr infrastructure at [`https://tabul Query the API using a `resource_id`. Several test resources are available in the fake database: - **`aaaaaaaa-1111-bbbb-2222-cccccccccccc`** - Main test resource with 1000 rows - - **`aaaaaaaa-5555-bbbb-6666-cccccccccccc`** - Resource with database indexes - - **`dddddddd-7777-eeee-8888-ffffffffffff`** - Resource allowed for aggregation - - **`aaaaaaaa-9999-bbbb-1010-cccccccccccc`** - Resource with indexes and aggregation allowed + - **`aaaaaaaa-5555-bbbb-6666-cccccccccccc`** - Smaller sample resource + - **`dddddddd-7777-eeee-8888-ffffffffffff`** - Smaller sample resource + - **`aaaaaaaa-9999-bbbb-1010-cccccccccccc`** - Smaller sample resource + + Local Parquet fixtures live under `db/parquet/{resource_id}.parquet`. Set `PARQUET_BASE_URL` to that directory when running the Tabular API against the test stack (the pytest suite does this automatically). ### 🏭 Run with a real Hydra database @@ -320,9 +322,8 @@ column_name__sort=desc ``` #### Aggregation Operators -> ⚠️ **WARNING**: Aggregation requests are disabled by default. -> You can allow or disallow it for all resources with the `ALLOW_AGGREGATION` config. If `ALLOW_AGGREGATION` is set to `false`, you can specify allow exceptions by listing these in the `ALLOW_AGGREGATION_EXCEPTIONS` list. -> You can get the current status and exceptions at `/api/aggregation-exceptions/` endpoint. + +Aggregation is available on all resources and columns (except JSON limitations below). ``` # group by values @@ -486,16 +487,19 @@ Configuration is handled through TOML files and environment variables. The defau | Option | Default | Description | |--------|---------|-------------| -| `PGREST_ENDPOINT` | `http://localhost:8080` | PostgREST server URL | +| `PGREST_ENDPOINT` | `http://localhost:8080` | PostgREST server URL (metadata for Tabular API; data for Metrics API) | +| `S3_ENDPOINT` | `""` | S3 endpoint hosting public Parquet files (set via env in deploy) | +| `S3_BUCKET` | `""` | S3 bucket name (set via env in deploy) | +| `PARQUET_BASE_URL` | `""` | Optional override for Parquet base URL/path (tests use a local directory). When empty, URLs are `https://{S3_ENDPOINT}/{S3_BUCKET}/parquet/{resource_id}.parquet` | | `SERVER_NAME` | `localhost:8005` | Server name for URL generation | | `SCHEME` | `http` | URL scheme (http/https) | | `SENTRY_DSN` | `None` | Sentry DSN for error reporting (optional) | | `PAGE_SIZE_DEFAULT` | `20` | Default page size | | `PAGE_SIZE_MAX` | `50` | Maximum allowed page size | -| `BATCH_SIZE` | `50000` | Batch size for streaming | +| `BATCH_SIZE` | `50000` | Max rows for CSV/JSON export | | `DOC_PATH` | `/api/doc` | Swagger documentation path | -| `ALLOW_AGGREGATION` | `False` | Whether aggregation queries are allowed (`column__groupby`). If False, it can still be explicitly allowed with `ALLOW_AGGREGATION_EXCEPTIONS` | -| `ALLOW_AGGREGATION_EXCEPTIONS` | `["dddddddd-7777-eeee-8888-ffffffffffff", "aaaaaaaa-9999-bbbb-1010-cccccccccccc"]` | List of resource IDs allowed for aggregation | + +> Disk caching of Parquet files is not implemented yet; DuckDB reads remote (or local) Parquet via `httpfs` / filesystem. A local cache may be added later if latency requires it. ### Environment Variables @@ -503,13 +507,14 @@ You can override any configuration value using environment variables: ```shell export PGREST_ENDPOINT="http://my-postgrest:8080" +export S3_ENDPOINT="s3.example.com" +export S3_BUCKET="my-bucket" export PAGE_SIZE_DEFAULT=50 export SENTRY_DSN="https://your-sentry-dsn" ``` -Once the containers are up and running, you can directly query PostgREST on: + +PostgREST remains available for metadata inspection and for the Metrics API: `/?` -like for example: -`http://localhost:8080/eb7a008177131590c2f1a2ca0?decompte=eq.10` ### Custom Configuration File diff --git a/api_tabular/__init__.py b/api_tabular/__init__.py index 529be02..5df6faf 100644 --- a/api_tabular/__init__.py +++ b/api_tabular/__init__.py @@ -55,6 +55,21 @@ def check(self): """Sanity check on config""" pass + def parquet_url(self, resource_id: str) -> str: + """Public Parquet URL (or local path override) for a resource.""" + assert self.configuration is not None + base = (self.configuration.get("PARQUET_BASE_URL") or "").rstrip("/") + if not base: + endpoint = (self.configuration.get("S3_ENDPOINT") or "").strip() + bucket = (self.configuration.get("S3_BUCKET") or "").strip() + if not endpoint or not bucket: + raise ValueError( + "Parquet location is not configured: set PARQUET_BASE_URL, " + "or both S3_ENDPOINT and S3_BUCKET" + ) + base = f"https://{endpoint}/{bucket}/parquet" + return f"{base}/{resource_id}.parquet" + def __getattr__(self, __name): assert self.configuration is not None return self.configuration.get(__name) diff --git a/api_tabular/config_default.toml b/api_tabular/config_default.toml index cd3c418..73b5c54 100644 --- a/api_tabular/config_default.toml +++ b/api_tabular/config_default.toml @@ -1,4 +1,9 @@ PGREST_ENDPOINT = "http://localhost:8080" +# Set via environment (no defaults in repo). Used when PARQUET_BASE_URL is empty. +S3_ENDPOINT = "" +S3_BUCKET = "" +# Optional override for tests / local paths. When empty, URLs are built from S3_*. +PARQUET_BASE_URL = "" SERVER_NAME = "localhost:8005" SCHEME = "http" SENTRY_DSN = "" @@ -7,8 +12,3 @@ PAGE_SIZE_DEFAULT = 20 PAGE_SIZE_MAX = 50 BATCH_SIZE = 50000 DOC_PATH = "/api/doc" -ALLOW_AGGREGATION = false -ALLOW_AGGREGATION_EXCEPTIONS = [ - "dddddddd-7777-eeee-8888-ffffffffffff", # without indexes - "aaaaaaaa-9999-bbbb-1010-cccccccccccc", # with indexes -] # list of resource_ids diff --git a/api_tabular/core/query.py b/api_tabular/core/query.py index 2cab7d1..8778a61 100644 --- a/api_tabular/core/query.py +++ b/api_tabular/core/query.py @@ -1,8 +1,6 @@ import re from collections import defaultdict -from api_tabular.core.utils import is_aggregation_allowed - def build_sql_query_string( request_arg: list, @@ -11,6 +9,7 @@ def build_sql_query_string( page_size: int | None = None, offset: int = 0, ) -> str: + """Build a PostgREST query string (used by the Metrics API).""" sql_query = [] aggregators = defaultdict(list) sorted = False @@ -39,11 +38,6 @@ def build_sql_query_string( else: raise ValueError(f"argument '{arg}' could not be parsed") if aggregators: - if resource_id and not is_aggregation_allowed(resource_id): - raise PermissionError( - f"Aggregation parameters `{'`, `'.join(aggregators.keys())}` " - f"are not allowed for resource '{resource_id}'" - ) agg_query = "select=" for operator in aggregators: if operator == "groupby": diff --git a/api_tabular/core/swagger.py b/api_tabular/core/swagger.py index 5415ee4..155e0cc 100644 --- a/api_tabular/core/swagger.py +++ b/api_tabular/core/swagger.py @@ -2,8 +2,6 @@ import yaml -from api_tabular.core.utils import is_aggregation_allowed - TYPE_POSSIBILITIES = { "string": [ "isnull", @@ -200,10 +198,6 @@ def swagger_parameters(resource_columns: dict, resource_id: str) -> list: # see cast for db here: https://github.com/datagouv/csv-detective/blob/master/csv_detective/output/dataframe.py for key, value in resource_columns.items(): for op in OPERATORS_DESCRIPTIONS: - if not is_aggregation_allowed(resource_id) and OPERATORS_DESCRIPTIONS[op].get( - "is_aggregator" - ): - continue if op in TYPE_POSSIBILITIES[value["python_type"]]: op_name = cast(str, OPERATORS_DESCRIPTIONS[op]["name"]) op_description = cast(str, OPERATORS_DESCRIPTIONS[op]["description"]) diff --git a/api_tabular/core/utils.py b/api_tabular/core/utils.py index 1aba153..434cff2 100644 --- a/api_tabular/core/utils.py +++ b/api_tabular/core/utils.py @@ -5,10 +5,6 @@ from api_tabular.core.error import QueryException -def is_aggregation_allowed(resource_id: str) -> bool: - return config.ALLOW_AGGREGATION or resource_id in config.ALLOW_AGGREGATION_EXCEPTIONS - - def process_total(res: Response | ClientResponse) -> int: # the Content-Range looks like this: '0-49/21777' # see https://docs.postgrest.org/en/stable/references/api/pagination_count.html diff --git a/api_tabular/tabular/app.py b/api_tabular/tabular/app.py index 4feb496..113e0cc 100644 --- a/api_tabular/tabular/app.py +++ b/api_tabular/tabular/app.py @@ -15,7 +15,6 @@ from api_tabular.core.utils import build_offset from api_tabular.core.version import get_app_version from api_tabular.tabular.utils import ( - get_potential_indexes, get_resource, get_resource_data, stream_resource_data, @@ -62,8 +61,6 @@ async def resource_profile(request): resource: dict = await get_resource( request.app["csession"], resource_id, ["profile:csv_detective"] ) - indexes: set | None = await get_potential_indexes(request.app["csession"], resource_id) - resource["indexes"] = list(indexes) if isinstance(indexes, set) else None return web.json_response(resource) @@ -73,10 +70,7 @@ async def resource_swagger(request): resource: dict = await get_resource( request.app["csession"], resource_id, ["profile:csv_detective"] ) - indexes: set | None = await get_potential_indexes(request.app["csession"], resource_id) columns: dict[str, str] = resource["profile"]["columns"] - if indexes: - columns = {col: params for col, params in columns.items() if col in indexes} swagger_string = build_swagger_file(columns, resource_id) return web.Response(body=swagger_string) @@ -102,9 +96,9 @@ async def resource_data(request): offset = build_offset(page, page_size) - sql_query = await try_build_query(request, query_string, resource_id, page_size, offset) - resource = await get_resource(request.app["csession"], resource_id, ["parsing_table"]) - response, total = await get_resource_data(request.app["csession"], resource, sql_query) + query = await try_build_query(request, query_string, resource_id, page_size, offset) + await get_resource(request.app["csession"], resource_id, []) + response, total = await get_resource_data(resource_id, query) next = build_link_with_page(request, query_string, page + 1, page_size) prev = build_link_with_page(request, query_string, page - 1, page_size) @@ -146,16 +140,6 @@ async def get_health(request): return await check_health(request, f"{config.PGREST_ENDPOINT}/migrations_csv") -@routes.get(r"/api/aggregation-exceptions/") -async def get_aggregation_exceptions(request): - """Return the list of resources for which aggregation queries are allowed""" - body = { - "allowed": config.ALLOW_AGGREGATION, - "exceptions": config.ALLOW_AGGREGATION_EXCEPTIONS, - } - return web.json_response(body) - - async def app_factory(): async def on_startup(app): app["csession"] = ClientSession() diff --git a/api_tabular/tabular/duckdb_exec.py b/api_tabular/tabular/duckdb_exec.py new file mode 100644 index 0000000..5ef02a5 --- /dev/null +++ b/api_tabular/tabular/duckdb_exec.py @@ -0,0 +1,202 @@ +"""Execute DuckDB queries against remote or local Parquet files.""" + +from __future__ import annotations + +import asyncio +import csv +import json +from io import StringIO +from typing import Any + +import duckdb +from aiohttp import web +from aiohttp.web import StreamResponse +from aiohttp.web_request import Request + +from api_tabular import config +from api_tabular.core.error import QueryException +from api_tabular.tabular.duckdb_query import DuckDBQuery, quote_ident + + +def _connect() -> duckdb.DuckDBPyConnection: + con = duckdb.connect(database=":memory:") + # httpfs is needed for https:// Parquet URLs; harmless for local files + try: + con.execute("INSTALL httpfs;") + except duckdb.Error: + pass + try: + con.execute("LOAD httpfs;") + except duckdb.Error: + pass + return con + + +def _parquet_source_sql(parquet_path: str, columns: list[str]) -> tuple[str, list]: + """Build FROM clause ensuring a synthetic __id when missing from the file.""" + if "__id" in columns: + return "read_parquet(?)", [parquet_path] + return ( + f"(SELECT file_row_number AS {quote_ident('__id')}, " + f"* EXCLUDE (file_row_number) FROM read_parquet(?, file_row_number = true))", + [parquet_path], + ) + + +def _describe_columns(con: duckdb.DuckDBPyConnection, parquet_path: str) -> list[str]: + rows = con.execute("DESCRIBE SELECT * FROM read_parquet(?)", [parquet_path]).fetchall() + return [row[0] for row in rows] + + +def _assemble_sql( + source_sql: str, + query: DuckDBQuery, + *, + for_count: bool = False, +) -> str: + if for_count: + return f"SELECT COUNT(*) FROM {source_sql} AS src {query.where_sql}" + + parts = [ + f"SELECT {query.select_sql} FROM {source_sql} AS src", + query.where_sql, + query.group_by_sql, + query.order_sql, + ] + sql = " ".join(p for p in parts if p) + if query.limit is not None: + sql += f" LIMIT {int(query.limit)}" + if query.offset >= 1: + sql += f" OFFSET {int(query.offset)}" + return sql + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, (list, dict)): + return value + return str(value) + + +def _rows_to_dicts(con: duckdb.DuckDBPyConnection, result) -> list[dict[str, Any]]: + columns = [desc[0] for desc in result.description] + return [{col: _json_safe(val) for col, val in zip(columns, row)} for row in result.fetchall()] + + +def _execute_sync( + parquet_path: str, + query: DuckDBQuery, +) -> tuple[list[dict[str, Any]], int | None]: + con = _connect() + try: + try: + columns = _describe_columns(con, parquet_path) + except duckdb.Error as e: + message = str(e).lower() + if "404" in message or "not found" in message or "no files found" in message: + raise web.HTTPNotFound() from e + raise QueryException(400, None, "Database error", str(e)) from e + + source_sql, source_params = _parquet_source_sql(parquet_path, columns) + data_sql = _assemble_sql(source_sql, query, for_count=False) + data_params = source_params + query.where_params + + try: + result = con.execute(data_sql, data_params) + rows = _rows_to_dicts(con, result) + except duckdb.Error as e: + raise QueryException(400, None, "Database error", str(e)) from e + + total: int | None = None + if not query.is_aggregation: + count_sql = _assemble_sql(source_sql, query, for_count=True) + count_params = source_params + query.where_params + total = int(con.execute(count_sql, count_params).fetchone()[0]) + + return rows, total + finally: + con.close() + + +def _count_sync(parquet_path: str, query: DuckDBQuery) -> int: + con = _connect() + try: + try: + columns = _describe_columns(con, parquet_path) + except duckdb.Error as e: + message = str(e).lower() + if "404" in message or "not found" in message or "no files found" in message: + raise web.HTTPNotFound() from e + raise QueryException(400, None, "Database error", str(e)) from e + + source_sql, source_params = _parquet_source_sql(parquet_path, columns) + count_sql = _assemble_sql(source_sql, query, for_count=True) + return int(con.execute(count_sql, source_params + query.where_params).fetchone()[0]) + except duckdb.Error as e: + raise QueryException(400, None, "Database error", str(e)) from e + finally: + con.close() + + +def _fetch_all_sync(parquet_path: str, query: DuckDBQuery) -> list[dict[str, Any]]: + rows, _ = _execute_sync(parquet_path, query) + return rows + + +async def execute_query( + resource_id: str, + query: DuckDBQuery, +) -> tuple[list[dict[str, Any]], int | None]: + parquet_path = config.parquet_url(resource_id) + return await asyncio.to_thread(_execute_sync, parquet_path, query) + + +async def count_rows(resource_id: str, query: DuckDBQuery) -> int: + parquet_path = config.parquet_url(resource_id) + return await asyncio.to_thread(_count_sync, parquet_path, query) + + +async def stream_parquet_data( + request: Request, + resource_id: str, + query: DuckDBQuery, + format: str, + response_headers: dict, +) -> StreamResponse: + total = await count_rows(resource_id, query) + if total > config.BATCH_SIZE: + raise QueryException( + 403, + None, + "Output is too long", + f"The output has more than {config.BATCH_SIZE} rows, please consider downloading the source file directly", + ) + + export_query = DuckDBQuery( + where_sql=query.where_sql, + where_params=list(query.where_params), + order_sql=query.order_sql, + select_sql=query.select_sql, + group_by_sql=query.group_by_sql, + is_aggregation=query.is_aggregation, + limit=config.BATCH_SIZE, + offset=0, + ) + rows = await asyncio.to_thread(_fetch_all_sync, config.parquet_url(resource_id), export_query) + + response = web.StreamResponse(headers=response_headers) + await response.prepare(request) + + if format == "json": + await response.write(json.dumps(rows, default=str).encode()) + else: + buffer = StringIO() + if rows: + writer = csv.DictWriter(buffer, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + await response.write(buffer.getvalue().encode()) + + await response.write_eof() + return response diff --git a/api_tabular/tabular/duckdb_query.py b/api_tabular/tabular/duckdb_query.py new file mode 100644 index 0000000..0d1679b --- /dev/null +++ b/api_tabular/tabular/duckdb_query.py @@ -0,0 +1,276 @@ +import re +from collections import defaultdict +from dataclasses import dataclass, field + + +@dataclass +class DuckDBQuery: + """SQL fragments ready to run against a Parquet source aliased as `src`.""" + + where_sql: str = "" + where_params: list = field(default_factory=list) + order_sql: str = "" + select_sql: str = "*" + group_by_sql: str = "" + is_aggregation: bool = False + limit: int | None = None + offset: int = 0 + + +def quote_ident(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def get_column_and_operator(argument: str) -> tuple[str, str]: + *column_split, comparator = argument.split("__") + column = "__".join(column_split) + return column, comparator.lower() + + +def build_duckdb_query( + request_arg: list[str], + page_size: int | None = None, + offset: int = 0, +) -> DuckDBQuery: + where_parts: list[str] = [] + where_params: list = [] + order_sql = "" + select_columns: list[str] | None = None + aggregators: dict[str, list[str]] = defaultdict(list) + has_sort = False + + for arg in request_arg: + if arg.startswith("or=("): + clause, params = parse_operator(query=arg, operator="or", top_level=True) + where_parts.append(clause) + where_params.extend(params) + continue + _split = arg.split("=") + if len(_split) == 2: + result = add_filter(*_split) + if result is None: + continue + kind, payload = result + if kind == "where": + where_parts.append(payload[0]) + where_params.extend(payload[1]) + elif kind == "order": + order_sql = payload + has_sort = True + elif kind == "select": + select_columns = payload + elif len(_split) == 1: + if "__" not in _split[0]: + raise ValueError(f"argument '{arg}' could not be parsed") + if _split[0].split("__")[-1] in ["isnull", "isnotnull"]: + result = add_filter(_split[0], "") + assert result is not None and result[0] == "where" + where_parts.append(result[1][0]) + where_params.extend(result[1][1]) + else: + column, operator = add_aggregator(_split[0]) + aggregators[operator].append(column) + else: + raise ValueError(f"argument '{arg}' could not be parsed") + + is_aggregation = bool(aggregators) + if is_aggregation and select_columns is not None: + raise ValueError("the argument `columns` cannot be set alongside aggregators") + + select_sql = "*" + group_by_sql = "" + if is_aggregation: + select_parts: list[str] = [] + group_cols: list[str] = [] + for operator, columns in aggregators.items(): + if operator == "groupby": + for column in columns: + ident = quote_ident(column) + select_parts.append(ident) + group_cols.append(ident) + else: + for column in columns: + ident = quote_ident(column) + alias = quote_ident(f"{column}__{operator}") + select_parts.append(f"{operator.upper()}({ident}) AS {alias}") + select_sql = ", ".join(select_parts) + if group_cols: + group_by_sql = ", ".join(group_cols) + elif select_columns is not None: + select_sql = ", ".join(quote_ident(c) for c in select_columns) + + if not has_sort and not is_aggregation: + order_sql = f"ORDER BY {quote_ident('__id')} ASC" + + where_sql = "" + if where_parts: + where_sql = "WHERE " + " AND ".join(where_parts) + + return DuckDBQuery( + where_sql=where_sql, + where_params=where_params, + order_sql=order_sql, + select_sql=select_sql, + group_by_sql=f"GROUP BY {group_by_sql}" if group_by_sql else "", + is_aggregation=is_aggregation, + limit=page_size, + offset=offset, + ) + + +def add_filter( + argument: str, + value: str, + *, + in_operator: bool = False, +) -> tuple[str, tuple] | None: + if argument in ["page", "page_size"]: + if in_operator: + raise ValueError(f"Argument `{argument}` can't be set within an operator") + return None + if argument == "columns": + if in_operator: + raise ValueError(f"Argument `{argument}` can't be set within an operator") + if '"' in value: + raise ValueError('Forbidden character `"` in a column name') + return ("select", value.split(",")) + if "__" not in argument: + raise ValueError(f"argument '{argument}={value}' could not be parsed") + + column, normalized_comparator = get_column_and_operator(argument) + ident = quote_ident(column) + + if normalized_comparator == "sort": + if in_operator: + raise ValueError(f"Argument `{argument}` can't be set within an operator") + direction = value.upper() + if direction not in ("ASC", "DESC"): + raise ValueError(f"argument '{argument}={value}' could not be parsed") + return ("order", f"ORDER BY {ident} {direction}") + if normalized_comparator == "exact": + return ("where", (f"{ident} = ?", [coerce_value(value)])) + if normalized_comparator == "differs": + return ("where", (f"{ident} IS DISTINCT FROM ?", [coerce_value(value)])) + if normalized_comparator == "isnull": + return ("where", (f"{ident} IS NULL", [])) + if normalized_comparator == "isnotnull": + return ("where", (f"{ident} IS NOT NULL", [])) + if normalized_comparator == "contains": + return ("where", (f"{ident} ILIKE '%' || ? || '%'", [value])) + if normalized_comparator == "notcontains": + return ("where", (f"NOT ({ident} ILIKE '%' || ? || '%')", [value])) + if normalized_comparator == "in": + values = value.split(",") + placeholders = ", ".join("?" for _ in values) + return ("where", (f"{ident} IN ({placeholders})", [coerce_value(v) for v in values])) + if normalized_comparator == "notin": + values = value.split(",") + placeholders = ", ".join("?" for _ in values) + return ( + "where", + (f"{ident} NOT IN ({placeholders})", [coerce_value(v) for v in values]), + ) + if normalized_comparator == "less": + return ("where", (f"{ident} <= ?", [coerce_value(value)])) + if normalized_comparator == "greater": + return ("where", (f"{ident} >= ?", [coerce_value(value)])) + if normalized_comparator == "strictly_less": + return ("where", (f"{ident} < ?", [coerce_value(value)])) + if normalized_comparator == "strictly_greater": + return ("where", (f"{ident} > ?", [coerce_value(value)])) + raise ValueError(f"argument '{argument}={value}' could not be parsed") + + +def coerce_value(value: str): + """Coerce query-string values. Keep numbers as strings so DuckDB can cast to the column type.""" + lower = value.lower() + if lower == "true": + return True + if lower == "false": + return False + return value + + +def add_aggregator(argument: str) -> tuple[str, str]: + if "__" not in argument: + raise ValueError(f"argument '{argument}' could not be parsed") + column, operator = get_column_and_operator(argument) + if operator in ["avg", "count", "max", "min", "sum", "groupby"]: + return column, operator + raise ValueError(f"argument '{argument}' could not be parsed") + + +def split_top_level(s: str) -> list[str]: + parts = [] + current = "" + depth = 0 + for char in s: + if char == "(": + depth += 1 + current += char + elif char == ")": + depth -= 1 + current += char + elif char == "," and depth == 0: + parts.append(current) + current = "" + else: + current += char + if current: + parts.append(current) + return parts + + +def find_arg_val(param: str) -> tuple[str, str]: + if param.count('"') not in {0, 2, 4}: + raise ValueError(f"argument '{param}' could not be parsed") + column_operator_pattern = r'^"[^"]*"__[a-z]+' + value_pattern = r'\."[^"]*"$' + if param.count('"') == 0: + # col__op.val — value may contain dots (e.g. 0.1) + if "." not in param: + raise ValueError(f"argument '{param}' could not be parsed") + argument, value = param.split(".", 1) + if "__" not in argument: + raise ValueError(f"argument '{param}' could not be parsed") + return argument, value + if param.count('"') == 4: + col_op = re.findall(column_operator_pattern, param) + val = re.findall(value_pattern, param) + if len(col_op) != 1 or len(val) != 1: + raise ValueError(f"argument '{param}' could not be parsed") + return col_op[0].replace('"', ""), val[0][1:] + col_op = re.findall(column_operator_pattern, param) + val = re.findall(value_pattern, param) + if not col_op: + return param.split(".", 1)[0], val[0][1:] + return col_op[0].replace('"', ""), param.split(".")[-1] + + +def parse_operator(query: str, operator: str, top_level: bool = False) -> tuple[str, list]: + if not query.endswith(")"): + raise ValueError(f"argument '{query}' could not be parsed") + params_sql: list[str] = [] + params_values: list = [] + inner = re.findall(rf"^{operator}{'=' if top_level else ''}\((.*)\)$", query)[0] + for param in split_top_level(inner): + if param.startswith(("and(", "or(")): + clause, values = parse_operator(query=param, operator=param.split("(")[0]) + params_sql.append(clause) + params_values.extend(values) + elif param.endswith(("__isnull", "__isnotnull")): + result = add_filter(param.replace('"', ""), "", in_operator=True) + assert result is not None and result[0] == "where" + params_sql.append(result[1][0]) + params_values.extend(result[1][1]) + else: + argument, value = find_arg_val(param) + # Strip surrounding quotes from values like "12.4" + if len(value) >= 2 and value.startswith('"') and value.endswith('"'): + value = value[1:-1] + result = add_filter(argument, value, in_operator=True) + assert result is not None and result[0] == "where" + params_sql.append(result[1][0]) + params_values.extend(result[1][1]) + joiner = f" {operator.upper()} " + return f"({joiner.join(params_sql)})", params_values diff --git a/api_tabular/tabular/utils.py b/api_tabular/tabular/utils.py index 999c8ed..8e172a6 100644 --- a/api_tabular/tabular/utils.py +++ b/api_tabular/tabular/utils.py @@ -2,10 +2,9 @@ from aiohttp.web_request import Request from api_tabular import config -from api_tabular.core.data import stream_data from api_tabular.core.error import QueryException, handle_exception -from api_tabular.core.query import build_sql_query_string -from api_tabular.core.utils import process_total +from api_tabular.tabular.duckdb_exec import execute_query, stream_parquet_data +from api_tabular.tabular.duckdb_query import build_duckdb_query async def get_resource(session: ClientSession, resource_id: str, columns: list) -> dict: @@ -34,35 +33,8 @@ async def get_resource(session: ClientSession, resource_id: str, columns: list) return record[0] -async def get_resource_data( - session: ClientSession, resource: dict, sql_query: str -) -> tuple[list[dict], int | None]: - headers = {"Prefer": "count=exact"} - url = f"{config.PGREST_ENDPOINT}/{resource['parsing_table']}?{sql_query}" - skip_total = False - if any(f".{agg}()" in url for agg in ["count", "max", "min", "sum", "avg"]): - # the total for aggretated data is wrong, it is always the length of the original table - skip_total = True - async with session.get(url, headers=headers) as res: - if not res.ok: - handle_exception(res.status, "Database error", await res.json(), resource.get("id")) - record = await res.json() - total = process_total(res) if not skip_total else None - return record, total - - -async def get_potential_indexes(session: ClientSession, resource_id: str) -> set[str] | None: - q = f"select=table_indexes&resource_id=eq.{resource_id}" - url = f"{config.PGREST_ENDPOINT}/resources_exceptions?{q}" - async with session.get(url) as res: - record = await res.json() - if not res.ok: - handle_exception(res.status, "Database error", record, resource_id) - if not record: - return None - # indexes look like {"column_name": "index_type", ...} or None - indexes: dict = record[0].get("table_indexes", {}) - return set(indexes.keys()) if indexes else None +async def get_resource_data(resource_id: str, query) -> tuple[list[dict], int | None]: + return await execute_query(resource_id, query) async def try_build_query( @@ -72,22 +44,18 @@ async def try_build_query( page_size: int | None = None, offset: int = 0, ): - indexes: set | None = await get_potential_indexes(request.app["csession"], resource_id) try: - sql_query = build_sql_query_string(query_string, resource_id, indexes, page_size, offset) + return build_duckdb_query(query_string, page_size=page_size, offset=offset) except ValueError as e: raise QueryException(400, None, "Invalid query string", f"Malformed query: {e}") - except PermissionError as e: - raise QueryException(403, None, "Unauthorized parameters", str(e)) - return sql_query async def stream_resource_data(request: Request, format: str): resource_id = request.match_info["rid"] query_string = request.query_string.split("&") if request.query_string else [] - sql_query = await try_build_query(request, query_string, resource_id) - resource = await get_resource(request.app["csession"], resource_id, ["parsing_table"]) + query = await try_build_query(request, query_string, resource_id) + await get_resource(request.app["csession"], resource_id, []) mime = "application/json" if format == "json" else "text/csv" response_headers = { @@ -95,10 +63,10 @@ async def stream_resource_data(request: Request, format: str): "Content-Type": mime, } - return await stream_data( - session=request.app["csession"], + return await stream_parquet_data( request=request, - url=f"{config.PGREST_ENDPOINT}/{resource['parsing_table']}?{sql_query}", - accept_format=mime, + resource_id=resource_id, + query=query, + format=format, response_headers=response_headers, ) diff --git a/db/parquet/aaaaaaaa-1111-bbbb-2222-cccccccccccc.parquet b/db/parquet/aaaaaaaa-1111-bbbb-2222-cccccccccccc.parquet new file mode 100644 index 0000000000000000000000000000000000000000..267d63548b664090b6df146408400bef9b88c18d GIT binary patch literal 31720 zcmZUZWn2?n*v1703tx=>oP%@NijZPoK<5y;1Fe;2md!GVAAP3K6$sQOGJ$UdS>0fSl|6<~NqIsedqBHjL_rWhWXTRJjTmSgUUiKz9 z*l~91PLcV?Hhbwm!Ty_{dhg^{_P?;db_(`!{8WD@Te!c#Uh*i|YjdXfPTG2Z>d##E zos`1wBI`B@c`+H|;!Ca6nY}(~c~9;IoSfl{yu+%5CXjzejxf1p_Tc-t*w_l{+0igBP2Y`pT8{^gEgX|hBhfB9JU zEv>?ic4-ngkQ+7@dkenVRxM3D3E(In4ZNl9-j*v(mPwJH!DA1zj@Sps1o9+3@&>)EAu1&VlsSJCD2h3Tzd2AVn5`f zCwxuixuZPz^~+D~`<54V;h$BWI?94eZf1h_H7;PDMt>hXw-0YtvDy?i&M*9^`MvBM z8D6Vmwkc?w$NZ7xd-Si~NL~K~5t-BBzl5z}+!!a5s!A+!f;jcfmNroiR>uCyXQ95#s=N zz}Um>F;C!6Fm`Y|j4j+2V*|IrSi`L`R&XnfCEOBY0k^=I!_6^fa5Ic4+!SL1H^CUg zjWI@WBa9*35QBsxF$Qo0%wzatj6Pf+qX*Z+=)!d|I&d9~He4H{1=qr8!Zk4(a19Is zj=-qH)iICYk1%R*HH<1;6{7-I!90XN#3;j+F-mYHj3QhSqX1XH$iw9^a&S3}EL;{N z1DC-_!=*7&a4C!=ToNMzm%xa_#W7-VF^ni&6e9u`!3e{JF+y-5j38VPBLEk`z~OKV zKb#-K2j|1^!g(<~a2^adoEyUh=fZHpIWaIe48sBEz_7#FF>G))3@e-!!vbf)K;cjf zGn^U21ZTo9!Wl6Pa0U!LoE}35r^C>~X)zEu1OtYHF*I-*3^kk@Lj|Y8fZ!kuC7cpN z0jIzK;Xn*IoE$?2C&Q4!NihKUBYNV`M7c!fMD#?TALKqTe?b4h{7v$l1-;?xp~!O8 z)^gps*FUB%cEV2wUY0*>E&DJFVpd#HC{(akU{+X>FO;{IXO>@)E0nXAW0qTyEtIvE zWtLr$DU`96VU}5uE|j*GUU>s5J}jCpYAi}E@+mSX5-9=|9TrX(HWsE9`V<-ziWGth z4-2LX8VgbjdS%FlRcaF()<0C&wU1BnOmpm_41{n4OyKlWmYKk`2l}%$m+>%u3Dj z$uh_i$pU2^W=>}|W~OHPWEy0OWP&meGo~{dGg32rG7K_AGC&!J>C@?r>8a^H=?3W{ z>7ew(wCS|QwA3`8G=nseG*H@M>U3&jYHF%aszItqDk$|ZWjduXB{jt-#UMo_1(b4_ zJe}N_oSN*DY>+IH3`#ysnoep=N=@=fGDs3h0woA1$Y)Ht6wgE)~mP~0JU8r_IaMf;!) z&?0CM`VcjZYDA@?d{71`5flh@7&{%?7@HdF6KfDF5<5K?KO8dbF>E=kJuElOKg>8x zI=u8_CwNJF<+%8$m{5FBykERmyi>eYyji?n{Iz(sc)1u~yjVP6JX<_dO#B~K$DGKP z{$p>(8Gw<{PyGK#toz#GCGk*R;uM%Yi;?ds$^ZP{cwH-&UtIjLHh*KIXA=BxMVjvi za3^>W{9CMFBBQNr&^Nv|c4FiDRJBjd3`Nc>ohl>AHBEIY$Qg0rYTc^-(}P!V6`-QQ zL<(kq6MI`xIm8=CI%i z$s>b5HpCs0NuR7wq_L5V1AC$vphCHKA?vPO45=d8SXhV!Cr^d>y#=1W`nNye$MqFY^>2R;X ztu+@Goz7Ieh)iRR9S4R{@;zcA=xf+#$N~{Ly15ZVoEkeMwf9?>9!IVjqBOGGhTJzp zGl@)BAVwH8p~E=~4T_QpO#-tGg&u~jd9m3~nB=3vSE8Gz*Pc3{C?jMK*m%W$W3H__ zTTAXMDq3YwubtUjkVV|=d%9@~xGPy2iR@Q&2B*T4IGXJkbW`B6ay+JD-bkv9tnhK z_O?@3mJW;CL&OC^vGG*KQW}#2&)f@FrA&;t&-its<7d%OI^2=3{4&kO&lS7O{woIG zAFxzDT(Y{MGGs(bz*G=hZyVo9{MqN>BP9ZN(H1YNv%?t(`a0~by!j%W&RW3Y@0Zon zann3I0}xy5n#flQZetxlhSSox0h;>P5UY<6)h{(Fl9x-q|L|W;w7q{?n#GTR z!UKd91->i26>mCYwkWDNrkQ5ln&~zA<{q-(!p`6Gi(C4WR?P?Cje4All?mA6H$x(> zPB)cSt2@l3kr{XOgj%f*a+Z*U>527H^+SBOtW(SRQ>mG#FwEdvXydz6$?Q}Yf^_Mo zxl9NwA<)o1JTG@UMb|qbK2wL=F8G-#>z-C8f*B62tNuxL4Fqu82t$tozP_>`@lTR% zu|xq+jhfW>4*7jm56@$f5xOh;c_eHx2@Q$)LmVNr$;WuP5)O8ikOOo^U%Iw(t4l~ zdc{R;xB_-C+?t_|@F#|`nJhg$WnU!c?!|BVRUmR1g1vB&S`JBh!ShwZxkMboV9v=JR)c#zFuW-CZtR$` zQO)xzaY9%cCDnq`ujA@*9`{M?tE=!=<zAEzCyN{{5K%*i!t~8;nUl7BSL0!0;H@z$M617+l zC6Z@SvnIYlr>`q?HzU;Zc$cBvTr3*=z|a#h;#N9BkH&jVXeK4`Mj%ru1knT53Pu=v zpUm)M4sIBR{PCJ(SD)z^WJ7zi44UlCE;!=;{GuXfXGB_>lfQIW%f$8xX11?V5woV? zuu{@c?)Twgrl$TfcZ75XOP4r1A$p4|wz5aPeF^H_Esc9m3}|ew^&U|%d!3N{>Yt4-^P!C4EL%$YMa>gw&>pJ|50LWU9kard)?kEt)MBpO>-|*+f zd1GT)^jGHT109eX;e2QZa=Bg?#ly?WaAVOs>U=A>QrtQ5n99P^hl~tMB~enA+S;#Q zcRa@a)mfI2`~%?Q6J#|Y{#u!{VRMyi%~|wUO0m>)Som2Q)2;&bAJ3ID&(cPgdWJ-u z;nl>Ih441}_qKN_&H?JU2xk_n02~Z}i?KWuDVclty%&W2<&A=%>;V@qEq>795=Q}U zNm9?*k1}=#($>gF2?}<`=H)a9jvq8s-Z;nYbE|5j$Cu_(C6C`ah>Y>I(p@vu`Muko zuuTz@rUGSu5htG6VED!zrDW-*3yn3=zk88h+T+}wPst5tqPNXSRX*Y~f&cWlWD?6$?5clJpL0ETwfazTHY%M!+v-{A0)olhXjEsg zoRa=3_<5$7y?`(QE3di%2|E1Z3F=fYd^@*5JV~a^OqZ$+;?Q5BVLFriATtME{eg^E z-`cM5S>Iwsgpn1D31)m^igvFG14HO@g4I2ghD;gan1~j+h9aD&oN_7ntUdJ%C2`(S zGuK+?Keqd?^yqx(aPofG1&4*xFf(f^Tu9N_Oh-PdA3J@kJf0cy_xwm01NcE){ZT4R zw&Kul=!Ou9j#0JgC59CiBe!wQn{L`CLeK4h|yGhko9qs0gGSd+@-Bk*e7`fV`eS@Gf(Mo$#BSf+v-|??j9eMI!t^(qT z(E=3?M9=E6_2-PQUdjs3c@akOF$U2LOHY$F;M{LDnLB{Zi57nqo1D&XP1axO)S+yr zxS+dl8U14qldE8oZy$cwu~wu~xCji8m37zA?yD zC72x4ekd6|Y^@vnvWk^5P1_GE4S#W?pvg&P%51YM+fx_;Fn~J_{`%K>!5SQG5;^&~ zvG-l_ME)C=a3vbur?h|>(3lpP%B7`sp{rW5hZ$_`sVGx>aE91#xB5g`<+&psYjstI zIN+_T)uYv`=iHoo!jz%FL{nt@ppRfrgGi`R1 z(mQcSO@LT#DPbO|NVWX8-8_JQ}-t!9^R7$ZDK#LeFuZ6FpN+V+7-y zbSaw|D>YCp;#L=#83`V^n#if0UNU3VD1SOA6CRtHTpY|J?mxRa zDG^vY*fk{ssB%7Omp!wHd~;-u6(0ev2zcO&9YqDF#adKK zxGnAjx4OB$`(&6d!o^~k0-@es=tOr*XU0tTLYPL`dstCt>1o;0s9Cx9VG|L|>nMx5 z>IM1Sy9;lgOx`;GfIkpH6NpU%g0q2;aDyGd;z{`b_4G4geenue9sVTE82Z{84frBOF(CRiPSYpGTM2DH!xa$++33sQN%cILUtkXo z(-m&yL(P2b+%LSY`u(KMnjooH7Ggb;g_8V}?&0qp$7x~;z z3QP<+Lf~fwAiD3m#saRi`8Zilh{Z)>?obUhm|oVBj67$uAOTzT^#fA_{Lh zaGvOq`lZ`9tN54gBUcs_!$Ln$y`4@n$NPhSxk`JCk{zMW{aY@=n2sQ$5f+#!wC(H> zw`C(f#JUsT4s(aFlV(t)t1{ZOA5m=HpEp=?E`@aP*EV-r)^HiVaH^P>af$Z5Sz%yc z1wVP((tz(j-)qARp)MO0T$LtC6?%1L{gUoJHd}w`Nh4I)#_vCFqENXxKv&$me;!Sd zq9$*C*)U#tqEf5-PI+Np2?(dr?fOavR*C{{JSkG9Xahff{0R;L0@gRLmVsm2$`=37 z;{^a=ZSoJJV%530fS=xdO`iGoX~4wvoAdRHE`9C)Rx9{c88Fbg^`_IJCkkw#hiYat z|5rOon#1qjE-rfhP-9T{Qp{$Oh874Q#yk-OwLu|p{B$3CFxF|Z*O zy5IZd)$6Bp*QU%!LK3U~-~ln!e_J62RwL@psEV0+ubkte=h97?I^+^97qsS@|7@9h zQ5?lZvH(;{a%B=+6P}RYzd{V6Qe?b_u_eN;Bpwy-xei8y96WA?7_#M{)(?@V3Dre^ z9G8UhVk#;r84UUEMS(FufYaA2guB-auSj(;{0DchSXG?xE+0xM?$1x!(2VY^k2gf)pI6Soz+fR-O)0d0Xg08#-30aQQXB< zx-})6#>Pu~aX~+c+YxO@d9`4XzfX50iXws~e~iW|J=R7~U?&+#ee`GP&ZC|DrMu^R=(iW@DUr$=JRL2++AV;Ng_)@J0hOndak6KFDld*oh=wcK@wsF zA)faSvb7*O9f_%NZ@yl=LdX34?=Y-H6ES5v%ATN;BiziBAD=<1N*yinq=8Wwl4m~h zjZ*1q&@N7CLaAyCybq7?UCVNQ_>W$snkn-;RT`;kYFPPLcPc`3d3Qy>698Y@i zu~3Cc?fpzBk2s9y4#Pep3g$8s+|_v1N_**DHPjQzgn_BNj*T<*x?ih+QL&!T<*R48 zZ(EtjO)5AHCQN6xxh^pvOc;|I|3a&cSoTn^8d<2GS}D|}2jv@lh!pR)pS-Xl#RQ4O z@AXpOrmtX7`yl9dAlATpWuN3Q?B`a}x&13)fxH%yiw_Q9k#n!rUW1p?;u2<8&UxY^ zpJQzyR*}TZf50 ze5(;Xm9UK!m$_av2S3}>ZTQ@h?;+R!$mIys56d9vCDO%pWR1TwH`}{Y`odonD4DmvP=*$dx|@EhKEqA+ zNe=q(FY9sD#ntLLtV;7G-b@t@O9Ocdic4=On%N>7*dBUtjJRq-+I3$m)2y|l0isdt zqNj{YQYJ9+CG^O@%@B$aN)aI2Gg(}qzp^tJ)Mv<4 zJyp9OPfeopT5L%pVx=s}R$g5(w>UlwdQ9P_ApC-CP14#k`e_}=KFSh;^yYwe8(R9+ z-kbW~e#xiJQb@}hg%hb}Ez3P!$IR8VY771*ZFX9Uz#^i6+>wYAXsu?|JumQX;(O{& zG8xfAr#o51XuSkQG{2t|skByB$04NqX zo0||*OUT^-docpO@+8XpU}Hz?L30yVO`Vy1mGwDHJMdcQBUZe@=!PEE*aGdoCqZ7( z>?Andi5H1wLrKDI0o&+;6)lxd7F;U60eS9$J0y++bj&Up4BE6Nk|vo3uyTDuTKo?j zKDdVUqgF!3MGaHXKWUSzH3k;K2@#W@$SW^+oh!XXh%Dus`HqaOIO~d7Kz;}D?)^OL z?bL(MO_)oPB1;-jC{X+dL?mU$koPT@uQ!Ll#7D&m^}QuP3@uu4zgGx`V02u%eO^zF z+AZWA;GX$KRmpSjGQpeNr~j%u%j2yBhoCA69_%ql`m2iKSC&Hi4F)nI+=GObS-MD`#_1@-;!WuvzEDMx)MC zqM|^fII|(!Ei|q#QODkapq2h&cL7`mTt~0)J|1Zp}Cp_#n9&Uppz|p;C*lACs|b{A`b^d24~dEU2v$+@a8hrs<&$JX|Ng#> zb~tsdIxOMnG?=0n_qoq*N`;)dMp)DmPgSukOxKNk{Y0tM8N}9|VY^42@~zc@ z$PGa}f{!Rh@o24X7Il!WKdOOREx%BYI;;~pK1zG#PMYI8ccarfI`(|U7mHFka$Y`y z+N(mOH>bTvm!FLY#@^xpgV*@Zwl|HMHsl3$8YrK;k;Y)iWIZv58TX~*)ztS^)<;Pf zwOYD)Ibsv03eN*xugg3iJfH9xg%Z8<P+F0V~@lheQtJ)=oaX zLOsn#oYTJkFs`gW32KXGe0Hkuc{so38$;Z(3%Gvb*k&$8C0fP5ve(=929RgoPNaDq z|N4!T8q&nNGNRx4X_~`vuA~MquAK6WxZW=o+v=SF0;JL=*(68T#d3;;(7R0*oU6uY}l6@MR+$ADHvTdTP^?GM}(!zbX|J zu0=V6Cofm5-U>yVXx29s7m(2A+EZP;ErzaFgeVvA6tj5Qhh^d%qNlmPLLiIp+g=h)PvzC?!Zxmoz=yzR+OMh%~Ge zNijL5DRWDi$#X9KpG74x*{DHj-5v7#B$)P;sW|Ei3x zJ`y@``V1FhNgi1%esdLbPS8gCnTau>SSu4uQvb3njaNlM;%Vhc=ti|3zt>om@KW9S z3Tj-=U9aThYdSt?bafmf@Fh3^GWZuNUMW_mZdTq%R}M|z$stRBsWN!?xg1QRPMRXz zzMp9%TQpEmSo7@&SV%0NJG_*x@yd}UF;#&kaUuwBOC`E$E2wqTmtF~G{oPt}3b^`M z&b#4fkhSwshW3=zhBFK=W+{Xu@F$n=o|rlEEj@W5PC#b#`TE_B6<_OOZ66X4Pe0u1 z&^g4@M?-pz^ZPhhNxM~;G**48JlalAaU z6MzL2`XBTc<8AkWxSiYP=&zLulC>)z9yh6R4`)T ztz}fINh?<#y)&I1upp2kRrhb#i1t0;Z5Shq5kj()Ghhq7xrM3=Rjy5r&7!?b$pvQ}GR~Kc8*XtGK_I{S?p%r0NE%m) z&g^Nq0=dv`c@=xsY^@3*`51Cn2FY4}yDZOo@DmlDr9|TYLH!D7e_Tw)llAIXZTg#1 zW%Bhmv@WZaq%InZgg>E6znLtFKT-1FREEFY*pwabFWGJD$U~iqNCI;=Vtl{>gz1_2ze9WQ7ohJ$B@B} zcBq)*79Q63B)_z^MkPh5KZD6lU&#atml+af|6j(1db38_L;%U$SaZ5AQYSa>O=vlB|5n;kCdckXHD-xec8$}m$~vvC*j%`U6~L5XI# zEalqoeQ{zy^1G&I+~*-jw~JOxoQj8VY#T)R$oCI&%b*TsG1`yLw!;eOO;{9f#Rp1! zx#GtBg8)bkRBAk|GAMu*9p!8M?YL0108FUM(?n+cQbNRx;zbmPoSTtr*vV(fh@a`G zx1W`tQnRJhce3NC|9<(&ulJYE|BFdTZqYn%mMY2e%XsCK)B5LHAkW+!4h@qOuMAbz zC5a?8Hxr7c_*E|2y~sp~a%Gxhu&aL|un8pRUE5J(UW+;D$m6Wah~KlQ^sTCVaO#zFO^>TgHH5yww%6aIWn?UST+)i1(yqSvg3;Xp zzeuV{`rqA9K$m((KuDZ;2>OM_Fx#{|#BlR?dEd-%>^Q`wXniZ&x|J2LDH%xT_O_<6 zsM|8@R((8^mM|bYi!xsY^eF7UGmPm7hnf8t3#de-5ua&f$p>ifCHfd^Uht@k)4CIo zkU-u?`-9@!|H)C%RBjded1duK1NZ9A9OF<^gR^u%{~g=9xvPUhT;UO*kzX^+_^e#; z^s(7n>T`oqjqw(s_c4`jX*k{^nEIu}X;lSV81MSGiu-ix+0d{{VD)5g!pFqrh$W~K zy>Nr#W46ed1OK|<^O*nAAEe<(NauYuMj^$|T1J0I8Oj#|?)&Y@9n;%@O2k|7l5c|0 z(VKXE_VyL>b=*NSo$Vu5d!=015N5kjk zzeGyEOQvLrH_Pba1tMgj_y?nD(-dl16@H`lDtlRIU{3K<*%tdS(Oscl1SOMQ*X)_| zH+*6-C)I|H+#=LwJ)fjC+EbHxgdrYF*+iCVrL=Z$b@4&ILYiEUr_Cd-;U=$~xFuCO zX2<3!FGbi3NgLuRgO{o%rK#$rG#a1I`SvCi5+a5*2PEso`#TgkGrsd;%?xRAFG77c zyrBR3;)4SmQf>9UPXU|0b}%qXQs6>RaFvOM&y7`X!0$p#^+zou+{S#X4%ZjxGvnhF zEqIVgV+D_@Q_u-;2ZxE60hrNf7j-WpGI^_O7kLTsy3f8n`xy0wY@$~FNU49G-92oc zvmesGEb_}BWsXa>AK5~aUU3X|cR|c9fAyFY%TAEKcX(=AWyAA?3cW%cl531E62Pq)*WfrQyISg{~S!C zBPauv&(fcsFQB%oK!&I=nSkLA%@hKY5R@h5ooJ`EriwI;W!GHg<}O+e{DF-o8GQ-8 zVqj63uK{^0u()qP3hzNh#?#76+qDH!^T47=lr|(WI7{^tK2x&C(qMnMLOX%gvvYeX z|NVM^Lt9lG)?S`&S=Vz=bgA*{so*4pN|dq~EjD0H%!GUQ3&*@wMtdP*&#b5uoJ?{h zsb&#V^x#|1^{Z~#k~D4TBq`lk9neU}?-iRGJt4&5+h@xHx~fsV@(Kxi#D2ncE|Uh5rVGBtda#wNW%PQNoIV{#{;Rl7&jKr3V{f=k@fY2rRNN}KqIpj zcw9#&@OOaaN(G9(E$%;!VN}qRjLAJ0Ap2JJL9}BwWxO(tEoVqWwrAV085zq+N=J4m2 zsQ|^161;!b{<9o(_FUE2FUDd_k)!<`Th9f=g}#btDCW_hba}WMbPu! zt}>6D5BaHR>CE%BDko=r0KZ+#I2W#)4$bVet=Mray;pKk|FbHNraRo@KLjB3zS*T>|HdvljoTw*vCX&ls{{gGgogq=zs1tO1ZnG{qenG)^H&Hl$j-} zg9R9V$?N9%3pKZ`-X`Rs7*b;;G)rf+$-$wgiM3HfPEY=EgZQf|zv-+Xjt}NIlCk zMzde)YZzp5DdWRYF}_%V#+Sjod76)2U-VXr(;p<36wP!!vjmaNp=*cIf4kZn(!tAI zLxx|_0_R-uB;>th4*9Vr-f0#3SB4#h{r5l5ytlt+C5Yfi@Ty;zOnuA4KUWzbvtHTK z@7YdgU&u@A{Zfd|mtYe@yUBX+JC^fF-yh%I zsf3&7b^$9&n*QTcT(CI72H%Z>U>CPhu%4(?E1U>KMi3H`AWN~ zrKQu_O?dzsQw07nfBh@=u0nh2(4;RkdbJV7uih3Yn>U>OJg&>Z)xBAm6vI6kC9BRx zOHC8#AAOGC;t8|FO3N1qmR1G}`|}R3g=_%P;Nu=!N>sr}-f|+@9cT>lq%#l~g!g{` zHp+5CeXJ*cw?dR?IrNsyon^w^#9Y{D98ET0xTNo1PCCs_P5r`X?Amd%PXC7pVbCW>T6D7G8S zdXrWylxmW2XH48u%HL+kS2^$T9R|Is{k=5Z9>8I{kL&49uxrSEmgdD26MtF6`r^N@ z`^-X{7*x}tP)w31h*%T+x&g8tZ1KpoEyDeGPJzk6cA%^p*QU0FRboY-O<%%|z02I< z^2kl^!-?Q>nsOG$PXR-%abe1)AK3r8va%w%1W}cg zexjU!-OL*C27>ewgEWw))S2N9DlDOQ>aMA`Xjt`AgU8<|RD;7smXJ|Jf!wj{@#mgc zi=#X~l7ttnDE>{WSNIyEEd@Nu8a6S(9NToxQCv;ufVaiC2YBo(?MYkp2hZ_O&5S;j zE^CgHsiNSHLYEpH} zy8+oNuTgQGYEqa8#s#&-)1)SbN53e)H3Q%co1dohSkr{p50O{e(w4pz9ZT5O^&Xr& zLY5ft-~ld>8&v;h&31PznUG|CE=$7rjnEjFNp0Ra>F8Vo;$o2`;|GghN=E&gKBM;6g109%L@^47kAn(x%dk8O#2)^Txza&koe}Q=3WXTW8n@VByPSyK48ZH^ z!fM;ix}=SZTg31n^sNa+l#ZKyR<*ojM|*;*@v2B+O^=c1I)&!*zf{Ek@Ptyvtsojr zU%7cXvE^xzpQw|i1ILbOA-kH8bI}g9kt1GTMZF~oednhys9(H6UFj)=@| z-#Z7CttI4m5lacHyavZdvha=IiL>=%9VjSTVow!Ox857nNCCdERrjnPbdd~ZU}D;G zO*xT5EPs^ge>i0(xN;V6uZQLASem{e=1&8{t7cy41}u&SV?(4oIZHLiFp}~qxvz@r zW>0BC5a7g9prB@Q2*uKuK+*A32)=MM zNr&=~qyGHxzJ|dup@W<%!qoZ);)?5{r^n;r@GIiY%mAnAAEW5} zoT}(*txr)F7c9QfU44OB3F<}i7-v zash8U8(yxlquF(yr3ZBsKN(Et*LLhnB{@_L>=$QD=&x^)jd zmqkU9)<3~sbjY?*#-i1i6Ovj<|3IRpBAE5w&9gF?j52Dgap9d>iyS&DuG}pIsP|rW z1tYRg1-WJSA0jf*zmg)n5=dkzz{+=e&N6s zYpVg@&35fZGbQ<=3b+7>imSu6?x~0#st=TT;XK5ZyHW9A)*=Wup!i*&?aXzy9oQ@q zvZR0DEHG=d`X&03%bSE_w{q8AjXImzu6G zf7Y$j^2^uK&xrF}!26dWt_32w4y%DYB-z9#Y&!4dLtJD&?xSkeb9Js4C^=8-$OQ>J z-jpsCKpHJle#xZ9j6bVO#^^s+>WI>!ltF|*9H*hW$G-XlF1w=^IiMt-T_UNQ8IBa}+wjFCp#xf@2OvfHJDd_ze_@4JK z<8q*}FO+@+;0IBBgF6kf@2(~EiB$z+&dM!G!d=yG(r|I4W?(P@C1gc#ot^-_9PS-G_@9m z3#-;Qnj3k7nyrd&moOmP4)@mJ5xs?LCN6g?)kfMB$s$1B0umx=W=#;cb1T~c_IUP~;$D6Gz zy>3}P{AqA8?gkPnM;n265zE@DxN__Kp+0IL2H5ZQq;z!M6M$T?5c#VJ%4Jbn&ChR! zGrN&jy;+qs?NA7Fr}=1op$1tO*gsFeSB%8TQ%Ve~&`x!`e5Cxh%YQ-m=gm+V`d5owbfv5QV5`w^{ffDe_TZ9re0PlP8dX@rGx z^Tl)FLV+ar#d9UQ_YODWjclJj_3*z9zcn&_bfg}E$AX?G4J)f@U9gpg$v-1ywy$Fi zmpjKNGIR5jHJVHWphJe5JU)#eRL06fsXFUKD?2;T%{Mg42lM zpZ{z>cznRE8M3y~0}}7)xNhdy=F1vB@ZMX;M6oZE322#Qjc0+!!f2!w9mv(V3mTF} z0Ly<;9vCm5_+bXzsJS~%2taJ z-nF}GSKFS|)!n~xWPl~sj8Fs~hLSQCpJZ=qzJ`!%F_uzDg7&E(cLd$~E0f?c%hO|0 zePEg5BO1lc9$?$0QQZi9RIaG=4S|-a&HUj;W1+Bua7&za1iTtj;Cg!z>3ZK;mQQpH zsBcK6WlVwCLFm2+3PCm21dy!CV`;(Y}XfbE~H>N)3sU8Ws4iaw*+knZYwyphz8CRzpST5^y;8?zz8^L z(6}Ekylm6U^j?r73Uy=Z3 zS&MrUso!o0bulI!&G*N03zok$Sa5)H4Q+YA0okHqOwCP90@h_Y396d(M}E{-To1(Z z{@i4%yj`GoW8ezHIoMQoXOLtdgmfO#x_->PNu`keofvEbsWzm^UZ{v9PSUTya#7kv z*sh5Ja3r?&C|2%ET=Gr?9K@Co4@YlAb>fVU7<60`s5P+j%v~+SI4nWPQq@5*;D9S6 zL7cni7O?^a1FuXgGGZ#T)N`P9PskZ}doE|?a@PbR@whju?uuwnJ85lje|iX62&;YI zsa93^Hah9Pqew^1awuHKI5ZFzF*$haA{C*;&^A;}N85J$N~9g?Ld`C|@Pgdz7Q@oS zdvtUfz2q0eK3q*L@VYgsA2zWN3pgCHnpJt>OQz3{s%s&`H2_x2gOZEDd5$M=S(E-D-%pI1xj`LKMp=8e@zCeDWyR{&pZ!#3>3i#kULZ z`(3Lv=LyB0B|$eZD8soKDr0!-u9tEL6`0QG4P8Pk`2;`9l)~O_{>66NiofvY5j%dj zh-XK`%@b4Eo@BAwwl*h+*Q)W%cq%He|XJ&DU(}+#g61C;A@3ZbKpw z4Qc753KO2!^*gfmWZh3c+UQkI3f4HHp*m!whYMe!Nh=VRYZ9h4*ZeS8q7phw6bdl} z=_&9`d^HNzbihpFg^Ei#>nRsNEJKF9@&%SJrvA2aQQ78}B%~gLtq8oD)tvO4Gt3T1 zD89L-|Bfxrjp!*3Qynd;tfmjWJjItij4Emd4vpE$p1_=w>$*ri(oL6u>?|cNGB5H5 zg5ug9D911Ki!A|PQdKG3>L)H@*}Tt^Zbxm-0Xm~w35Se2C<&s9``n7y!YPnONC?C& z4KMQ)ZCT(HX!n_!7@Bz-W-7<)%feDsg#dk4H=o(5O)m(-Nys8K`MVXUjYiBTcp*Y{@e;#9$O zs)$i^e)PHvg4uer9z_`H#{gFhDMgiXu=QjuPv^wGmgA@4g{meV#IXr7WevLw@%lQ3 zk(S(1irpeka?YqJJ@|b)?1)gkBMC!6fYHhM7allN;jhK8#)wO&5kkG7*TcPeWN~%( zS)(cO`MqQ)$pM}?^IyB2x%%z})%cG2nqFSC zd!j}@s2a))PHOh})N(5FJAJ>>MRp>eNKU(*)1sVMz$S>G&rUePJ{OsdjA($!FpvYc zMj`~^A>so-s6B_z39}69=PrE$g2Wk{WfY9Q@fG0Cv-D=E0eTcqsfe%90*i&KLRN;D z-uL%HtQw*Ibk^JgEoVi~)kI_iF`K|sL2}_-I$DZ^L5{Rc9j@w(xxq9+okPZ`Xl$l} zT7JnSdVz8pZof^n6wA(Zet#57`is7!Xg1mD+|<$DF~`G9DEore;=HM|NAoC4-S&6(((~0*>u|1olk&4r5)N5V3KTQ6ge~=3%VHYOu=B}&Mil7&7p0p!>=GeV51SaUj@#! zn)C;FTZj_3;651Kwg#S8w#2V@r)BC^jo~V9=DyqjNm&E&%nb^4CWIwI5D7avk|=Z8 z@KuZvk3w&rWcP9hq2O2zA+Y-n3$6Z)2SX(|Qqs2A~qyr?IW2W>Wtizi0&^6t%Ju&)UM^n9?Yl|s$JMq^F zZ7quZ;2HLu=g@mPXA`<+t4qR|OHq`AaT!bF97JmUSI-1$HCgg>(1~4+DfNTQ4ON@9 zw941}GEeQ1ezHXn@SuHvMHO2ZlCn7Nx)j622N`GhERjgkLqwce4me}mlBK#YwhX4^S+5wK&UGn8L~ed+ICr}^d}5t%GzGpeO>ZCWLT8yl&uzcNpN*8T`(drZ~o z^qkin8~oM#B_W<-UX^P<*;+q4d&e`cW5nqr1gfN*blWZjGmn#Lvta!+ZXUeRNL&34 z{N6r#bthv@+bl-ale*j((~^w%m8V?1#8mN0HE$X99($W3KwhVUQUpamR~L+q22TFwNlXPXlcZ~Ae(sQ< zo^4$XO~DPgsvYTN;lfvOCk@v%SaM*2FDkkO$l{Dtk^=EA#*j~2^>@`Tt*;WiH>>Qr z5jV#OH~f;@T&(02!7ZhN&Y@|_2_=mWNdA!W_!fA+t>Myh)Fdtx%9Pkf+%RBZM6P9+_VVuLO35R5mj?-U=o#2D?(an&2HTEYp|w*Df_KwJu4)k z=_7VxY(uv(mXJ`;vWt?lhgjw`exTl(H|>RYdv9~a>#TheuEa>xt#{RA7^5p1Mh!)BjBlA%>dVe~ zBcMN7aBvAzn}Wn5GahQOD^z2f^p50a%4DK(hWAKa(Rw;_QZyIF10&<|JOe5@G55-r z%$VfQ2?Q65&DEo;+*od{2Ep>M zqNFZxi>b&x!0`s1Gz<>6yeIs22)Qe_#WA=qbr{dRfx9Mn4T}1pEzU)nB1`F*9R#77 zsVF1~_m8u5t%B9jSj&kwhw9^b{DfXA6jfAboz$& zmtnRFstH`U$c~@LqQ5l|2%|6ZIz6X3#u=(>cYla+A@S6TnmMJkfPn!WTm7iB9MU4W z=<4gwu`mm{J)#-4{53|^^75-bx6!N55g_&Zxc)7k?@b!v*MiM?Uu*-zh>zf6>}^cHdc^vsBmu zWEhv2dTo$+a*Vrp0Mvlt3Tnh`%vI1DkvqAA`slbL7NMxFIR=*sjYs3ueoZ&|Uet@y zgmuZ`nz>MoaF7<=-dCPsdDF>dSqCpn?V7^1rz2x?As*99WMr=9JPiX%lNN?m=I`8b z_3I3DibxLvk^3W63JYOOP$>^zf|c>>QP($}^5Mo0s;XjsSnDjcPiyS+g|C=*2F5;@C zevurqQ7d*m>XpgzPgmuyRe*jcpV1R__RO}aB@$=9n)y<-8EuD?a9g!VLU1iIQ(1-4 zkTEh?$g*QPQuQ<6Ts?K(m1-+n=a5`7b|8sMY~}1tad;Cf{cA{kv;l|WzFNY828B(Y z*wCdu5B|Es`x$q7>Wy~M`Lu<+olWYzuu!c?uQz{R6ZEAXzJIU-l3SssXTIBG2Gin0 zLbXQa-*nDtcF|h00tRt4Mf2Lr+&a#D#jo(=g$;Ri$Kyrb;leJeGUJg*W{|;CBrElW zGWZ_Cl|LSB>H@0Nnk*dSx>j%G^A5aXEpV?N+$FpD5bKGA)c*XnXrpYRV_FM z@KJ-8H-UV`ryBM~b)xXR6t7PD*tDjz60IvKul`Dyz%@Z1#9(DZQ$bBywtS8$TLnG@ zNA4ud6X2~CJ(t&H$AtA>CGO6=dR2AA+X8blSV8Hjy$yl`lV+gR%+uG@2~W!{L75b& zI-_gdbD^?OXz|&&2$q`tN!q&MBp7QKXX z0n^SmYtV9kcx$tekNoC$A+rd~pQbPTXB#zv(n=b_lo%WNQb0($l4a^Sn=muJOoGrSD*;q7g0 zueO`M&C*zL{0u2mK%wGZ{#Ku48Z2%Ry zK0MFVh*E^p!BNZT__jM#6qaJrki~{VcD_Uxtf)$ga z)okxD0$5EVrKOhx&+069WZojlFSclSLtI9LZH9$>DanW6OF`sx(ueBS4!!KjbBFK& zsxg&$pU2;!iJ;e29%F=%uk-b5Z!SzRc*QkiHuovkS&^t88Ak#y^X@RF0_bK7J{E$? zfU5Zq)aMN))G>)7tfy!WTe&GfwA5~@Sx)JaKUdY*RLgW4HP~U~m&L^V92zgiCuJu! z?LldPX8iA~JTbTz-A68n+`Hn7C|Zba(%|3emy}$vwsR$OgYNPROcS_FM)MK@mkTs{ z@4Uvj`1xDx(d|E>HwD&WHzKTbC3BygOrP`hs$F`ZdZ&2E>hG#Uod?XX@m7fS|p-v7GQhI@SA8CN|{NPx&jP zmWim65R5;`lrb}M?B#JT_1H5lwf{WRoG+-Xd4k(k+O?okD02vI8O-MJ51MzX;!s?2 zAXu=fDys5O{(ZC_y=Cnlw9xy8K{FvJ5G(q4Y?G*dSYvg%62!VLWFlgH_Tf4ij}vT) zZ1@%Onf&X9)$wbzJ1I7~;|2$Kfi(lV0z}-hg-W&))Hw)>2<;fahIGCudTq5Y%Zx6y z^#$XOrDG)QO$hJX>f}&YPkw*i4-vHLjFaj;R5u~~%*BjrlaIYAnY=Zhxh-U|MUph9 z>H-R99BcHDctU-gzYMp2} zis=YrH_51Da6jnC{5st0c81c_QTK-6oaFw1;C1rg#?oY+%;Lb{hZXQLBu^98%#N4k*}5{;c+>z!owE{AAU zsWn{|_N(D{$*MShM$k9ib$(&{M$XzNTFt`&+B|7kf@{R7ySlDnDqJMREWE+-jSM>^ zC~YXC>2NbwuahUPXf9QJWY|TK0A=m`P+WN{CInjon37cGxw5_02_H?VNt4!q#8zJp zW&ism#t=^Tu*~B@kKIkqlzEPqlb->Kxsq3ALjv9ul)CA+XJzJai9k5{P&OVLbV$;JOJz3bmFq4b>R|f zkbU_e1M%=A&My`@nmC8c?$?l$aP^`SluGF_HaO{gfdu|>X9Lw#q6H$lAxsyg<*I^f zo(!aV$mk=ED_zZY=Q87Nw&sJ9)wwuC6JpV_{oLH}Q@NU3t?A7w zxe(LmSRmylJ>Y!U5a$symlqrAg(b3Pt$y@>8uz>)p=-A0gE&p`SB#}VuzLxU=r$C% zR2`qoi?~5jOw>|Q^3}^sLww!^NihkHAtR`4mF7sMj9kEJV^jX`g|gzDJrrLUVq>{l zE_9Miac>g7oo#Y$j#xpq(EkI4=ZiJZ^1p%bh<+OZ#-*E4C5(=pI1%*d*S7u%C0LD| z5A!kA+z8zr7EO(NssX}pmW}D&Z5s@a9?rel2buh}ze~v!v{oOqY&;|3*O|{l5sh2{ zVAp_WfE^MfZ)%i>(g|+f+T;7s@(ne5pV6f9^*N8q_k+_fPbv5{=>#!9Hl(15y6b9@ z+jk#DD2^GhX$a#hguV{LW^H?seD>%`dnvJrEG~|2E;TesuMY?pe9^cG4oHqZ@RIlR z)V!5QxyT79qB>Z^S5SA8mGs2S#(@&N88tR9_`rktJ`;OjJfk{OzE)QZ;w5Lb9CXeikIJ`nSd9QNMFPqGdq+h$|Nd231)|A!T zle21it@us6mD~_A^~V9JvZG-qa=2ak?_Z!$1@6O8iK;&(%S{jIcIJ*9*hLg{8`X0b zQT6!J%J-|WUiif*6RSY2=|`<1tfUoI0#}w2))%>?ceb;!qmrHxR8Un9T`7{6PhJMs z>6r$rFYnW9=+|x?iI2o1vLpgIFS-Y^nX%Ygr_ey!0iqi#_QPMkOHVj>`d~iuk>eg0 ztcq9cGh-SBko7Ci^KB{yEoghTmuM6Y5QQBGIHKEN6=<4U8qp<7>J)hI=9xMeU6#eC z_{oN$^W`6ql;aJYikdMHNwVKU9rDf62V#hS)CQ6Ja)eD-xu@@UE>jiJ{@9tVrZqj6 zBR1~xJ1K2lfz^{0vbN(|9&MWS`Q@2QnX+{Xq-L3tR!F+=-6oXE^aM&n(piej&fDyU1sz+w~MAtQ^>X{a=v=f zb#>*eKa-8fkv(iH4PK%L*Emvb%s9#z%hZ_+>rc1%qU+C`BU{Q&bzWVI#y5v-Gw|{< zvMTIGbVNg-)r>bAXWpnUzI50U%2l#SpV=3D0%|C!Q1NSdv;LTIyZW-?IKPCb-j9aO z*eOw(T56qFLH&ixb)&xBeJRCkft6Yi?fa;Xm1fEmj$B%&W^O;@As@yQeDT<2G{s=) z%f(~NOlfL%3qQltYtveCJ+dX3DUas0)r!pi#O7w>v_|=Rzp(~6%Qri+z@0+U41=d* z`LXbC3r#0Y=pzOk1kk|~l zK6t3Z!qMUvF{EYVnOI@(KY;I@2mI0*oxARw>+T{TwQvMW_nJ$liSl{bMpw?dq*{BL z73AQf>Vj>^;f+ZCfLM#E$YJx&2)GQBQ{In-zwS%FqCHwsbTbA!P%p^3n_6|YhrPPgn_$s5POpta-}U~< zfrHVPH_M!FzCOLARj{CZmqoF!k$4W`7_SHCTbjDhl+&&@gvsqY9qrCL-mqNgGAPwN z$*fLJ_~qDjoSL0(f9E~%0w;<_!{4;_*wrdk?TdF+nh^_TMo@eQRVT$ETL#tiI1?8F zD(B>D-Mi^Lmp7$%zC`4f=2UZ5at&2-sYjz^o47^d>S>iz|J6IgbRa%C%tn=yY{ zf$R|3=$mX13)zBTT=rPtt&o*O6w0lF46)I{GH=E7)EN0KMla8PVe-#f;Jslu=h_(N zhKe4yE3Y<*d5rME5Aot7e%zLxL|24o!``M6pedi4o!QaguMGIQ=N4LT#$+|`3bV8! zkl1$}k!JA5(Vy>Daj`2Gsm6P??Q+goJ{_Gferg_N*U#@*O_W`-vp&z!KmBp5>j{Gi zYb9U2+kJaR(wEZ9%qs&Knc_JoeDCLLQ4i|EJ01!Gk)sWp!!5hmL(_gFS%JhYj^ry{ zQ|;AtR~*KEgjm73{-hR%Key+Zt z?7EQ?p=r$LX$gGmnx_W-ftEA0a^GwClYUr-c(fVN&mZ44d(#yZZC5K4Ct*UqKJz4m zpG)WKAvB{3NwjX(NshHL5HrF5GDP&4G~VN;x=vk~w&+!CMo-fww0<3qLX@|(55M|YeQwQ})YX{8D^~<<`y!XX-D3Mk#;MB6 z8!1o4a>vy8?#Z)!zw~(x!ou%7&QD)^e+5xKgXD=@*p;Mc(?Yl=f|sode<3VZR*8ae zcP1IO*iI~S70Z7^QZgbBDeJ~9ol2JkK@qyFwt|9j!w!fuIdG4UP%i>| ze0_O*+TQ{HvXv#nsn=~**i5Q`>qs3mB) z)%R?>E6kAo2uvSkg6v6et0B0aLhH-^-e|>EuFJ)4EBc;fqG4+}`)pBkeF1*)o1`LJ z^UV_NE?VOh!QJd1$I`O3G)hkFsTUu3*7x8;=|ld5kxU*6JjVgIbVU%ve~h7p7?u~2 zU1v|Sm`hz2u_1DD?;ro2cqGwxWH#ZY(TY2WWEbqgtr?BQ94?iK#!QjXV$TOY^?cO( z3pgnlrz^9i_NuF)Fn6QgUOG_Qc$r+~(kB*e^-)O$TM(cuA552u{V8Wo^85ah*Lz(U`^Ti@K ztl3O>h{q!3K($-5ce>NV{F36AT5)PEF)`OZiF-#V$oSQ}Tbieq+m@8Gz#1VTqwBqU zgH8Swzsn7~@ypwKpQ-@SL`18mCE%fx{4^Y3pY2uvv6+!IUNUuJs%;z3YLLs>aMFKw zj@;wUBA%`~5t#{hTIP&;Lic)ln3LvKs&OZvXZD9ffVt#_ICaW_&Uo}2Yc?)nXj}P# zF~0?!w*;<4)-x_+QUKr9^uwe$*PPN}>j;a)oJwMI{c^hS7!*5xPmt ztET1WKHl7=x>$|LES4fUn=`#f!4Q-UiH=-MH-$v>zH=eXkKz5kki7f?62eQqt za+sqPv~7v9v&e;gL&g?ljPv%|7&<*Ir>wOLF{KbX^xjs-E>#Cr3JK*5pCChqoyrX+ z+>`I+k{UH#I@fW0cev2_tNf+&{7~J&c8C^aS)3I=3fr@&no8!qVrrf&IoU(Kv*gq-Wqr^XO-E0LX2zf<6>P7Aq@ zF=Vm-fWY*umk+ZIce4I*o*bFZSe^dui!%c$`(Ob@AD3x$vBe2zI0QZ;J(m=(Gb1kU zF}q)Y``5LB)W|U{lgl9O$(ZRMD2|RM@F!rWn3thK)rxcA*N6nb7VN>C2jz^9h>oIu zyUu$~1v1+?^BnriY(Y+zA-tJ_8ym*J`_JQ>Qt+-|l7(HlY*)teiz@p-GAxcH+oRT| z;61t3H5<+Gl*9aO0JAiDgPQc|G7kg{M)&+Nkk*n7np1}>BF13&90E;7YoPD7mNj&X z8q3unqR^jPs9UluU{N+C^XqjjKEZQhnIS(&cKbGDhv^~V zME^*{vSUI=%lA1P;wzzB;7a#n3Nyh8AvNC_D$d(9wa|T%_vQ)u-}W=IGC{!S)F&Mo zs~Y(~Qy<9x;f*5(+?W6W@mZc(6fhXkfB++r&-KlS5R-l@9-@Q@vrni3hO{8aq&Zuz z?6?yM_(-YZ#Iq2#623~Ts-yL6!}@evsYX|1Qe8U>h%G7vBqonJC<%T;=3O`jpEP0? z9yB}wag$TEZbx|XLeMK)LmUEtEmAErmHZe=+p05L$>2V+Uet6hY7uhCQBej| z1rS%m6jF#UPdjK~QI$t2Av~(uMptv-WPQWfbYM*>6Mab&Gh(k)q9*iFAo+0dzOxIJ9tt&)psw`99@KOK zJk=W7D_cZ39R#@y0UsDV_2-WKeqaRJ%0-d0A!J!>y*7Gi(m^yZJz1h;oobow^oz8a z?H$xefG9fHN#yFlvuQQEJsIEVw-HV$oIN z7*c(?+IX4^rhoeYC(brOyJ1H7E;?^l=+}AjJ&I~tnt71mBU@93cC$znqe5AFXG==5 zmZ~Ts;VEV&y>^)^yn2S#jxnzIenlS zKN<`@{mM$1OsiM5Vhy(#xKM&E*SDl&Bj{@8PPs_Jc;>5i*eQXZ8qB(+zWB`)L$pNw zuG0uX{D|EOn_+6D40N?M^V8ac&Gw>X_673J#_SPGrjqq^b?X9Aidm`tMaT@K8U3T@7Xo6vjs3V!}xa(Xbp!gW}3{ z;ZluY@U!~8Q)8@cm02726l{XMdKt%JclH|H;qHuVopY}~lc8HWQ(uyRj5x@GIH(ep%vO>--{C@oy`hJo z0gwbp0Uqcn3}4yQG2mNg+kd&WCnJ7~Q3+&7SVLk>YcDlgFO5(VLJJBvGRvod{NpYQ z8#Hq}3nU>S0gyOUKk)09N`rEZ`Tf&q z@fMy=$nA~Pvr9;&gcI`;{vrzzp&|Dyhu(+RY)&DPlh4o?4rjElmb*z7lXVY~hI5tc z4IkAsDq4)f27`B1sGk%LqL@<2#UbU<6`_+NA=9>DCuLIB3jL9}`z9r7t5CaJgi$+k z^HcLn9l55C1SK&iU7X4*J&4tC7#=ZDyk5mLrMPqPPe}dyB;h{c&$otEyy-VwxXVW} zQP7>LrSZ+!^7-FV4CUf%y8Bm&qi8Yq9%LB zUx#xJmpaC}=n(xnl!GE3@N=t>hnxMs4}7`_0T1nYCa0=YcK9$>E5|&%?(&eyy^YnYE6wG8SHbsraH6>?U(QB77tu zm7O7M!*uNAdI?b+(nP)kX_D`)^Vbi8_rnfm{1+Iq3w=gTji9mV6FKx4x=xPat68;9^y&QbzNnL0tLINp`4k_{ zuOqHRsZ3iq$aIxMlouCSA^AWC5aaJ|_Y6Y>?5&}**;6{bQG#@vzu zrd|fCt_0QTYr?si4`qa-D5Mt5g-v;MbOHL`?NuelUwYq~ZC99Ez2f9zk+yBz7#hA) z;=eiJ+e_H_WR$gRx=mU_Cr1YJ4^=0N+rFn{YxNc>RA|`B!7uig_Db9t`y;RX&HP=( z30p7a$B(FqKkiqf#|XTRhxlr$aq(384xjV)PRBkzO(l;LIIkV={o`(yA9Vfg8g43^ z;Ki|a!=prvWZM2xqYt339Xy^?>mhyWOJ47p2?O(I49;A_4 zb4ATW1SRt($@e;WP}x&P!ufcIBMF3uN7&jQFLc_WbgtV*UvF!=%{^;9VD9k;nnioo zq3Ee`y#^DvpTg1ahiX?AE)o)hO4L*zvZrZP;} zXYyea>ndtrf5r-TE;Z|Kk55;i-zT5h-QHSb)F4$eTaiBWclk1z_55|T&ROtqar}zk zo3t!{uaT~(S*2yWUQE^Fv_>vs_mXfvOo9hWQ(O~#C}Td2hxR9V3&c!|^}j)auRB#0fXm@+U(MUQ^5pn$RqJFFp7;7uNIm<>Nn4;JdiOhO#@8?u z(ySnN&f0yiIcK8b9jF_BZ1it&-VqUn!C})kb;I{ANsu%@8Swogg3P?|_Y9z!WLkOW6PpZmGiP_SA|g{PGh|zf!Zo5>;bAuNsQw1xZt0Au~w``*9~EUh}^$mqk4( z?(jY$9Divq$*^(Q4V6~f6N*u%d5%zKw^-tKw>t}-hkoSuzu74|O{rOSBkuUVQu=rI-kf0d3(RBIh2NBvhX%IL#_&L4{AFZP`(4#? z@~AvzO{?iKo^>GukDjJuwngHvb9KQdP%D= zvCYX-*J>^zx9x=+ZH?x|D$5S52h8X}I7$W*;^(9-9ci!^#Ws3rxx|IML%saOA6-7F zL(KkMXYq*e@0+yjwWmGWtE;CTIjV*$1Lkm%NFE6ChV9y(!Kca{U$_*vc!Q`#ZV6KU zP8dFa`Hef0ap;wu6On#z@V#x-?k**UU(XD>H9Skvj%+VJ#mypeQ}2l2-UN-`WfOhW zLatkuD6NYrIZkd-_@h=tVu>slU9p!C^6jA4@CtO(T_FV)W4$u7xsvzU&$ zCXvBXTc$wCeBkw{Ctl0wpU>bm3W5^7FQwm5;dyX&wvl^ZH=P}0jkHHQ^+k;sU(O<@1 zxx|)d=_Svw_|1!};xUk442sOL<3t?sHBS42;KMjer zF6K;QjsD3&z#MM-loDA(DX4HTQKU%$B%LT=eB|o_CG2MLwoFhxbnX$ zeWH*`Xo>Jo>BLir;!5aMD%B*yf+)QluUN_HFDD@BFgeM1R&CQa-0m(8p!{=n+<1@wlp92?6O! zcH!;3&PDr1fLMrUBc_D0-3?RHTR-7dAuo&M94vAwAFD@-cnDOz2=*~Z_vy2E6m z#lMRDHx4-!Yj^QWnwH_|N!p3@!F~_ePO+n+y|g=l%0y-UuM=kxS_7e)Q|s|Ba}k{{ z2-6WpdG8#`x|W@Pei1BAZ-{F!$j`q5#>0N$mQ$ZZKl*>wwO{1gubtT%hWE$jGAiRm z8`V~;*-I!B=G#^v&o6!dO_QMx&Bt=CcK6V ztKPy2!&ms5RlT$MxILjM^lAAAgMmK6~H6 zjTogU0h?cUPCk_JzO@qKSAhK#q|@yd#YVru&p&P4t|UHa?H_mVa5O^a3+w?~GsJUk)D{A{Mey15W z@60YWZ9PvN59$0R>p!(joF8Xkm04fmE(8@P3=hc_*p+diQ|G2 zQG%>|{#IF%`F+9@j*$EgaZsb>Z};jdPw=#|kd~dj=UN3HMCuFsO=l+?@s!mc-}ojM zo5w|Cz;craDQx1vi^wnM*;MS5i>5%*H-#H!U`S_XfSLiy(1Fj2q$W#pMc=ItWZ?tdhfu!NLsgi`qS}nx)yE^S?#7`SNVIj z{N8qG=ha>}{!{~oCCpZ;XE0thKMtk-d}J?C&FOo>0kPy*S_eGbW}1<@L-bX$$L)Sj z;4VtHOZ|zo8yk=ZbTEHy+u+TgRaJ) z|5zkb$}Fl4y_Xh04(3B?hsWRUix_ZRe0dZiUT#Krzjy~7z3D6D2EWkGx;h0%g_rH#us$_Afm4$929@myk42m{zqNnzf!d5_;dgGkVl` zu6r7`LTD*t@Q<}l082vQ)6n(*nCeJ~raryQ{$uFUV=;hcAYnCtV<2HO00%G@yJ0c? zV;+OVs)qbu<}pNoG3@8HFU} zP-3(IU2XJ~&rP5tnsj6(#l~iMbd)yGR%QY4SYi2AQIMh56v<@jR*-T~qNVVpWHuIo zq-Ob;`H;z|QBsP!RI)Sd=Dd9JZy|SWCow64tQ9fUhec}2K;ccC zQKq7_O9#y&c0-0c$eyJaCjQ8#8fIMS&jaVZV|VxNdF8($-}40ki|LKRMuJ7rd^o3= zlU$sW9joJNHdM#mYFVL^EyEXVIhvU_=Vx}G`yxyYJ9VM_x!Het`r+8L-ELIgV2S*f zD*wa)u+J7|`;0$>Du4tP3_v0R4j=)*0>JVC#h*khaNHP901(N{&Geg@$yXOsD|0(z zEaXo*B{&fB>sM}MhD z1R^R}0U+e*r}99^f7~Hs18Dxo6&l(n=^tLi|L}fl_Ah7W9VB*YH#k)~kpPgw!oq@u z)xzAAk(+~ug^`2x(=iVZ4;w3^2`h&=H?Iko3AdREkb;GU#ms`$!i3R`lf#UW!<3DS zk%xobgpu2V^>am^g`1m;jqUT{|36CmZ{^|tqx`>h`LF)1zSwUpRJH5i0zlZ$5`KnT zVIb^B#ofH3iIMQ98vhsSe~bI9M@U@Ye>ca!5??W~XVIvN;~=QYqag@@kfxt9|Eor^ zL16!r@F|F@bq4|~AN=3C1^h?7|7y*D#nppj4}ep>WlpNWZ`9GFncXqLMuqSb6W#VMwAfYjFv$i!OVc{`n=P>2q;5MPf_4#M0 Q|F17V>a(M&eERtQKc3rCH2?qr literal 0 HcmV?d00001 diff --git a/db/parquet/aaaaaaaa-5555-bbbb-6666-cccccccccccc.parquet b/db/parquet/aaaaaaaa-5555-bbbb-6666-cccccccccccc.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a4ad0ce2e58547bf531de76b7011d6c86987a825 GIT binary patch literal 1545 zcmWG=3^EjD5RDPd(-9Q`GT21f7#K9R>i<>HD`aF~VqjnpVPpi7AQm$avj8zG5Cc^) zumdp%5OV@C7Z3xruVEAY!Hi~7!Ze1d>*sE5pJyk%>eEEESe^5SBIdGu>bzaobmS$Ec{`h<0fT}7 zi=)Hg=$5kiksZ&M&E@>a-R~NCQ$Vb8b;|SdW#I?E?JnFOv|(E_hlg7PANTK)uFo9b zzG;c&3+V+i^F~_+UE!18`G-4Yy^r0-jeC_-jvO*`D*e%SS6R@Sb&G!tEImK5bxPc9O(*YtSIQUGp4c>5V;vG_Bm z+(G3Emi`BVHcjO#m#%uX^b06B7)AFnq6LRS*hVI3!1C#Cn)`nFGyA(LY>iW$U)#5f z-8}a___F<?3dNeat%@6P{kMXVM0#*lz!@N@Sh;!$?)RFFCbpd@={aePTpDNq`ui!~{;s3ZfV5zNcUEC#A%U=S0Ll##4aV~_#bFUlYZ zu}PFiOhl|sYyzVihm8aaDECOPfby9bg9Z{$P!K2t(#ZqX33L`%-z7G&c`Rx(7;Pj} zQcaUAEX+)jbW;pd40KIQOpSCcQY?&hO%2Ucj15x~&C}A1BvcG6%*;&R literal 0 HcmV?d00001 diff --git a/db/parquet/aaaaaaaa-9999-bbbb-1010-cccccccccccc.parquet b/db/parquet/aaaaaaaa-9999-bbbb-1010-cccccccccccc.parquet new file mode 100644 index 0000000000000000000000000000000000000000..273843dcdd512f7eb7ac80fd1cc1f3ca9d70d20a GIT binary patch literal 1545 zcmWG=3^EjD5RDPd(-9Q`GT21f7#K9R>i<>HD`aF~VqjnpVPpi7AQm$avj8zG5Cc^) zumdp%5OV@C7Z3xruVE8yU_mn}VH(3$b_OwVbCC(6Cj3htyesNivrV^eYgmn*@)u@f z27LwthGjo~{SWx@*UO0i{Wk7o9i8Q3Tb6Mw&EX7EaCLv5Hu-49+f!2__Lv{@3~o}8 zU=ZMFQc&1Z8Lp~U_M%^RvcnhVd3RYI?Wa^s+<7dh`0i=x6FqLbvnLd&%$?D<`s<9% zSEpWQ(sujlnQdK_&M{N*-EoDd>83X9$*D`4_XX})-K=_4P-x%r7^hj?E|UK55}&;2 z6|e1H{Vb2ycgL3uo~V1bKMHy_uILkfdvb$9)7cf4>{H)=__JH-n1Q>G=uFGXF4G&9 zoD3`D4pPLYrK8L;62cybBv-n z==P-NGBO;1q)<@Gj5dhn)KM_hQ7~eQWn<=G@UU$7Hf<8)J5fH zBPBICKewPH6{L;6%zr8tNt{S)}MD8kw7!ni{1hC0STVs2G@8CMKq&S?H!3 zBpd0Pm>Q?*CR!M#=$fY`r&(GUn3!0aB?4`xqG@oKbAeqB2}OwG8O5G3s_kR7k>G%( zWeE;g>IOOjizwVMNC3j40TGCc*ny!4O4brApu`LcMFbD7odN6?Sfqh<8;QB7Z30FC zBclW(NE?`jYk&khBwoSFPceyIVpRJE)WBtEX<@EwV5)0m1Y#Q+=o*;m8khi$z#)Tf zC^&h54gJIjN(U#nY$TY0iCBU&+5nP{V}XXDiGU3e>*E&_QR5K1z|Ekc;Q~y|E=~$% zsYS(^`FRRuhI*!YCJGu!rI|S?3I-Ob#wN)oCgw?+k~K06448;P6qt7cfRz9MKYEJT literal 0 HcmV?d00001 diff --git a/db/parquet/dddddddd-1111-eeee-1212-ffffffffffff.parquet b/db/parquet/dddddddd-1111-eeee-1212-ffffffffffff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..784424c5e10a2de80b575aca9cb1e58f59aae10d GIT binary patch literal 1517 zcmb_ceM}o=7=PdEwUknDb-l|~7^w`-&T)8GT6$GX=>m>L$-3evj6mBfqYesd0d?S< zS~8r&VfdPh8lx`3uVpdWhG8UQOq`pUWE!$KrfkFR%S`R9A`n7GjNvX@DgX&k1HcQV0JMNq zfDQoRC)Lz(RT5KO5PhvdhrE2&fqlIh`Uze?;h7oq9?S1;IW=~8=IYec(RbU+A4DvA z09|_i(RgFqulCB$+pkz|ZOeOS#|!7W1bVPCt*SESJ>*`dC!rkDZC>`$GkI+LmEBxl z$%5+r^J=%N|HZezn)F7`pUOP3>#@Xg1!sR<~|T zEcUxF)Em#)Fxl6^IJaIZTc7#nXYp`)#mGW~zUt?5^M-Y=4rdEbN%6T=({-1rk8XW@ zaK%_*>^nk>$a(F>^y4c>*4FJlnmzdC$@GhG=+l_`948&xys3#mrQ}LzO9plm=#gL6 z_Y8&OuHnhY9dEeLx;l*e*Iw8<=Q`5b$B<7?yH-?5;Bvv$GvvB_^ON~A6*pWX*4or6R>G1Zcsr|e~nV#NKB$9-Lea^P75U;y%HchJZH&8~Pdt>_L2lkm%`icJ8)uW?_7k)7Gc=J`8 z_I|N^^>qJOxOvCGpT9}pgeP)4;zmQNkwm37Qj1E6?G=^PuaA$OV4KzoB2_LN#3^X^jv<78A z1e+3ZL{|?twFzE6q#JU$m#P*^&%JK16IyvFAQ=%QII%CeW2bc@-~~Z$VZcw|B-E}a-C9n zqKiuL3z$BG+1J#?jCN7Xj8+ji$Qz3Im$yq~i3To>LE+nv>3+=qsDU;r;szN(8%fI@ ztEG6P6tg%rB|Jhu$zDk*W-w;39Hu;%0up^s3|4WUBIpyCJpx112~JVqcn4<{Wn17m zzKG*(K=B_t$)t)t6w+>j-iz6ou9#7Qn3>dS{vSO9S<)MYg#PIs+Q@3@!8ByCl!BT| y*O=Rbk!YwnY;G3{>;*Qn#orccl+3&sEVKn|Hiv(esWS_qyDvn;pW6dJ`o96F?RVz@ literal 0 HcmV?d00001 diff --git a/db/parquet/dddddddd-3333-eeee-4444-ffffffffffff.parquet b/db/parquet/dddddddd-3333-eeee-4444-ffffffffffff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..485cc88fe0b19b5a7e38282fe3605888d579c753 GIT binary patch literal 1402 zcma*nPm9w)6aesPw$>n3tS}*kBD00otSsF&{|`k`_Pi{zH(g_z47=imLY&d+j3L!3uAt2k>_oQh6Gr=n9UISrkL zPD7_vaymL4osLegvS3O9<)FEaFLk-i3?g;X1zw`K&vSqj}7`>&2GuF8H3$pM`FaByQ*xby`oe z{5emLnHSAwX?VOgH{2!n9LML$_Ihdi$W342kBm)Fwk>)BzKh;rqs)qLVnr;se{x}=21y*wyEHk zb>IeY19={m`9?1b{^lBR6S#>yk4pSX!wYxb0B3=-(B)Cd{r^-H<$h-6Y$#`Axje#h z(&vUG_C@LYb)qQy@PFBV%$7V(f+%Fm(ZCq!Oqry?)MrDRYr3cF)?}xB(IMntFLDNt J(L;FZ{sy+<@F@TQ literal 0 HcmV?d00001 diff --git a/db/parquet/dddddddd-7777-eeee-8888-ffffffffffff.parquet b/db/parquet/dddddddd-7777-eeee-8888-ffffffffffff.parquet new file mode 100644 index 0000000000000000000000000000000000000000..273843dcdd512f7eb7ac80fd1cc1f3ca9d70d20a GIT binary patch literal 1545 zcmWG=3^EjD5RDPd(-9Q`GT21f7#K9R>i<>HD`aF~VqjnpVPpi7AQm$avj8zG5Cc^) zumdp%5OV@C7Z3xruVE8yU_mn}VH(3$b_OwVbCC(6Cj3htyesNivrV^eYgmn*@)u@f z27LwthGjo~{SWx@*UO0i{Wk7o9i8Q3Tb6Mw&EX7EaCLv5Hu-49+f!2__Lv{@3~o}8 zU=ZMFQc&1Z8Lp~U_M%^RvcnhVd3RYI?Wa^s+<7dh`0i=x6FqLbvnLd&%$?D<`s<9% zSEpWQ(sujlnQdK_&M{N*-EoDd>83X9$*D`4_XX})-K=_4P-x%r7^hj?E|UK55}&;2 z6|e1H{Vb2ycgL3uo~V1bKMHy_uILkfdvb$9)7cf4>{H)=__JH-n1Q>G=uFGXF4G&9 zoD3`D4pPLYrK8L;62cybBv-n z==P-NGBO;1q)<@Gj5dhn)KM_hQ7~eQWn<=G@UU$7Hf<8)J5fH zBPBICKewPH6{L;6%zr8tNt{S)}MD8kw7!ni{1hC0STVs2G@8CMKq&S?H!3 zBpd0Pm>Q?*CR!M#=$fY`r&(GUn3!0aB?4`xqG@oKbAeqB2}OwG8O5G3s_kR7k>G%( zWeE;g>IOOjizwVMNC3j40TGCc*ny!4O4brApu`LcMFbD7odN6?Sfqh<8;QB7Z30FC zBclW(NE?`jYk&khBwoSFPceyIVpRJE)WBtEX<@EwV5)0m1Y#Q+=o*;m8khi$z#)Tf zC^&h54gJIjN(U#nY$TY0iCBU&+5nP{V}XXDiGU3e>*E&_QR5K1z|Ekc;Q~y|E=~$% zsYS(^`FRRuhI*!YCJGu!rI|S?3I-Ob#wN)oCgw?+k~K06448;P6qt7cfRz9MKYEJT literal 0 HcmV?d00001 diff --git a/docker-compose.yml b/docker-compose.yml index 469a9b8..7387550 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,11 @@ services: - "8005:8005" environment: - PGREST_ENDPOINT=http://postgrest-test:8080 + - PARQUET_BASE_URL=/home/datagouv/db/parquet + - S3_ENDPOINT=${S3_ENDPOINT:-} + - S3_BUCKET=${S3_BUCKET:-} + volumes: + - ./db/parquet:/home/datagouv/db/parquet:ro depends_on: - postgrest-test profiles: diff --git a/pyproject.toml b/pyproject.toml index 18d9ded..e65a760 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ authors = [ dependencies = [ "aiohttp>=3.14.1,<4.0.0", "aiohttp-swagger>=1.0.16,<2.0.0", + "duckdb>=1.5.4", "gunicorn<24.0.0,>=23.0.0", "sentry-sdk>=2.49.0,<3.0.0", ] diff --git a/scripts/generate_test_parquets.py b/scripts/generate_test_parquets.py new file mode 100644 index 0000000..c55db9d --- /dev/null +++ b/scripts/generate_test_parquets.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Regenerate db/parquet/*.parquet fixtures from db/initdb/0-init.sql COPY blocks.""" + +from __future__ import annotations + +import csv +import re +from pathlib import Path + +import duckdb + +ROOT = Path(__file__).resolve().parent.parent + + +def parse_cols(cols_raw: str) -> list[str]: + cols: list[str] = [] + i = 0 + s = cols_raw.strip() + while i < len(s): + if s[i] in " \t\n,": + i += 1 + continue + if s[i] == '"': + i += 1 + buf: list[str] = [] + while i < len(s): + if s[i] == '"' and i + 1 < len(s) and s[i + 1] == '"': + buf.append('"') + i += 2 + elif s[i] == '"': + i += 1 + break + else: + buf.append(s[i]) + i += 1 + cols.append("".join(buf)) + else: + m = re.match(r"[^\s,]+", s[i:]) + assert m is not None + cols.append(m.group(0)) + i += len(m.group(0)) + return cols + + +def qident(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def main() -> None: + sql = (ROOT / "db/initdb/0-init.sql").read_text() + tables_index = { + row["parsing_table"]: row["resource_id"] + for row in csv.DictReader(open(ROOT / "db/tables_index.csv")) + } + out_dir = ROOT / "db" / "parquet" + out_dir.mkdir(exist_ok=True) + for path in out_dir.glob("*.parquet"): + path.unlink() + + con = duckdb.connect() + for m in re.finditer(r'COPY "csvapi"\.(\w+) \((.*)\) FROM stdin;\n', sql): + table = m.group(1) + cols_raw = m.group(2) + start = m.end() + end = sql.find("\\.\n", start) + data = sql[start:end] + resource_id = tables_index.get(table) + if not resource_id: + continue + cols = parse_cols(cols_raw) + rows = [] + for line in data.split("\n"): + if not line.strip(): + continue + values = line.split("\t") + if len(values) != len(cols): + raise ValueError(f"{table}: column/value mismatch") + rows.append({c: (None if v == "\\N" else v) for c, v in zip(cols, values)}) + + path = out_dir / f"{resource_id}.parquet" + con.execute("DROP TABLE IF EXISTS tmp_export") + col_defs = ", ".join(f"{qident(c)} VARCHAR" for c in cols) + con.execute(f"CREATE TABLE tmp_export ({col_defs})") + placeholders = ", ".join(["?"] * len(cols)) + quoted_cols = ", ".join(qident(c) for c in cols) + for row in rows: + con.execute( + f"INSERT INTO tmp_export ({quoted_cols}) VALUES ({placeholders})", + [row[c] for c in cols], + ) + + select_parts = [] + for c in cols: + q = qident(c) + if c == "__id": + select_parts.append(f"CAST({q} AS INTEGER) AS {q}") + elif c == "score": + select_parts.append(f"TRY_CAST({q} AS DOUBLE) AS {q}") + elif c == "decompte": + select_parts.append(f"TRY_CAST({q} AS INTEGER) AS {q}") + elif c == "is_true": + select_parts.append( + f"CASE WHEN lower(cast({q} AS VARCHAR)) IN ('t','true','1') THEN true " + f"WHEN lower(cast({q} AS VARCHAR)) IN ('f','false','0') THEN false " + f"ELSE NULL END AS {q}" + ) + else: + select_parts.append(q) + con.execute( + f"CREATE OR REPLACE TABLE tmp_export AS SELECT {', '.join(select_parts)} FROM tmp_export" + ) + con.execute(f"COPY tmp_export TO '{path}' (FORMAT PARQUET, COMPRESSION ZSTD)") + n = con.execute("SELECT COUNT(*) FROM tmp_export").fetchone()[0] + print(f"wrote {path.name} ({n} rows)") + + +if __name__ == "__main__": + main() diff --git a/tabular_swagger.yaml b/tabular_swagger.yaml index fda2cf2..546f780 100644 --- a/tabular_swagger.yaml +++ b/tabular_swagger.yaml @@ -150,23 +150,6 @@ paths: type: string format: uuid example: "aaaaaaaa-1111-bbbb-2222-cccccccccccc" - /api/aggregation-exceptions/: - get: - tags: - - Data retrieval - description: Returns the list of resource IDs for which aggregation queries (groupby, count, sum, etc.) are allowed. - summary: List aggregation-allowed resources - operationId: getAggregationExceptions - responses: - '200': - description: List of resource UUIDs - content: - application/json: - schema: - type: array - items: - type: string - format: uuid /health/: get: tags: @@ -186,17 +169,11 @@ paths: components: schemas: ResourceProfileResponse: - description: Profile endpoint response; contains the csv_detective profile and optional column indexes. + description: Profile endpoint response; contains the csv_detective profile. type: object properties: profile: $ref: '#/components/schemas/ResourceProfile' - indexes: - description: List of column names that have an index (for filtering/sorting). Null if none. - type: array - items: - type: string - nullable: true ResourceProfile: description: Column types, formats and statistics from csv_detective (encoding, separator, row count, etc.) type: object diff --git a/tests/conftest.py b/tests/conftest.py index ab13050..d8df34b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,12 @@ import csv +import json from datetime import datetime, timezone from pathlib import Path from typing import Any, AsyncGenerator, Generator import pytest import pytest_asyncio +from aiohttp import web from aiohttp.test_utils import TestClient, TestServer from aioresponses import aioresponses @@ -19,24 +21,15 @@ DELETED_RESOURCE_ID = "deadbeef-dead-beef-dead-beefdeadbeef" NULL_VALUES_RESOURCE_ID = "dddddddd-1111-eeee-1212-ffffffffffff" +PARQUET_DIR = Path(__file__).parent.parent / "db" / "parquet" -@pytest.fixture -def rmock(): - # passthrough for local requests (aiohttp TestServer) - with aioresponses(passthrough=["http://127.0.0.1"]) as m: - yield m - -@pytest_asyncio.fixture -async def client() -> AsyncGenerator[TestClient, Any]: - app = await app_factory() - async with TestClient(TestServer(app)) as client: - yield client - - -@pytest.fixture -def base_url() -> Generator[str, Any, Any]: - yield f"{config.SCHEME}://{config.SERVER_NAME}" +@pytest.fixture(autouse=True) +def parquet_base_url(): + """Point DuckDB at local fixture Parquet files instead of S3.""" + config.override(PARQUET_BASE_URL=str(PARQUET_DIR.resolve())) + yield + config.override(PARQUET_BASE_URL="") def timestamptz_to_utc_iso(date_str: str) -> str: @@ -75,3 +68,65 @@ def tables_index_rows() -> Generator[dict, Any, Any]: @pytest.fixture def exceptions_rows() -> Generator[dict, Any, Any]: yield csv_to_dict("exceptions") + + +@pytest.fixture(autouse=True) +def mock_tables_index(mocker, tables_index_rows): + """Serve `tables_index` metadata from CSV fixtures (no PostgREST required).""" + + async def _get_resource(session, resource_id: str, columns: list) -> dict: + if resource_id not in tables_index_rows: + raise web.HTTPNotFound() + row = tables_index_rows[resource_id] + if row.get("deleted_at") is not None: + deleted_at = row["deleted_at"] + dataset_id = row.get("dataset_id") + message = ( + f"Resource {resource_id} has been permanently deleted on {deleted_at} " + "by its producer." + ) + if dataset_id: + message += ( + f" You can find more information about this resource at " + f"https://www.data.gouv.fr/datasets/{dataset_id}" + ) + else: + message += " Contact the resource producer to get more information." + raise web.HTTPGone(text=message) + + requested = set(columns) + record: dict[str, Any] = { + "deleted_at": row["deleted_at"], + "dataset_id": row["dataset_id"], + } + if "created_at" in requested: + record["created_at"] = row["created_at"] + if "url" in requested: + record["url"] = row["url"] + if "parsing_table" in requested: + record["parsing_table"] = row["parsing_table"] + if any(c.startswith("profile") or "csv_detective" in c for c in requested): + record["profile"] = json.loads(row["csv_detective"]) + return record + + mocker.patch("api_tabular.tabular.utils.get_resource", side_effect=_get_resource) + mocker.patch("api_tabular.tabular.app.get_resource", side_effect=_get_resource) + + +@pytest.fixture +def rmock(): + # passthrough for local requests (aiohttp TestServer) + with aioresponses(passthrough=["http://127.0.0.1"]) as m: + yield m + + +@pytest_asyncio.fixture +async def client() -> AsyncGenerator[TestClient, Any]: + app = await app_factory() + async with TestClient(TestServer(app)) as client: + yield client + + +@pytest.fixture +def base_url() -> Generator[str, Any, Any]: + yield f"{config.SCHEME}://{config.SERVER_NAME}" diff --git a/tests/test_api.py b/tests/test_api.py index 703db8a..178d93d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -77,20 +77,12 @@ async def test_api_resource_meta_not_found(client): RESOURCE_ID, ], ) -async def test_api_resource_profile(client, tables_index_rows, exceptions_rows, _resource_id): - indexes = ( - list(json.loads(exceptions_rows[_resource_id]["table_indexes"]).keys()) - if _resource_id in exceptions_rows - else None - ) +async def test_api_resource_profile(client, tables_index_rows, _resource_id): res = await client.get(f"/api/resources/{_resource_id}/profile/") assert res.status == 200 body = await res.json() assert body["profile"] == json.loads(tables_index_rows[_resource_id]["csv_detective"]) - if indexes is None: - assert body["indexes"] is None - else: - assert sorted(body["indexes"]) == sorted(indexes) + assert "indexes" not in body async def test_api_resource_profile_not_found(client): @@ -285,75 +277,19 @@ async def test_api_with_unsupported_args(client): @pytest.mark.parametrize( - "params", - [ - (INDEXED_RESOURCE_ID, True), - (AGG_ALLOWED_INDEXED_RESOURCE_ID, False), - ], -) -async def test_api_exception_resource_indexes(client, tables_index_rows, exceptions_rows, params): - _resource_id, forbidden = params - detection = json.loads(tables_index_rows[_resource_id]["csv_detective"]) - indexes = list(json.loads(exceptions_rows[_resource_id]["table_indexes"]).keys()) - res = await client.get(f"/api/resources/{_resource_id}/profile/") - assert res.status == 200 - content = await res.json() - assert content["profile"] == detection - # sorted because it's made from a set so the order might not be preserved - assert sorted(indexes) == list(sorted(content["indexes"])) - - # checking that the resource is readable with no filter - res = await client.get(f"/api/resources/{_resource_id}/data/?page=1&page_size=1") - assert res.status == 200 - - # checking that the resource can be filtered on any columns - for col in detection["columns"].keys(): - if detection["columns"][col]["python_type"] == "json": - # can't handle json type for now - continue - res = await client.get( - f"/api/resources/{_resource_id}/data/?{col}__exact=1&page=1&page_size=1" - ) - assert res.status == 200 - - # checking that the resource cannot be aggregated on a non-indexed column - non_indexed_cols = [col for col in detection["columns"].keys() if col not in indexes] - for col in non_indexed_cols: - res = await client.get( - f"/api/resources/{_resource_id}/data/?{col}__groupby&page=1&page_size=1" - ) - assert res.status == 403 - - # checking whether aggregation is allowed on indexed columns - for idx in indexes: - res = await client.get( - f"/api/resources/{_resource_id}/data/?{idx}__groupby&page=1&page_size=1" - ) - assert res.status == 403 if forbidden else 200 - - -@pytest.mark.parametrize( - "params", + "_resource_id", [ - (RESOURCE_ID, True), - (AGG_ALLOWED_RESOURCE_ID, False), + INDEXED_RESOURCE_ID, + AGG_ALLOWED_INDEXED_RESOURCE_ID, + RESOURCE_ID, + AGG_ALLOWED_RESOURCE_ID, ], ) -async def test_api_exception_resource_no_indexes(client, tables_index_rows, params): - _resource_id, forbidden = params +async def test_api_resource_aggregations_allowed(client, tables_index_rows, _resource_id): detection = json.loads(tables_index_rows[_resource_id]["csv_detective"]) - # checking that we have an `indexes` key in the profile endpoint - res = await client.get(f"/api/resources/{_resource_id}/profile/") - assert res.status == 200 - content = await res.json() - assert content["profile"] == detection - assert content["indexes"] is None - - # checking that the resource is readable with no filter res = await client.get(f"/api/resources/{_resource_id}/data/?page=1&page_size=1") assert res.status == 200 - # checking that the resource can be filtered on all columns for col, results in detection["columns"].items(): if results["python_type"] == "json": continue @@ -361,16 +297,10 @@ async def test_api_exception_resource_no_indexes(client, tables_index_rows, para f"/api/resources/{_resource_id}/data/?{col}__exact=1&page=1&page_size=1" ) assert res.status == 200 - - # if aggregation is allowed: - # checking whether aggregation is allowed on all columns or none - for col, results in detection["columns"].items(): - if results["python_type"] == "json": - continue res = await client.get( f"/api/resources/{_resource_id}/data/?{col}__groupby&page=1&page_size=1" ) - assert res.status == 403 if forbidden else 200 + assert res.status == 200 @pytest.mark.parametrize( @@ -380,11 +310,23 @@ async def test_api_exception_resource_no_indexes(client, tables_index_rows, para (200, 200, ["status", "version", "uptime_since"]), ], ) -async def test_health(client, rmock, params): +async def test_health(client, mocker, params): postgrest_resp_code, api_expected_resp_code, expected_keys = params - rmock.head( - f"{config.PGREST_ENDPOINT}/migrations_csv", - status=postgrest_resp_code, + + class FakeResponse: + def __init__(self, ok: bool): + self.ok = ok + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + mocker.patch.object( + client.app["csession"], + "head", + return_value=FakeResponse(ok=postgrest_resp_code == 200), ) res = await client.get("/health/") assert res.status == api_expected_resp_code @@ -392,13 +334,6 @@ async def test_health(client, rmock, params): assert all(key in res_json for key in expected_keys) -async def test_aggregation_exceptions(client): - res = await client.get("/api/aggregation-exceptions/") - aggregations = await res.json() - assert not aggregations["allowed"] - assert aggregations["exceptions"] == config.ALLOW_AGGREGATION_EXCEPTIONS - - @pytest.mark.parametrize( "_resource_id,batch_size", [ diff --git a/tests/test_config.py b/tests/test_config.py index 8f00885..d2bd2ab 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,11 +40,12 @@ def test_custom_config_file_override(): def test_env_override(): os.environ["PGREST_ENDPOINT"] = "https://example.com" os.environ["PAGE_SIZE_MAX"] = "200" - os.environ["ALLOW_AGGREGATION"] = "true" - os.environ["ALLOW_AGGREGATION_EXCEPTIONS"] = "a,b,c" + os.environ["S3_ENDPOINT"] = "s3.example.com" + os.environ["S3_BUCKET"] = "my-bucket" config = Configurator() assert config.PGREST_ENDPOINT == "https://example.com" assert config.PAGE_SIZE_MAX == 200 - assert config.ALLOW_AGGREGATION - assert config.ALLOW_AGGREGATION_EXCEPTIONS == ["a", "b", "c"] + assert config.S3_ENDPOINT == "s3.example.com" + assert config.S3_BUCKET == "my-bucket" + assert config.parquet_url("abc") == "https://s3.example.com/my-bucket/parquet/abc.parquet" diff --git a/tests/test_duckdb_query.py b/tests/test_duckdb_query.py new file mode 100644 index 0000000..8f0cf69 --- /dev/null +++ b/tests/test_duckdb_query.py @@ -0,0 +1,51 @@ +from api_tabular.tabular.duckdb_query import build_duckdb_query + + +def test_duckdb_query_limit_and_default_order(): + q = build_duckdb_query([], page_size=12) + assert q.limit == 12 + assert q.offset == 0 + assert q.order_sql == 'ORDER BY "__id" ASC' + assert q.select_sql == "*" + assert not q.is_aggregation + + +def test_duckdb_query_filters(): + q = build_duckdb_query( + ["score__greater=0.9", "decompte__exact=13"], + page_size=20, + offset=20, + ) + assert 'WHERE "score" >= ? AND "decompte" = ?' == q.where_sql + assert q.where_params == ["0.9", "13"] + assert q.limit == 20 + assert q.offset == 20 + + +def test_duckdb_query_contains_and_or(): + q = build_duckdb_query( + ["or=(id__exact.abc,score__less.0.1)"], + page_size=50, + ) + assert q.where_sql.startswith("WHERE (") + assert " OR " in q.where_sql + assert q.where_params == ["abc", "0.1"] + + +def test_duckdb_query_aggregation(): + q = build_duckdb_query( + ["decompte__groupby", "score__avg"], + page_size=50, + ) + assert q.is_aggregation + assert '"decompte"' in q.select_sql + assert 'AVG("score") AS "score__avg"' in q.select_sql + assert q.group_by_sql == 'GROUP BY "decompte"' + assert q.order_sql == "" + + +def test_duckdb_query_columns_conflict_with_agg(): + import pytest + + with pytest.raises(ValueError): + build_duckdb_query(["columns=a,b", "a__groupby"]) diff --git a/tests/test_query.py b/tests/test_query.py index 69f75ff..ed0bac2 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -214,17 +214,11 @@ def test_query_build_multiple_with_unknown(): ], ) def test_query_aggregators(allow_aggregation, mocker): - if allow_aggregation: - mocker.patch("api_tabular.config.ALLOW_AGGREGATION_EXCEPTIONS", [RESOURCE_ID]) query_str = [ "column_name__groupby", "column_name__min", "column_name__avg", ] - if not allow_aggregation: - with pytest.raises(PermissionError): - build_sql_query_string(query_str, resource_id=RESOURCE_ID, page_size=50) - return results = build_sql_query_string(query_str, resource_id=RESOURCE_ID, page_size=50).split("&") assert "limit=50" in results assert "order=__id.asc" not in results # no sort if aggregators diff --git a/tests/test_swagger.py b/tests/test_swagger.py index 68f8e08..99eb213 100644 --- a/tests/test_swagger.py +++ b/tests/test_swagger.py @@ -30,14 +30,17 @@ async def test_swagger_endpoint(client, _resource_id): @pytest.mark.parametrize( - "params", + "_resource_id", [ - (RESOURCE_ID, False), - (AGG_ALLOWED_RESOURCE_ID, True), + RESOURCE_ID, + AGG_ALLOWED_RESOURCE_ID, + INDEXED_RESOURCE_ID, + AGG_ALLOWED_INDEXED_RESOURCE_ID, ], ) -async def test_swagger_no_indexes(client, tables_index_rows, params): - _resource_id, allow_aggregation = params +async def test_swagger_includes_all_columns_and_aggregators( + client, tables_index_rows, _resource_id +): detection = json.loads(tables_index_rows[_resource_id]["csv_detective"]) columns = {c: v["python_type"] for c, v in detection["columns"].items()} res = await client.get(f"/api/resources/{_resource_id}/swagger/") @@ -58,46 +61,9 @@ async def test_swagger_no_indexes(client, tables_index_rows, params): else [p] ) for _p in _params: - if allow_aggregation: - if f"{c}__{_p}" not in params: - missing.append(f"{c}__{_p} is missing in {output} output") - elif OPERATORS_DESCRIPTIONS.get(_p, {}).get("is_aggregator"): - assert params[f"{c}__{_p}"].get("allowEmptyValue") - else: - if ( - not OPERATORS_DESCRIPTIONS.get(_p, {}).get("is_aggregator") - and f"{c}__{_p}" not in params # filters are in - ): - missing.append(f"{c}__{_p} is missing in {output} output") - assert params[f"{c}__{_p}"].get("allowEmptyValue") is None - if ( - OPERATORS_DESCRIPTIONS.get(_p, {}).get("is_aggregator") - and f"{c}__{_p}" in params # aggregators are out - ): - missing.append(f"{c}__{_p} is in {output} output but should not") + if f"{c}__{_p}" not in params: + missing.append(f"{c}__{_p} is missing in {output} output") + elif OPERATORS_DESCRIPTIONS.get(_p, {}).get("is_aggregator"): + assert params[f"{c}__{_p}"].get("allowEmptyValue") if missing: raise ValueError("\n" + ";\n".join(missing)) - - -@pytest.mark.parametrize( - "_resource_id", - [ - AGG_ALLOWED_INDEXED_RESOURCE_ID, - INDEXED_RESOURCE_ID, - ], -) -async def test_swagger_with_indexes(client, tables_index_rows, exceptions_rows, _resource_id): - detection = json.loads(tables_index_rows[_resource_id]["csv_detective"]) - indexes = list(json.loads(exceptions_rows[_resource_id]["table_indexes"]).keys()) - non_indexed_cols = [col for col in detection["columns"].keys() if col not in indexes] - res = await client.get(f"/api/resources/{_resource_id}/swagger/") - swagger = await res.text() - swagger_dict = yaml.safe_load(swagger) - - for output in ["json", "csv"]: - params = swagger_dict["paths"][ - f"/api/resources/{_resource_id}/data/{'' if output == 'json' else 'csv/'}" - ]["parameters"] - params = set([p["name"].split("__")[0] for p in params if "__" in p["name"]]) - assert all(c in params for c in indexes) - assert not any(c in params for c in non_indexed_cols) diff --git a/uv.lock b/uv.lock index 35895ad..78f9e6f 100644 --- a/uv.lock +++ b/uv.lock @@ -263,6 +263,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/ae/afb1487556e2dc827a17097aac8158a25b433a345386f0e249f6d2694ccb/devtools-0.12.2-py3-none-any.whl", hash = "sha256:c366e3de1df4cdd635f1ad8cbcd3af01a384d7abda71900e68d43b04eb6aaca7", size = 19411, upload-time = "2023-09-03T16:56:59.049Z" }, ] +[[package]] +name = "duckdb" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/29/9bad86ed7aa812d8c822a27c15c355b6d5423b991feeec86ed18027b6daa/duckdb-1.5.4.tar.gz", hash = "sha256:f9e32f1cdd106793d79d190186bed9e75289d51e68bd9174e47c04bffedeab6f", size = 18046634, upload-time = "2026-06-17T10:48:52.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/bb/7921dabd50daef3969f14cd8a5a14c24eee337db7914a462f2defa8add92/duckdb-1.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3fb41d9cfccb7e44511eeeed263ae98143ca63bdb1ef84631ba637c314efa1b5", size = 32663142, upload-time = "2026-06-17T10:47:45.471Z" }, + { url = "https://files.pythonhosted.org/packages/a6/83/2137765eaba6a9aefe3bb9848ddaac7407fe3ba19b292f98b31f3b7ab27f/duckdb-1.5.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8ba7b666bc9c78d6a930ee9f469024149f0c6a23fb7d2c3418aad6774339bec0", size = 17321485, upload-time = "2026-06-17T10:47:47.778Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b2/a02c1ee43fd7e8cf1fc2e3d377f3dcf9d4a3e58a4549557516e1866ff0da/duckdb-1.5.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9d9e6817fcbc09d2605a2c8c041ac7824d738d917c35a4d427e977647e1d7944", size = 15470820, upload-time = "2026-06-17T10:47:49.977Z" }, + { url = "https://files.pythonhosted.org/packages/d8/48/a243d30223b024bc6057abe472b002cff01e97efefb4d2f0b0dcc5aece0b/duckdb-1.5.4-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02dd9f9a6124069213f13e3a474c208028c472fe1acdae12b38761f954fe4fc6", size = 19341849, upload-time = "2026-06-17T10:47:52.205Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/a5d48de4771e2403a8ef26a20dc7457b1c8f7e398ff0caf9c0cad8805f89/duckdb-1.5.4-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccc7f2694d02b4763fee61021d45e12f7bc5743993686563957df0cef799fbae", size = 21451698, upload-time = "2026-06-17T10:47:54.653Z" }, + { url = "https://files.pythonhosted.org/packages/79/b8/8244d7741b4afae67775cf0cb0d4eb9e923a83110907e4801e17fa078480/duckdb-1.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:4c430e788d99b50854209bf2833ba36a45df75e57f86efb477046cd408bbd077", size = 13132643, upload-time = "2026-06-17T10:47:56.75Z" }, + { url = "https://files.pythonhosted.org/packages/e4/57/8169822a37f6dd7d561c567f9007e3cf04bf97bccb619afe90db849c0962/duckdb-1.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:e2dc8340cfb6006025a798c50f40126d6e945a1d2487be94667bb4166556ce7b", size = 13986386, upload-time = "2026-06-17T10:47:59.345Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f2/e2f4b477ae3a3b40e8b5f429832e48edb62ed9da99807cc4902e157e5646/duckdb-1.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:291a9e7502551170af989ff63139a7a49e99d68edbc5ef5017ac27541fe54c65", size = 32708876, upload-time = "2026-06-17T10:48:01.527Z" }, + { url = "https://files.pythonhosted.org/packages/2e/2b/b698d82a5e1e30b6a05748d72045f672994c6b22f4f0f8423523608b991f/duckdb-1.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83e8c089bbb756ca4471d8b05943b80a106058697cf00615e70423106bb783bc", size = 17346125, upload-time = "2026-06-17T10:48:04.035Z" }, + { url = "https://files.pythonhosted.org/packages/71/75/37e13f39268eaf34864453b3a039c4a1ff0b088d3eae45a4289b41c98c1b/duckdb-1.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ff96d2a342b200e1ec6f1f19986c77f4ac16a49b6112f71c5b763989203a9d60", size = 15488133, upload-time = "2026-06-17T10:48:06.312Z" }, + { url = "https://files.pythonhosted.org/packages/cc/59/2d082af578f689231798245b54562c61416e49049b0bda81a06c56a4b53e/duckdb-1.5.4-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f935ef210ab00bc94bb1e3052697adaa36bb0ce7bdfeda8b0f34e2ff1643870", size = 19367895, upload-time = "2026-06-17T10:48:08.59Z" }, + { url = "https://files.pythonhosted.org/packages/52/2b/55c34d2863a76ca824ef8274691e84240b4ff1acde3d231709e82557c240/duckdb-1.5.4-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0cda263d8c20addb8d4f95464787cbe0af1144f7ab7e21db3709fb826ee01725", size = 21486499, upload-time = "2026-06-17T10:48:10.963Z" }, + { url = "https://files.pythonhosted.org/packages/cf/30/ade5952b8182fac86fab43b95ebe3836e66381d0ad64eb1e54bd8207c988/duckdb-1.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:266c7c909558ce7377f57d082cee408aadebdd9111be017558ca54e44a031037", size = 13147934, upload-time = "2026-06-17T10:48:13.061Z" }, + { url = "https://files.pythonhosted.org/packages/f5/00/278f0f70e25b9911afe2fd227b9460f2e6d76177f0dcc03f7f1454afefa5/duckdb-1.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:f14e79a006341f29ee5a2692a24dac5114e77533d579c57ec39124adf0135033", size = 13965235, upload-time = "2026-06-17T10:48:15.782Z" }, + { url = "https://files.pythonhosted.org/packages/da/69/3fcb34e523a9bad1f0557a6c7691a71ba66c43a05e5be9ee96a9a841ed65/duckdb-1.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:42a612e67d64450b446eb69695290d460713eef46e0f64467ab9dfe96264ee05", size = 32708366, upload-time = "2026-06-17T10:48:18.084Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5f/bff5054c2c1d65decab36aa6296621e51a2a575a9f250db7ab9b83a325d6/duckdb-1.5.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3fb6f07d54ecf4d0d3c5179a2361fdddfafa14de4fc42696de4632479b703421", size = 17345735, upload-time = "2026-06-17T10:48:20.67Z" }, + { url = "https://files.pythonhosted.org/packages/93/12/d1b2b344e9699246aada6f9de5156e708fb476e2780e5bff9b5d95fe11d9/duckdb-1.5.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f32ad7e0286c1c29ab6c73b29118c86101f8eee46aae54f54d0b50916f542f6", size = 15488568, upload-time = "2026-06-17T10:48:23.038Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d1/ac56c6096e3e95da60b2c5dd5a0f0eb5540a80622e2e4f8faab893ec4e96/duckdb-1.5.4-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:698ec90bd5d5538bd5f6d212a4b61af443d240703cf45f134738535026556ea5", size = 19368184, upload-time = "2026-06-17T10:48:25.601Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/2ae4c3e157a19d9b4ac1f09a5dea6f93012334cc2db09f1e0c71eb99693d/duckdb-1.5.4-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136cea7f886b78caf4035485b4b1e766e8b309e999f9e83a966f81ebb8122844", size = 21486523, upload-time = "2026-06-17T10:48:27.817Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/c3d8d21e0d0db8faa81eeeb3a55b9932f5a0a16466cb968dc713a653d701/duckdb-1.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:bd6777e8ddd74fb603a6d09766bfcff28638189f8aaa61fc0dffd9e9a4baa8e5", size = 13147807, upload-time = "2026-06-17T10:48:30.017Z" }, + { url = "https://files.pythonhosted.org/packages/44/48/ddf8d3740e3d28582944f70d84e720b5dc28c10ec22b668a0e0bd965f2f2/duckdb-1.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:73f4878a3012283024a64a1909e440aac12091ef336f671fc142f7e87449ce0c", size = 13965189, upload-time = "2026-06-17T10:48:32.251Z" }, + { url = "https://files.pythonhosted.org/packages/62/01/67ac4cbc8e552a1e14c029b5c443d828e68f94d5d913c574f577e1db277e/duckdb-1.5.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:4647968629d0677bbcc2416c7aeda8685eb84e4ca15a6dbd4f82a66cfc91a532", size = 32714364, upload-time = "2026-06-17T10:48:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/eb44d983fa56b175f971eea251bde284a36d26cbb93fcb68035061f54078/duckdb-1.5.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e8fcef301cf68d3951ea1eb8ac4d76cea0a6f6a08f4c78fe4026fc96d217bebc", size = 17349820, upload-time = "2026-06-17T10:48:37.126Z" }, + { url = "https://files.pythonhosted.org/packages/10/b2/b9dc7624b105d414585b8530451c1162c0b4750c0be9be2e497bb47a8a9b/duckdb-1.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f6f39cd0dc6948dee17fd130aec55114f97a8ef6e1db519b9774087962bc5c8c", size = 15498160, upload-time = "2026-06-17T10:48:40.032Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/61356444f6a8c62dec3c3d129abfc53f428de1d484093d1bb381db441231/duckdb-1.5.4-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:262f068158beb5943f2c618f4e54b46db8306b959f90dce956f90a89f613673d", size = 19374183, upload-time = "2026-06-17T10:48:42.698Z" }, + { url = "https://files.pythonhosted.org/packages/b0/f4/d5d633dd7c5138d8f7c434e6ac2553c831b7fb658494efa8d0bc73df8623/duckdb-1.5.4-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d2307a76d199077b0055b354e90e857479461a0d875437535dd4833172c8b6d", size = 21487202, upload-time = "2026-06-17T10:48:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/c0/26/5be13bbd5c3421dccfc1ad4ca9da4b97c5a3ddd73f66542092f3167ec52c/duckdb-1.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:6dcbb81a1276bc48deb4d562bce4f8895e4fc6348750a096e30052345c6d6552", size = 13666989, upload-time = "2026-06-17T10:48:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/dc/82/4d52f3f9f9703a226b26b80bdae3f6905aeefe5221bf1815fc93ff02ca25/duckdb-1.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:0f8722346024e5d9f02b58bf7b0491a629f97fdc8a04a10e432940f471ee387a", size = 14449863, upload-time = "2026-06-17T10:48:50.18Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -917,6 +953,7 @@ source = { editable = "." } dependencies = [ { name = "aiohttp" }, { name = "aiohttp-swagger" }, + { name = "duckdb" }, { name = "gunicorn" }, { name = "sentry-sdk" }, ] @@ -936,6 +973,7 @@ dev = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.14.1,<4.0.0" }, { name = "aiohttp-swagger", specifier = ">=1.0.16,<2.0.0" }, + { name = "duckdb", specifier = ">=1.5.4" }, { name = "gunicorn", specifier = ">=23.0.0,<24.0.0" }, { name = "sentry-sdk", specifier = ">=2.49.0,<3.0.0" }, ] From 94be991d25fa481f56229738872f545d5aae3018 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 17 Jul 2026 19:21:24 +0200 Subject: [PATCH 2/2] fix: fix linting issues --- api_tabular/tabular/duckdb_exec.py | 10 ++- api_tabular/tabular/duckdb_query.py | 98 +++++++++++++++++------------ scripts/generate_test_parquets.py | 3 +- 3 files changed, 67 insertions(+), 44 deletions(-) diff --git a/api_tabular/tabular/duckdb_exec.py b/api_tabular/tabular/duckdb_exec.py index 5ef02a5..9c82764 100644 --- a/api_tabular/tabular/duckdb_exec.py +++ b/api_tabular/tabular/duckdb_exec.py @@ -112,7 +112,9 @@ def _execute_sync( if not query.is_aggregation: count_sql = _assemble_sql(source_sql, query, for_count=True) count_params = source_params + query.where_params - total = int(con.execute(count_sql, count_params).fetchone()[0]) + total = int( + con.execute(count_sql, count_params).fetchone()[0] + ) # ty: ignore[not-subscriptable] return rows, total finally: @@ -132,7 +134,11 @@ def _count_sync(parquet_path: str, query: DuckDBQuery) -> int: source_sql, source_params = _parquet_source_sql(parquet_path, columns) count_sql = _assemble_sql(source_sql, query, for_count=True) - return int(con.execute(count_sql, source_params + query.where_params).fetchone()[0]) + return int( + con.execute(count_sql, source_params + query.where_params).fetchone()[ + 0 + ] # ty: ignore[not-subscriptable] + ) except duckdb.Error as e: raise QueryException(400, None, "Database error", str(e)) from e finally: diff --git a/api_tabular/tabular/duckdb_query.py b/api_tabular/tabular/duckdb_query.py index 0d1679b..3d4bb44 100644 --- a/api_tabular/tabular/duckdb_query.py +++ b/api_tabular/tabular/duckdb_query.py @@ -1,6 +1,7 @@ import re from collections import defaultdict from dataclasses import dataclass, field +from typing import Any @dataclass @@ -17,6 +18,14 @@ class DuckDBQuery: offset: int = 0 +@dataclass +class FilterAction: + kind: str # where | order | select + sql: str = "" + params: list[Any] = field(default_factory=list) + columns: list[str] = field(default_factory=list) + + def quote_ident(name: str) -> str: return '"' + name.replace('"', '""') + '"' @@ -33,7 +42,7 @@ def build_duckdb_query( offset: int = 0, ) -> DuckDBQuery: where_parts: list[str] = [] - where_params: list = [] + where_params: list[Any] = [] order_sql = "" select_columns: list[str] | None = None aggregators: dict[str, list[str]] = defaultdict(list) @@ -47,26 +56,25 @@ def build_duckdb_query( continue _split = arg.split("=") if len(_split) == 2: - result = add_filter(*_split) - if result is None: + action = add_filter(*_split) + if action is None: continue - kind, payload = result - if kind == "where": - where_parts.append(payload[0]) - where_params.extend(payload[1]) - elif kind == "order": - order_sql = payload + if action.kind == "where": + where_parts.append(action.sql) + where_params.extend(action.params) + elif action.kind == "order": + order_sql = action.sql has_sort = True - elif kind == "select": - select_columns = payload + elif action.kind == "select": + select_columns = action.columns elif len(_split) == 1: if "__" not in _split[0]: raise ValueError(f"argument '{arg}' could not be parsed") if _split[0].split("__")[-1] in ["isnull", "isnotnull"]: - result = add_filter(_split[0], "") - assert result is not None and result[0] == "where" - where_parts.append(result[1][0]) - where_params.extend(result[1][1]) + action = add_filter(_split[0], "") + assert action is not None and action.kind == "where" + where_parts.append(action.sql) + where_params.extend(action.params) else: column, operator = add_aggregator(_split[0]) aggregators[operator].append(column) @@ -123,7 +131,7 @@ def add_filter( value: str, *, in_operator: bool = False, -) -> tuple[str, tuple] | None: +) -> FilterAction | None: if argument in ["page", "page_size"]: if in_operator: raise ValueError(f"Argument `{argument}` can't be set within an operator") @@ -133,7 +141,7 @@ def add_filter( raise ValueError(f"Argument `{argument}` can't be set within an operator") if '"' in value: raise ValueError('Forbidden character `"` in a column name') - return ("select", value.split(",")) + return FilterAction(kind="select", columns=value.split(",")) if "__" not in argument: raise ValueError(f"argument '{argument}={value}' could not be parsed") @@ -146,38 +154,47 @@ def add_filter( direction = value.upper() if direction not in ("ASC", "DESC"): raise ValueError(f"argument '{argument}={value}' could not be parsed") - return ("order", f"ORDER BY {ident} {direction}") + return FilterAction(kind="order", sql=f"ORDER BY {ident} {direction}") if normalized_comparator == "exact": - return ("where", (f"{ident} = ?", [coerce_value(value)])) + return FilterAction(kind="where", sql=f"{ident} = ?", params=[coerce_value(value)]) if normalized_comparator == "differs": - return ("where", (f"{ident} IS DISTINCT FROM ?", [coerce_value(value)])) + return FilterAction( + kind="where", sql=f"{ident} IS DISTINCT FROM ?", params=[coerce_value(value)] + ) if normalized_comparator == "isnull": - return ("where", (f"{ident} IS NULL", [])) + return FilterAction(kind="where", sql=f"{ident} IS NULL") if normalized_comparator == "isnotnull": - return ("where", (f"{ident} IS NOT NULL", [])) + return FilterAction(kind="where", sql=f"{ident} IS NOT NULL") if normalized_comparator == "contains": - return ("where", (f"{ident} ILIKE '%' || ? || '%'", [value])) + return FilterAction(kind="where", sql=f"{ident} ILIKE '%' || ? || '%'", params=[value]) if normalized_comparator == "notcontains": - return ("where", (f"NOT ({ident} ILIKE '%' || ? || '%')", [value])) + return FilterAction( + kind="where", sql=f"NOT ({ident} ILIKE '%' || ? || '%')", params=[value] + ) if normalized_comparator == "in": values = value.split(",") placeholders = ", ".join("?" for _ in values) - return ("where", (f"{ident} IN ({placeholders})", [coerce_value(v) for v in values])) + return FilterAction( + kind="where", + sql=f"{ident} IN ({placeholders})", + params=[coerce_value(v) for v in values], + ) if normalized_comparator == "notin": values = value.split(",") placeholders = ", ".join("?" for _ in values) - return ( - "where", - (f"{ident} NOT IN ({placeholders})", [coerce_value(v) for v in values]), + return FilterAction( + kind="where", + sql=f"{ident} NOT IN ({placeholders})", + params=[coerce_value(v) for v in values], ) if normalized_comparator == "less": - return ("where", (f"{ident} <= ?", [coerce_value(value)])) + return FilterAction(kind="where", sql=f"{ident} <= ?", params=[coerce_value(value)]) if normalized_comparator == "greater": - return ("where", (f"{ident} >= ?", [coerce_value(value)])) + return FilterAction(kind="where", sql=f"{ident} >= ?", params=[coerce_value(value)]) if normalized_comparator == "strictly_less": - return ("where", (f"{ident} < ?", [coerce_value(value)])) + return FilterAction(kind="where", sql=f"{ident} < ?", params=[coerce_value(value)]) if normalized_comparator == "strictly_greater": - return ("where", (f"{ident} > ?", [coerce_value(value)])) + return FilterAction(kind="where", sql=f"{ident} > ?", params=[coerce_value(value)]) raise ValueError(f"argument '{argument}={value}' could not be parsed") @@ -259,18 +276,17 @@ def parse_operator(query: str, operator: str, top_level: bool = False) -> tuple[ params_sql.append(clause) params_values.extend(values) elif param.endswith(("__isnull", "__isnotnull")): - result = add_filter(param.replace('"', ""), "", in_operator=True) - assert result is not None and result[0] == "where" - params_sql.append(result[1][0]) - params_values.extend(result[1][1]) + action = add_filter(param.replace('"', ""), "", in_operator=True) + assert action is not None and action.kind == "where" + params_sql.append(action.sql) + params_values.extend(action.params) else: argument, value = find_arg_val(param) - # Strip surrounding quotes from values like "12.4" if len(value) >= 2 and value.startswith('"') and value.endswith('"'): value = value[1:-1] - result = add_filter(argument, value, in_operator=True) - assert result is not None and result[0] == "where" - params_sql.append(result[1][0]) - params_values.extend(result[1][1]) + action = add_filter(argument, value, in_operator=True) + assert action is not None and action.kind == "where" + params_sql.append(action.sql) + params_values.extend(action.params) joiner = f" {operator.upper()} " return f"({joiner.join(params_sql)})", params_values diff --git a/scripts/generate_test_parquets.py b/scripts/generate_test_parquets.py index c55db9d..4e3273d 100644 --- a/scripts/generate_test_parquets.py +++ b/scripts/generate_test_parquets.py @@ -110,7 +110,8 @@ def main() -> None: f"CREATE OR REPLACE TABLE tmp_export AS SELECT {', '.join(select_parts)} FROM tmp_export" ) con.execute(f"COPY tmp_export TO '{path}' (FORMAT PARQUET, COMPRESSION ZSTD)") - n = con.execute("SELECT COUNT(*) FROM tmp_export").fetchone()[0] + count_row = con.execute("SELECT COUNT(*) FROM tmp_export").fetchone() + n = 0 if count_row is None else count_row[0] print(f"wrote {path.name} ({n} rows)")