fix(#167): restructure sortables response

Author: captaincoordinates

docs/index-config.md

@@ -18,6 +18,8 @@ Entries in `queryables` and `sortables` must have a corresponding entry in `inde
 
 Each queryable and sortable property must include a list of collections for which the property is queryable or sortable. The `*` wildcard value can be used to indicate all collections. It is **not** currently possible to wildcard partial collection IDs, such as `collection-*`.
 
+`storage_type` **must** reference a valid [DuckDB data type](https://duckdb.org/docs/stable/sql/data_types/overview.html).
+
 ### Queryables
 
 Queryables require a `json_schema` property containing a schema that could be used to validate values of this property. This JSON schema is not used directly by the API but is provided to API clients via the `/queryables` endpoints such that a client can validate any value it intends to send as query value for this property.

packages/stac-index/src/stac_index/indexer/creator/configurer.py

@@ -10,7 +10,7 @@ def add_items_columns(config: IndexConfig, connection: DuckDBPyConnection) -> No
         connection.execute(
             f"""
             ALTER TABLE items
-            ADD COLUMN {indexable.table_column_name} {indexable.storage_type.value}
+            ADD COLUMN {indexable.table_column_name} {indexable.storage_type}
         """
         )
 
@@ -57,14 +57,15 @@ def _configure_sortables(config: IndexConfig, connection: DuckDBPyConnection) ->
             sortables_collections.append((collection_id, name))
         connection.execute(
             """
-                INSERT INTO sortables (name, description, json_path, items_column)
-                    VALUES (?, ?, ?, ?)
+                INSERT INTO sortables (name, description, json_path, items_column, json_type)
+                    VALUES (?, ?, ?, ?, ?)
             """,
             [
                 name,
                 indexable.description,
                 indexable.json_path,
                 indexable.table_column_name,
+                indexable.json_type,
             ],
         )
     for collection_id, name in sortables_collections:

packages/stac-index/src/stac_index/indexer/creator/sql/01-tables/111-sortables.sql

@@ -3,6 +3,7 @@ CREATE TABLE sortables (
     description VARCHAR NOT NULL,
     json_path VARCHAR NOT NULL,
     items_column VARCHAR,
+    json_type VARCHAR NOT NULL,
     UNIQUE(json_path),
     UNIQUE(items_column),
 );

packages/stac-index/src/stac_index/indexer/creator/sql/11-views/002-sortables_by_collection.sql

@@ -4,6 +4,7 @@ CREATE VIEW sortables_by_collection AS
          , s.description
          , s.json_path
          , COALESCE(s.items_column, s.name) AS items_column
+         , s.json_type
       FROM sortables s
       JOIN sortables_collections sc ON s.name == sc.name
      UNION
@@ -12,6 +13,7 @@ CREATE VIEW sortables_by_collection AS
          , s.description
          , s.json_path
          , COALESCE(s.items_column, s.name) AS items_column
+         , s.json_type
       FROM sortables s
       JOIN (
            SELECT sac.name

packages/stac-index/src/stac_index/indexer/creator/sql/21-inserts/101-sortables.sql

@@ -1,7 +1,7 @@
-INSERT INTO sortables (name, description, json_path, items_column) VALUES
-    ('id', 'Item ID', 'id', NULL),
-    ('collection', 'Collection ID', 'collection', 'collection_id'),
-    ('datetime', 'Datetime, NULL if datetime is a range', 'datetime', NULL),
-    ('start_datetime', 'Start datetime if datetime is a range, NULL if not', 'start_datetime', NULL),
-    ('end_datetime', 'End datetime if datetime is a range, NULL if not', 'end_datetime', NULL),
+INSERT INTO sortables (name, description, json_path, items_column, json_type) VALUES
+    ('id', 'Item ID', 'id', NULL, 'string'),
+    ('collection', 'Collection ID', 'collection', 'collection_id', 'string'),
+    ('datetime', 'Datetime, NULL if datetime is a range', 'datetime', NULL, 'string'),
+    ('start_datetime', 'Start datetime if datetime is a range, NULL if not', 'start_datetime', NULL, 'string'),
+    ('end_datetime', 'End datetime if datetime is a range, NULL if not', 'end_datetime', NULL, 'string'),
 ;

packages/stac-index/src/stac_index/indexer/types/index_config.py

@@ -1,6 +1,5 @@
-from enum import Enum
-from re import sub
-from typing import Any, Dict, Final, List, Optional
+from re import IGNORECASE, match, sub
+from typing import Any, Dict, Final, List, Optional, Self
 
 from pydantic import BaseModel
 
@@ -11,20 +10,36 @@ class StorageTypeProperties(BaseModel):
     pass
 
 
-class StorageType(str, Enum):
-    DOUBLE = "DOUBLE"
-
-
 class Indexable(BaseModel):
     json_path: str
     description: str
-    storage_type: StorageType
+    # storage_type is required to match the name of a valid DuckDB type as per https://duckdb.org/docs/stable/sql/data_types/overview.html.
+    # This could be an enum for more strict control, but this enum would require ongoing maintenance to match DuckDB's types.
+    # If a caller provided an invalid enum value it would result in a runtime failure from this class, which is
+    # functionally no different to a runtime failure when DuckDB attempts to add a table column of this type.
+    # Don't attempt to validate that storage_type is a valid DuckDB type and provide documentation around this.
+    storage_type: str
     storage_type_properties: Optional[StorageTypeProperties] = None
 
     @property
-    def table_column_name(self) -> str:
+    def table_column_name(self: Self) -> str:
         return "i_{}".format(sub("[^A-Za-z0-9]", "_", self.json_path))
 
+    @property
+    def json_type(self: Self) -> str:
+        if self.storage_type in (
+            "JSON",
+            "UUID",
+            "VARCHAR",
+        ):
+            return "string"
+        elif match("^(DATE|TIME)", self.storage_type, flags=IGNORECASE):
+            return "string"
+        elif self.storage_type == "BOOLEAN":
+            return "boolean"
+        else:
+            return "number"
+
 
 class Queryable(BaseModel):
     json_schema: Dict[str, Any]

src/stac_fastapi/indexed/sortables/models.py

@@ -1,13 +1,22 @@
-from typing import Final, List
+from typing import Any, Callable, Final, Self, cast
 
-from pydantic import BaseModel
+from pydantic import BaseModel, model_serializer
 
 
 class SortableField(BaseModel):
-    title: str
-    description: str
+    type: str
 
 
 class SortablesResponse(BaseModel):
-    title: Final[str] = "STAC Sortables"
-    fields: List[SortableField]
+    title: Final[str] = "Sortables"
+    properties: dict[str, SortableField]
+
+    @model_serializer(mode="wrap")
+    def serialize_model(self: Self, serializer: Callable) -> dict[str, Any]:
+        return {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": "https://example.com/sortables",
+            "title": "Sortables",
+            "type": "object",
+            **cast(dict[str, Any], serializer(self)),
+        }

src/stac_fastapi/indexed/sortables/routes.py

@@ -18,14 +18,11 @@ def add_routes(app: FastAPI) -> None:
     )
     async def get_all_sortables() -> SortablesResponse:
         return SortablesResponse(
-            fields=[
-                SortableField(
-                    title=config.name,
-                    description=config.description,
-                )
+            properties={
+                config.name: SortableField(type=config.type)
                 for config in await get_sortable_configs()
                 if config.collection_id == collection_wildcard
-            ]
+            }
         )
 
     @app.get(
@@ -36,12 +33,9 @@ async def get_all_sortables() -> SortablesResponse:
     )
     async def get_collection_sortables(collection_id: str) -> SortablesResponse:
         return SortablesResponse(
-            fields=[
-                SortableField(
-                    title=config.name,
-                    description=config.description,
-                )
+            properties={
+                config.name: SortableField(type=config.type)
                 for config in await get_sortable_configs()
                 if config.collection_id in [collection_id, collection_wildcard]
-            ]
+            }
         )

src/stac_fastapi/indexed/sortables/sortable_config.py

@@ -9,9 +9,10 @@
 _logger: Final[Logger] = getLogger(__name__)
 
 
-@dataclass
+@dataclass(kw_only=True)
 class SortableConfig:
     name: str
+    type: str
     collection_id: str
     description: str
     items_column: str
@@ -31,13 +32,15 @@ async def _get_sortable_configs(_: str) -> List[SortableConfig]:
             collection_id=row[1],
             description=row[2],
             items_column=row[3],
+            type=row[4],
         )
         for row in await fetchall(
             f"""
         SELECT name
              , collection_id
              , description
              , items_column
+             , json_type
           FROM {format_query_object_name('sortables_by_collection')}
     """
         )

tests/with_environment/integration_tests/test_get_search.py

@@ -222,8 +222,10 @@ def test_get_search_token_immutable() -> None:
 
 def test_get_search_alternate_order() -> None:
     sortable_field_names: List[str] = [
-        entry["title"]
-        for entry in requests.get(f"{api_base_url}/sortables").json()["fields"]
+        entry
+        for entry in requests.get(f"{api_base_url}/sortables")
+        .json()["properties"]
+        .keys()
     ]
     assert "datetime" in sortable_field_names
     assert "id" in sortable_field_names