From 5053fbd786880555a513e883cc22eb909af0e37f Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Wed, 26 Feb 2025 13:26:17 +0100 Subject: [PATCH] Make mulit-version queries valid. --- .../src/freva_rest/databrowser_api/core.py | 48 ++--- .../freva_rest/databrowser_api/endpoints.py | 17 +- .../databrowser_api/mock/files.json | 203 ++++++++++++------ .../src/freva_rest/databrowser_api/schema.py | 1 - 4 files changed, 161 insertions(+), 108 deletions(-) diff --git a/freva-rest/src/freva_rest/databrowser_api/core.py b/freva-rest/src/freva_rest/databrowser_api/core.py index e6f01ef4..68d1a9c5 100644 --- a/freva-rest/src/freva_rest/databrowser_api/core.py +++ b/freva-rest/src/freva_rest/databrowser_api/core.py @@ -311,9 +311,7 @@ def primary_keys(self) -> list[str]: if v == "primary" ] else: - _keys = [ - k for (k, v) in self._freva_facets.items() if v == "primary" - ] + _keys = [k for (k, v) in self._freva_facets.items() if v == "primary"] if self.flavour in ("cordex",): for key in self.cordex_keys: _keys.append(key) @@ -535,9 +533,7 @@ async def _session_post(self) -> AsyncIterator[Tuple[int, Dict[str, Any]]]: ) async with aiohttp.ClientSession(timeout=self.timeout) as session: try: - async with session.post( - self._post_url, json=self.payload - ) as res: + async with session.post(self._post_url, json=self.payload) as res: try: await self.check_for_status(res) logger.info( @@ -546,9 +542,7 @@ async def _session_post(self) -> AsyncIterator[Tuple[int, Dict[str, Any]]]: ) response_data = await res.json() except HTTPException: # pragma: no cover - logger.error( - "POST request failed: %s", await res.text() - ) + logger.error("POST request failed: %s", await res.text()) response_data = {} except Exception as error: logger.error("Connection to %s failed: %s", self.url, error) @@ -592,10 +586,13 @@ async def validate_parameters( Translate the output to the required DRS flavour. """ translator = Translator(flavour, translate) + valid_facets = translator.valid_facets + if multi_version: + valid_facets = translator.valid_facets + ["version"] for key in query: key = key.lower().replace("_not_", "") if ( - key not in translator.valid_facets + key not in valid_facets and key not in ("time_select",) + cls.uniq_keys ): raise HTTPException( @@ -781,12 +778,10 @@ async def _insert_to_mongo( ) if bulk_operations: try: - result = ( - await self._config.mongo_collection_userdata.bulk_write( - bulk_operations, - ordered=False, - bypass_document_validation=False, - ) + result = await self._config.mongo_collection_userdata.bulk_write( + bulk_operations, + ordered=False, + bypass_document_validation=False, ) successful_upsert = ( result.upserted_count @@ -827,9 +822,7 @@ async def _insert_to_mongo( nMatched, ) except Exception as error: - logger.exception( - "[MONGO] Could not insert metadata: %s", error - ) + logger.exception("[MONGO] Could not insert metadata: %s", error) @ensure_future async def store_results(self, num_results: int, status: int) -> None: @@ -919,7 +912,12 @@ async def extended_search( ------- int: status code of the apache solr query. """ - search_facets = [f for f in facets if f not in ("*", "all")] + search_facets = [f for f in facets if f not in ("*", "all")] or [ + f for f in self._config.solr_fields + ] + if self.multi_version: + search_facets.append("version") + self.query["facet"] = "true" self.query["rows"] = max_results self.query["facet.sort"] = "index" @@ -927,7 +925,7 @@ async def extended_search( self.query["facet.limit"] = "-1" self.query["wt"] = "json" self.query["facet.field"] = self.translator.translate_facets( - search_facets or self._config.solr_fields, backwards=True + search_facets, backwards=True ) self.query["fl"] = [self.uniq_key, "fs_type"] logger.info( @@ -977,9 +975,7 @@ async def init_stream(self) -> Tuple[int, int]: search_status, search = res return search_status, search.get("response", {}).get("numFound", 0) - def _join_facet_queries( - self, key: str, facets: List[str] - ) -> Tuple[str, str]: + def _join_facet_queries(self, key: str, facets: List[str]) -> Tuple[str, str]: """Create lucene search contain and NOT contain search queries""" negative, positive = [], [] @@ -1244,9 +1240,7 @@ async def _process_metadata( ) if not is_duplicate: new_querie.append(metadata) - return [ - dict(t) for t in {tuple(sorted(d.items())) for d in new_querie} - ] + return [dict(t) for t in {tuple(sorted(d.items())) for d in new_querie}] async def _purge_user_data( self, search_keys: Dict[str, Union[str, int]] diff --git a/freva-rest/src/freva_rest/databrowser_api/endpoints.py b/freva-rest/src/freva_rest/databrowser_api/endpoints.py index 3cec289e..5492f3bc 100644 --- a/freva-rest/src/freva_rest/databrowser_api/endpoints.py +++ b/freva-rest/src/freva_rest/databrowser_api/endpoints.py @@ -91,9 +91,7 @@ async def metadata_search( uniq_key: Literal["file", "uri"], multi_version: Annotated[bool, SolrSchema.params["multi_version"]] = False, translate: Annotated[bool, SolrSchema.params["translate"]] = True, - facets: Annotated[ - Union[List[str], None], SolrSchema.params["facets"] - ] = None, + facets: Annotated[Union[List[str], None], SolrSchema.params["facets"]] = None, request: Request = Required, ) -> JSONResponse: """Query the available metadata. @@ -232,9 +230,7 @@ async def extended_search( multi_version: Annotated[bool, SolrSchema.params["multi_version"]] = False, translate: Annotated[bool, SolrSchema.params["translate"]] = True, max_results: Annotated[int, SolrSchema.params["batch_size"]] = 150, - facets: Annotated[ - Union[List[str], None], SolrSchema.params["facets"] - ] = None, + facets: Annotated[Union[List[str], None], SolrSchema.params["facets"]] = None, request: Request = Required, ) -> JSONResponse: """This endpoint is used by the databrowser web ui client.""" @@ -276,8 +272,7 @@ async def load_data( title="Catalogue type", alias="catalogue-type", description=( - "Set the type of catalogue you want to create from this" - "query" + "Set the type of catalogue you want to create from this" "query" ), ), ] = None, @@ -345,10 +340,8 @@ async def post_user_data( solr_instance = Solr(server_config) try: try: - validated_user_metadata = ( - await solr_instance._validate_user_metadata( - request.user_metadata - ) + validated_user_metadata = await solr_instance._validate_user_metadata( + request.user_metadata ) except HTTPException as error: raise HTTPException( diff --git a/freva-rest/src/freva_rest/databrowser_api/mock/files.json b/freva-rest/src/freva_rest/databrowser_api/mock/files.json index bb965a8b..6a6acc38 100644 --- a/freva-rest/src/freva_rest/databrowser_api/mock/files.json +++ b/freva-rest/src/freva_rest/databrowser_api/mock/files.json @@ -52,7 +52,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20210618" }, { "project": [ @@ -107,7 +108,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v20210618" }, { "project": [ @@ -162,7 +164,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:41.959Z" + "creation_time": "2023-07-15T02:48:41.959Z", + "version": "v20210618" }, { "project": [ @@ -217,7 +220,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20210618" }, { "project": [ @@ -272,7 +276,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -327,7 +332,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:41.959Z" + "creation_time": "2023-07-15T02:48:41.959Z", + "version": "v20210618" }, { "project": [ @@ -382,7 +388,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -437,7 +444,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20210618" }, { "project": [ @@ -492,7 +500,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -547,7 +556,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -602,7 +612,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -657,7 +668,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v20210618" }, { "project": [ @@ -712,7 +724,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -767,7 +780,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.415Z" + "creation_time": "2023-07-15T02:48:42.415Z", + "version": "v20210618" }, { "project": [ @@ -822,7 +836,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -877,7 +892,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:41.959Z" + "creation_time": "2023-07-15T02:48:41.959Z", + "version": "v20210618" }, { "project": [ @@ -932,7 +948,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -987,7 +1004,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.415Z" + "creation_time": "2023-07-15T02:48:42.415Z", + "version": "v20210618" }, { "project": [ @@ -1042,7 +1060,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -1097,7 +1116,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v20210618" }, { "project": [ @@ -1152,7 +1172,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -1207,7 +1228,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.415Z" + "creation_time": "2023-07-15T02:48:42.415Z", + "version": "v20210618" }, { "project": [ @@ -1262,7 +1284,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -1317,7 +1340,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -1381,7 +1405,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v1" }, { "project": [ @@ -1445,7 +1470,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v1" }, { "project": [ @@ -1500,7 +1526,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20200101" }, { "project": [ @@ -1635,7 +1662,8 @@ "time_aggregation": [ "avg" ], - "creation_time": "2023-07-15T02:48:41.959Z" + "creation_time": "2023-07-15T02:48:41.959Z", + "version": "v20190815" }, { "project": [ @@ -1690,7 +1718,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20201108" }, { "project": [ @@ -1745,7 +1774,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20191108" }, { "project": [ @@ -1800,7 +1830,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v1" }, { "project": [ @@ -1864,7 +1895,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v1" }, { "project": [ @@ -1928,7 +1960,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:41.959Z" + "creation_time": "2023-07-15T02:48:41.959Z", + "version": "v1" }, { "project": [ @@ -1983,7 +2016,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v1" }, { "project": [ @@ -2038,7 +2072,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:41.959Z" + "creation_time": "2023-07-15T02:48:41.959Z", + "version": "v1" }, { "project": [ @@ -2093,7 +2128,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v1" }, { "project": [ @@ -2209,7 +2245,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -2264,7 +2301,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -2319,7 +2357,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20210618" }, { "project": [ @@ -2374,7 +2413,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -2429,7 +2469,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20210618" }, { "project": [ @@ -2484,7 +2525,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -2539,7 +2581,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -2594,7 +2637,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20210618" }, { "project": [ @@ -2649,7 +2693,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -2704,7 +2749,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -2759,7 +2805,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -2814,7 +2861,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -2869,7 +2917,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -2924,7 +2973,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v20210618" }, { "project": [ @@ -2979,7 +3029,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -3034,7 +3085,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -3089,7 +3141,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -3144,7 +3197,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v20210618" }, { "project": [ @@ -3199,7 +3253,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -3254,7 +3309,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v20210618" }, { "project": [ @@ -3309,7 +3365,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v20210618" }, { "project": [ @@ -3364,7 +3421,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20210618" }, { "project": [ @@ -3419,7 +3477,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20210618" }, { "project": [ @@ -3474,7 +3533,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.132Z" + "creation_time": "2023-07-15T02:48:42.132Z", + "version": "v20210618" }, { "project": [ @@ -3538,7 +3598,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v1" }, { "project": [ @@ -3602,7 +3663,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.737Z" + "creation_time": "2023-07-15T02:48:42.737Z", + "version": "v1" }, { "project": [ @@ -3666,7 +3728,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.469Z" + "creation_time": "2023-07-15T02:48:42.469Z", + "version": "v1" }, { "project": [ @@ -3721,7 +3784,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20200101" }, { "project": [ @@ -3776,7 +3840,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.114Z" + "creation_time": "2023-07-15T02:48:42.114Z", + "version": "v20190815" }, { "project": [ @@ -3831,7 +3896,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.155Z" + "creation_time": "2023-07-15T02:48:42.155Z", + "version": "v20201108" }, { "project": [ @@ -3886,7 +3952,8 @@ "level_type": [ "2d" ], - "creation_time": "2023-07-15T02:48:42.312Z" + "creation_time": "2023-07-15T02:48:42.312Z", + "version": "v20191108" }, { "project": [ @@ -3943,4 +4010,4 @@ ], "creation_time": "2023-07-15T02:48:42.312Z" } -] +] \ No newline at end of file diff --git a/freva-rest/src/freva_rest/databrowser_api/schema.py b/freva-rest/src/freva_rest/databrowser_api/schema.py index 45de1532..5fedaaae 100644 --- a/freva-rest/src/freva_rest/databrowser_api/schema.py +++ b/freva-rest/src/freva_rest/databrowser_api/schema.py @@ -74,7 +74,6 @@ def process_parameters( for key, param in cls.params.items(): _ = query.pop(key, [""]) _ = query.pop(param.alias, [""]) - return query