Merge pull request #218 from ecmwf-projects/COPDS-2104-None-default-for-rate-limits-file

Set rate limits file default to None

Author: mcucchi9

cads_processing_api_service/config.py

@@ -100,26 +100,26 @@ class RateLimitsConfig(pydantic.BaseModel):
     )
     process_execution: RateLimitsRouteConfig = pydantic.Field(
         default=RateLimitsRouteConfig(),
-        alias="processes/{process_id}/execution",
+        alias="/processes/{process_id}/execution",
         validate_default=True,
     )
     jobs: RateLimitsRouteConfig = pydantic.Field(
-        default=RateLimitsRouteConfig(), alias="jobs", validate_default=True
+        default=RateLimitsRouteConfig(), alias="/jobs", validate_default=True
     )
     job: RateLimitsRouteConfig = pydantic.Field(
-        default=RateLimitsRouteConfig(), alias="jobs/{job_id}", validate_default=True
+        default=RateLimitsRouteConfig(), alias="/jobs/{job_id}", validate_default=True
     )
     job_results: RateLimitsRouteConfig = pydantic.Field(
         default=RateLimitsRouteConfig(),
-        alias="jobs/{job_id}/results",
+        alias="/jobs/{job_id}/results",
         validate_default=True,
     )
 
     @pydantic.model_validator(mode="after")  # type: ignore
-    def populate_fields_with_default(self) -> None:
+    def populate_fields_with_default(self) -> pydantic.BaseModel:
         default = self.default
         if default is RateLimitsRouteConfig():
-            return
+            return self
         routes = self.model_fields
         for route in routes:
             if route == "default":
@@ -132,23 +132,24 @@ def populate_fields_with_default(self) -> None:
                     if not set_value:
                         default_value = getattr(getattr(default, method), origin)
                         setattr(getattr(route_config, method), origin, default_value)
-        return
+        return self
 
 
-def load_rate_limits(rate_limits_file: str) -> RateLimitsConfig:
+def load_rate_limits(rate_limits_file: str | None) -> RateLimitsConfig:
     rate_limits = RateLimitsConfig()
-    try:
-        with open(rate_limits_file, "r") as file:
-            loaded_rate_limits = yaml.safe_load(file)
-        rate_limits = RateLimitsConfig(**loaded_rate_limits)
-    except OSError:
-        logger.exception(
-            "Failed to read rate limits file", rate_limits_file=rate_limits_file
-        )
-    except pydantic.ValidationError:
-        logger.exception(
-            "Failed to validate rate limits file", rate_limits_file=rate_limits_file
-        )
+    if rate_limits_file is not None:
+        try:
+            with open(rate_limits_file, "r") as file:
+                loaded_rate_limits = yaml.safe_load(file)
+            rate_limits = RateLimitsConfig(**loaded_rate_limits)
+        except OSError:
+            logger.exception(
+                "Failed to read rate limits file", rate_limits_file=rate_limits_file
+            )
+        except pydantic.ValidationError:
+            logger.exception(
+                "Failed to validate rate limits file", rate_limits_file=rate_limits_file
+            )
     return rate_limits
 
 
@@ -184,7 +185,7 @@ def profiles_api_url(self) -> str:
         "{base_url}/datasets/{process_id}?tab=download#manage-licences"
     )
 
-    rate_limits_file: str = "/etc/retrieve-api/rate-limits.yaml"
+    rate_limits_file: str | None = None
     rate_limits: RateLimitsConfig = pydantic.Field(default=RateLimitsConfig())
 
     @pydantic.model_validator(mode="after")  # type: ignore

tests/test_10_config.py

@@ -61,15 +61,19 @@ def test_validate_rate_limits() -> None:
     rate_limits = ["1/second", "10/minute"]
     config.validate_rate_limits(rate_limits)
 
-    rate_limits = ["not_valid_limit"]
+    rate_limits = ["invalid_limit"]
     with pytest.raises(ValueError):
         config.validate_rate_limits(rate_limits)
 
 
-def test_load_rate_limits(tmp_path: pathlib.Path) -> None:
+def test_load_rate_limits(tmp_path: pathlib.Path, caplog) -> None:
+    rate_limits_file = None
+    loaded_rate_limits = config.load_rate_limits(rate_limits_file)
+    assert loaded_rate_limits == config.RateLimitsConfig()
+
     rate_limits_file = str(tmp_path / "rate-limits.yaml")
     rate_limits = {
-        "processes/{process_id}/execution": {
+        "/processes/{process_id}/execution": {
             "post": {"api": ["1/second"], "ui": ["2/second"]}
         },
     }
@@ -78,6 +82,15 @@ def test_load_rate_limits(tmp_path: pathlib.Path) -> None:
     loaded_rate_limits = config.load_rate_limits(rate_limits_file)
     assert loaded_rate_limits == config.RateLimitsConfig(**rate_limits)
 
+    rate_limits_file = str(tmp_path / "invalid-rate-limits.yaml")
+    rate_limits = {
+        "/processes/{process_id}/execution": {"post": {"api": ["invalid_limit"]}},
+    }
+    with open(rate_limits_file, "w") as file:
+        yaml.dump(rate_limits, file)
+    loaded_rate_limits = config.load_rate_limits(rate_limits_file)
+    assert loaded_rate_limits == config.RateLimitsConfig()
+
     rate_limits_file = str(tmp_path / "not-found-rate-limits.yaml")
     loaded_rate_limits = config.load_rate_limits(rate_limits_file)
     assert loaded_rate_limits == config.RateLimitsConfig()
@@ -90,7 +103,7 @@ def test_rate_limits_config_populate_with_default() -> None:
                 "post": {"api": ["1/second"], "ui": ["2/second"]},
                 "get": {"api": ["2/second"]},
             },
-            "processes/{process_id}/execution": {"post": {"api": ["1/minute"]}},
+            "/processes/{process_id}/execution": {"post": {"api": ["1/minute"]}},
         }
     )
     exp_populated_rate_limits_config = {