Merge pull request #213 from ecmwf-projects/COPDS-2076

Filter out empty lines from config file

Author: malmans2

cads_processing_api_service/config.py

@@ -96,12 +96,13 @@ def profiles_api_url(self) -> str:
         return f"http://{self.profiles_service}:{self.profiles_api_service_port}"
 
     @property
-    def data_volume(self) -> str:
-        data_volumes_config_path = self.download_nodes_config
-        with open(data_volumes_config_path) as fp:
-            data_volumes = [os.path.expandvars(line.rstrip("\n")) for line in fp]
-        data_volume = random.choice(data_volumes)
-        return data_volume
+    def download_node(self) -> str:
+        download_nodes = []
+        with open(self.download_nodes_config) as fp:
+            for line in fp:
+                if download_node := os.path.expandvars(line.rstrip("\n")):
+                    download_nodes.append(download_node)
+        return random.choice(download_nodes)
 
 
 def ensure_settings(

cads_processing_api_service/utils.py

@@ -470,11 +470,11 @@ def get_job_from_broker_db(
     return job
 
 
-def update_results_href(local_path: str, data_volume: str | None = None) -> str:
-    if data_volume is None:
-        data_volume = config.ensure_settings().data_volume
+def update_results_href(local_path: str, download_node: str | None = None) -> str:
+    if download_node is None:
+        download_node = config.ensure_settings().download_node
     file_path = local_path.split("://", 1)[-1]
-    results_href = urllib.parse.urljoin(data_volume, file_path)
+    results_href = urllib.parse.urljoin(download_node, file_path)
     return results_href
 
 

tests/test_30_utils.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 # mypy: ignore-errors
-
 import datetime
+import pathlib
 import unittest.mock
 import uuid
 from typing import Any
@@ -244,19 +244,32 @@ def test_get_job_from_broker_db() -> None:
 
 
 def test_update_results_href() -> None:
-    data_volume = "http://data_volume/"
+    download_node = "http://download_node/"
 
     local_path = "protocol://results/1234"
-    updated_href = utils.update_results_href(local_path, data_volume)
-    exp_updated_href = "http://data_volume/results/1234"
+    updated_href = utils.update_results_href(local_path, download_node)
+    exp_updated_href = "http://download_node/results/1234"
     assert updated_href == exp_updated_href
 
     local_path = "results/1234"
-    updated_href = utils.update_results_href(local_path, data_volume)
-    exp_updated_href = "http://data_volume/results/1234"
+    updated_href = utils.update_results_href(local_path, download_node)
+    exp_updated_href = "http://download_node/results/1234"
     assert updated_href == exp_updated_href
 
 
+def test_update_results_href_from_config(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path
+) -> None:
+    config_path = tmp_path / "download-nodes.config"
+    config_path.write_text("\n\nhttp://download_node/\n\n$DOWNLOAD_NODE\n\n")
+    monkeypatch.setenv("DOWNLOAD_NODE", "http://download_node/")
+    monkeypatch.setenv("DOWNLOAD_NODES_CONFIG", str(config_path))
+
+    local_path = "protocol://results/1234"
+    updated_href = utils.update_results_href(local_path)
+    assert updated_href == "http://download_node/results/1234"
+
+
 def test_get_results_from_job() -> None:
     mock_session = unittest.mock.Mock(spec=sqlalchemy.orm.Session)
     job = cads_broker.SystemRequest(