Change upstream location of testdata to UHH Gitlab

.github/workflows/build-test.yml

@@ -92,6 +92,16 @@ jobs:
             jcheck: 2
 
     steps:
+      - name: Check Runner IP
+        shell: bash -l {0}
+        run: |
+          echo "Runner public IP:"
+          curl -s https://api.ipify.org
+          echo
+          echo
+          echo "Test server access with dummy file download:"
+          curl https://arts.mi.uni-hamburg.de/svn/rt/arts-xml-data/trunk/planets/Earth/afgl/readme.txt
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 10
@@ -118,6 +128,16 @@ jobs:
         run: |
           cmake --preset=${{ matrix.preset }}  # -DARTS_CTEST_USER_OPTIONS="-V"
 
+      - name: Download xml testdata
+        shell: bash -l {0}
+        run: |
+          DEBUG_DOWNLOADING=1 cmake --build build -j1 --target arts_testdata_folder_download_xml
+
+      - name: Download cat testdata
+        shell: bash -l {0}
+        run: |
+          DEBUG_DOWNLOADING=1 cmake --build build -j1 --target arts_testdata_folder_download_cat
+
       - name: Build ARTS Workspace
         shell: bash -l {0}
         run: |

testdata/CMakeLists.txt

@@ -28,7 +28,7 @@ if(DOWNLOAD_CATDATA_DEPS)
         COMMAND ${Python_EXECUTABLE} ${ARTS_SOURCE_DIR}/testdata/get_testdata.py cat ${DOWNLOAD_CATDATA}
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
         DEPENDS get_testdata.py
-        COMMENT "Downloading test data"
+        COMMENT "Downloading cat testdata"
     )
 
     add_dependencies(check-deps arts_testdata_folder_download_cat)
@@ -81,6 +81,7 @@ if(DOWNLOAD_XMLDATA_DEPS)
         COMMAND ${Python_EXECUTABLE} ${ARTS_SOURCE_DIR}/testdata/get_testdata.py xml ${DOWNLOAD_XMLDATA}
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
         DEPENDS get_testdata.py
+        COMMENT "Downloading xml testdata"
     )
 
     add_dependencies(check-deps arts_testdata_folder_download_xml)

testdata/get_testdata.py

@@ -1,18 +1,31 @@
+import logging
 import os
 import sys
 import requests
 import requests.adapters
 from requests.adapters import HTTPAdapter, Retry
-from time import sleep
 
+if os.environ.get("DEBUG_DOWNLOADING") is not None:
+    import http.client
 
-def download_file(url, destination):
+    http.client.HTTPConnection.debuglevel = 1
+
+    logging.basicConfig(
+        level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s"
+    )
+    logging.getLogger("urllib3").setLevel(logging.DEBUG)
+    logging.getLogger("urllib3").propagate = True
+else:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+
+def download_file(url, destination, nretry=5):
     """Download a file from a URL to a destination file."""
     try:
         # Send a GET request to the URL
         s = requests.Session()
         retries = Retry(
-            total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]
+            total=nretry, backoff_factor=1, status_forcelist=[500, 502, 503, 504]
         )
         s.mount("https://", HTTPAdapter(max_retries=retries))
         response = s.get(url)
@@ -27,33 +40,25 @@ def download_file(url, destination):
                 file.write(chunk)
 
         return True
-    except requests.exceptions.RequestException:
+    except requests.exceptions.RequestException as err:
+        logging.error(f"Download failed: {err}")
         return False
 
 
-def download_file_retry(url, destination, retries=30, delay=10):
-    """Download a file from a URL to a destination file with retries."""
-    x = 0
-    for i in range(retries):
-        sleep(x)
-        x = delay
-        if download_file(url, destination):
-            print(f"Downloaded {destination}")
-            return
-        print(f"Failed to download from {url}, sleeping for {delay} seconds")
-    raise RuntimeError(f"Failed to download file {destination} after {retries} retries")
-
-
 nargs = len(sys.argv)
 
-xml = "https://arts.mi.uni-hamburg.de/svn/rt/arts-xml-data/trunk/"
-cat = "https://arts.mi.uni-hamburg.de/svn/rt/arts-cat-data/trunk/"
+xml = "https://gitlab.rrz.uni-hamburg.de/atmtools/arts-xml-data/-/raw/main/"
+cat = "https://gitlab.rrz.uni-hamburg.de/atmtools/arts-cat-data/-/raw/main/"
+xml_fallback = "https://arts.mi.uni-hamburg.de/svn/rt/arts-xml-data/trunk/"
+cat_fallback = "https://arts.mi.uni-hamburg.de/svn/rt/arts-cat-data/trunk/"
 
 if sys.argv[1] == "xml":
     baseurl = xml
+    baseurl_fallback = xml_fallback
     basedir = "arts-xml-data"
 elif sys.argv[1] == "cat":
     baseurl = cat
+    baseurl_fallback = cat_fallback
     basedir = "arts-cat-data"
 else:
     baseurl = ""
@@ -63,6 +68,13 @@ def download_file_retry(url, destination, retries=30, delay=10):
     f = f"{basedir}/{file}"
     if not os.path.exists(f):
         os.makedirs(os.path.split(f)[0], exist_ok=True)
-        download_file_retry(f"{baseurl}{file}", f)
+        if not download_file(f"{baseurl}{file}", f):
+            logging.warning(
+                f"Failed to download file {file} from {baseurl}, trying fallback URL {baseurl_fallback}"
+            )
+            baseurl = baseurl_fallback
+            if not download_file(f"{baseurl_fallback}{file}", f):
+                raise RuntimeError(f"Failed to download file {file}")
+        logging.info(f"Downloaded {file}")
     else:
-        print(f"Skipping {file}, exists at {os.path.abspath(f)}")
+        logging.info(f"Skipping {file}, exists at {os.path.abspath(f)}")