feat(download): add default and configurable User-Agent for CLI, API, and workflows

Author: Integer-Ctrl

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+
+### Added
+- Workflow download steps support an optional `user_agent` field.
+- Download `--user-agent` CLI option and `user_agent` Python API argument to override the header on file HEAD and GET requests, including redirects and authentication retries. Download manifests preserve the override for replay.
+
+### Changed
+- File downloads now identify as `databus-python-client/<version> (https://github.com/dbpedia/databus-python-client)` instead of the Requests library default.
+
 ## [0.15] - 2025-12-31
 
 ### Added

databusclient/api/download.py

@@ -14,6 +14,8 @@
 from tqdm import tqdm
 from datetime import datetime, timezone
 
+from databusclient.version import __version__
+
 from databusclient.api.utils import (
     fetch_databus_jsonld,
     get_databus_id_parts_from_file_url,
@@ -41,6 +43,11 @@
     "xz": lzma,
 }
 
+DEFAULT_USER_AGENT = (
+    f"databus-python-client/{__version__} "
+    "(https://github.com/dbpedia/databus-python-client)"
+)
+
 GRAPH_MODES = {"download-url"}
 
 
@@ -420,6 +427,7 @@ def _download_file(
     validate_checksum: bool = False,
     expected_checksum: str | None = None,
     manifest_context=None,
+    user_agent: str | None = None,
 ) -> None:
     """Download a file from the internet with a progress bar using tqdm.
 
@@ -437,6 +445,8 @@ def _download_file(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
         expected_checksum: The expected checksum of the file.
     """
     _validate_graph_mode(graph_mode)
@@ -454,10 +464,10 @@ def _download_file(
         os.makedirs(dirpath, exist_ok=True)  # Create the necessary directories
 
     # --- 1. Get redirect URL by requesting HEAD ---
-    headers = {}
+    headers = {"User-Agent": DEFAULT_USER_AGENT if user_agent is None else user_agent}
 
     # --- 1a. public databus ---
-    response = requests.head(url, timeout=30, allow_redirects=False)
+    response = requests.head(url, headers=headers, timeout=30, allow_redirects=False)
 
     # Check for redirect and update URL if necessary
     if response.headers.get("Location") and response.status_code in [
@@ -470,7 +480,7 @@ def _download_file(
         url = response.headers.get("Location")
         print("Redirects url: ", url)
         # Re-do HEAD request on redirect URL
-        response = requests.head(url, timeout=30)
+        response = requests.head(url, headers=headers, timeout=30)
 
     # Extract hostname from final URL (after redirect) to check if vault token needed.
     # This is the actual download location that may require authentication.
@@ -493,7 +503,7 @@ def _download_file(
                 raise DownloadAuthError(
                     "Databus API key not given for protected download"
                 )
-            headers = {"X-API-KEY": databus_key}
+            headers["X-API-KEY"] = databus_key
             response = requests.head(url, headers=headers, timeout=30)
 
     # --- 2. Try direct GET to redirected URL ---
@@ -866,6 +876,7 @@ def _download_files(
     manifest_context=None,
     validate_checksum: bool = False,
     checksums: dict | None = None,
+    user_agent: str | None = None,
 ) -> None:
     """Download multiple files from the databus.
 
@@ -882,6 +893,8 @@ def _download_files(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
         checksums: Dictionary mapping URLs to their expected checksums.
     """
     for url in urls:
@@ -903,6 +916,7 @@ def _download_files(
             validate_checksum=validate_checksum,
             expected_checksum=expected,
             manifest_context=manifest_context,
+            user_agent=user_agent,
         )
 
 
@@ -1054,6 +1068,7 @@ def _download_collection(
     graph_mode: str = None,
     manifest_context=None,
     validate_checksum: bool = False,
+    user_agent: str | None = None,
 ) -> None:
     """Download all files in a databus collection.
 
@@ -1071,6 +1086,8 @@ def _download_collection(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
     """
     query = _get_sparql_query_of_collection(uri, databus_key=databus_key)
     file_urls = _get_file_download_urls_from_sparql_query(
@@ -1097,6 +1114,7 @@ def _download_collection(
         manifest_context=manifest_context,
         validate_checksum=validate_checksum,
         checksums=checksums if checksums else None,
+        user_agent=user_agent,
     )
 
 
@@ -1114,6 +1132,7 @@ def _download_version(
     graph_mode: str = None,
     manifest_context=None,
     validate_checksum: bool = False,
+    user_agent: str | None = None,
 ) -> None:
     """Download all files in a databus artifact version.
 
@@ -1130,6 +1149,8 @@ def _download_version(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
     """
     json_str = fetch_databus_jsonld(uri, databus_key=databus_key)
     file_urls = _get_file_download_urls_from_artifact_jsonld(json_str)
@@ -1155,6 +1176,7 @@ def _download_version(
         manifest_context=manifest_context,
         validate_checksum=validate_checksum,
         checksums=checksums,
+        user_agent=user_agent,
     )
 
 
@@ -1173,6 +1195,7 @@ def _download_artifact(
     graph_mode: str = None,
     manifest_context=None,
     validate_checksum: bool = False,
+    user_agent: str | None = None,
 ) -> None:
     """Download files in a databus artifact.
 
@@ -1190,6 +1213,8 @@ def _download_artifact(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
     """
     json_str = fetch_databus_jsonld(uri, databus_key=databus_key)
     versions = _get_databus_versions_of_artifact(json_str, all_versions=all_versions)
@@ -1221,6 +1246,7 @@ def _download_artifact(
             manifest_context=manifest_context,
             validate_checksum=validate_checksum,
             checksums=checksums,
+            user_agent=user_agent,
         )
 
 
@@ -1300,6 +1326,7 @@ def _download_group(
     graph_mode: str = None,
     manifest_context=None,
     validate_checksum: bool = False,
+    user_agent: str | None = None,
 ) -> None:
     """Download files in a databus group.
 
@@ -1317,6 +1344,8 @@ def _download_group(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
     """
     json_str = fetch_databus_jsonld(uri, databus_key=databus_key)
     artifacts = _get_databus_artifacts_of_group(json_str)
@@ -1337,6 +1366,7 @@ def _download_group(
             graph_mode=graph_mode,
             manifest_context=manifest_context,
             validate_checksum=validate_checksum,
+            user_agent=user_agent,
         )
 
 
@@ -1390,6 +1420,7 @@ def download(
     graph_mode=None,
     validate_checksum: bool = False,
     manifest_context=None,
+    user_agent: str | None = None,
 ) -> None:
     """Download datasets from databus.
 
@@ -1410,6 +1441,8 @@ def download(
         base_uri: Base URI for CSV -> Triple conversion (Layer 3).
         graph_mode: Optional graph sidecar mode. Currently supports 'download-url'.
         validate_checksum: Whether to validate checksums after downloading.
+        user_agent: HTTP User-Agent for file HEAD and GET requests. None uses
+                    the versioned databus-python-client default.
     """
     _validate_graph_mode(graph_mode)
     for databusURI in databusURIs:
@@ -1442,6 +1475,7 @@ def download(
                     graph_mode=graph_mode,
                     manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
+                    user_agent=user_agent,
                 )
             elif file is not None:
                 print(f"Downloading file: {databusURI}")
@@ -1468,6 +1502,7 @@ def download(
                     manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
                     expected_checksum=expected,
+                    user_agent=user_agent,
                 )
             elif version is not None:
                 print(f"Downloading version: {databusURI}")
@@ -1485,6 +1520,7 @@ def download(
                     graph_mode=graph_mode,
                     manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
+                    user_agent=user_agent,
                 )
             elif artifact is not None:
                 print(
@@ -1505,6 +1541,7 @@ def download(
                     graph_mode=graph_mode,
                     manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
+                    user_agent=user_agent,
                 )
             elif group is not None and group != "collections":
                 print(
@@ -1525,6 +1562,7 @@ def download(
                     graph_mode=graph_mode,
                     manifest_context=manifest_context,
                     validate_checksum=validate_checksum,
+                    user_agent=user_agent,
                 )
             elif account is not None:
                 print("accountId not supported yet")  # TODO
@@ -1568,4 +1606,5 @@ def download(
                 manifest_context=manifest_context,
                 validate_checksum=validate_checksum,
                 checksums=checksums if checksums else None,
+                user_agent=user_agent,
             )

databusclient/cli.py

@@ -321,6 +321,10 @@ def _write_manifest():
 @click.option(
     "--validate-checksum", is_flag=True, help="Validate checksums of downloaded files"
 )
+@click.option(
+    "--user-agent",
+    help="Override the HTTP User-Agent header for file download requests.",
+)
 def download(
     databusuris: List[str],
     localdir,
@@ -337,6 +341,7 @@ def download(
     graph_mode,
     validate_checksum,
     manifest_path,
+    user_agent,
 ):
     """
     Download datasets from databus, optionally using vault access if vault options are provided.
@@ -366,6 +371,7 @@ def download(
             "graph_mode": graph_mode,
             "all_versions": all_versions,
             "validate_checksum": validate_checksum,
+            "user_agent": user_agent,
             "authurl": authurl,
             "clientid": clientid,
         })
@@ -385,6 +391,7 @@ def download(
             base_uri=base_uri,
             graph_mode=graph_mode,
             validate_checksum=validate_checksum,
+            user_agent=user_agent,
             manifest_context=manifest_context,
         )
     except DownloadAuthError as e:

databusclient/manifest/replay.py

@@ -80,6 +80,7 @@ def _build_download_kwargs(
         "graph_name": replay_params.get("graph_name"),
         "base_uri": replay_params.get("base_uri"),
         "graph_mode": replay_params.get("graph_mode"),
+        "user_agent": replay_params.get("user_agent"),
         "validate_checksum": bool(replay_params.get("validate_checksum", False)),
         "manifest_context": None,
     }

databusclient/workflow/steps.py

@@ -81,6 +81,7 @@ def run(self, step_config: Dict[str, Any], context: StepContext) -> None:
             graph_name=resolved.get("graph_name"),
             base_uri=resolved.get("base_uri"),
             graph_mode=resolved.get("graph_mode"),
+            user_agent=resolved.get("user_agent"),
             validate_checksum=resolved.get("validate_checksum", False),
             manifest_context=capture_context,
         )

doc/cli-usage.md

@@ -40,6 +40,8 @@ databusclient download $DOWNLOADTARGET
   - Required when converting CSV/TSV to RDF triples. Used as the base for constructing subject URIs from CSV row identifiers. Example: `--format ntriples --base-uri https://example.org/data/`.
 - `--validate-checksum`
   - Validates the checksums of downloaded files against the checksums provided by the Databus. If a checksum does not match, an error is raised and the file is deleted.
+- `--user-agent`
+  - Overrides the HTTP User-Agent header for file download requests. Default: `databus-python-client/<version> (https://github.com/dbpedia/databus-python-client)`. Example: `--user-agent "DatabusDownloadBot/1.0 (mailto:you@example.org)"`.
 
 **Help and further information on download command:**
 ```bash
@@ -406,6 +408,7 @@ steps:
     uri: https://databus.dbpedia.org/dbpedia/mappings/mappingbased-literals/2022.12.01/mappingbased-literals_lang=az.ttl.bz2
     localdir: ./data
     graph_mode: download-url
+    user_agent: "DatabusDownloadBot/1.0 (mailto:you@example.org)"
 
   - name: publish_dataset
     command: deploy
@@ -419,6 +422,8 @@ steps:
     on_error: fail
 ```
 
+Download steps accept an optional `user_agent` field, equivalent to `download --user-agent`. If omitted, the client default is used.
+
 **Environment variables:** any value written as `${VARIABLE_NAME}` is resolved from the environment when the workflow starts. If the variable is not set, the workflow fails immediately with a clear error before any step runs — credentials should always be passed this way, never written directly in the file.
 
 **Step chaining:** a step's outputs can be referenced by later steps using `${steps.step_name.output_key}`:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databusclient"
-version = "1.1.0"
+version = "1.2.0"
 description = "A simple client for submitting, downloading, and deleting data on the DBpedia Databus"
 authors = ["DBpedia Association"]
 license = "Apache-2.0 License"