Hotfix for flu data support

docs/docs/configuration.md

@@ -32,6 +32,7 @@ Every key below corresponds to an entry in `respro/config/defaults.toml`, where
 | `genbank_timeout` | int | `30` | `urlopen` timeout (seconds) for a single NCBI nuccore efetch attempt. |
 | `genbank_max_retries` | int | `3` | Max efetch attempts before raising `RuntimeError`. |
 | `genbank_backoff_base` | float | `1.0` | Exponential backoff base (seconds); doubled each retry (1, 2, 4). |
+| `genbank_request_interval` | float | `0.5` | Minimum pause (seconds) between successive GenBank efetch requests during a bulk download. Keeps `databases --download` under NCBI eutils' limit of 3 requests/second (without an API key), preventing HTTP 429 rate-limit errors. |
 
 ### `[urls]` — external service URL templates
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'hatchling.build'
 
 [project]
 name = 'respro'
-version = '0.3.1'
+version = '0.3.2'
 description = 'Pathogen-agnostic framework for genotypic antiviral resistance profiling from SANGER or NGS-derived variants.'
 readme = 'README.md'
 license = {text = 'AGPL-3.0-only'}

respro/config/cli_settings.py

@@ -16,12 +16,15 @@ class CliTimeoutConfig:
     pubchem: int
     pubmed: int
     crossref: int
-    # GenBank/NCBI nuccore efetch timeout, retry count, and exponential-backoff
-    # base (seconds; doubled each retry). Promoted from respro/io/maintained_db.py
-    # module constants so they join the overridable config surface.
+    # GenBank/NCBI nuccore efetch timeout, retry count, exponential-backoff
+    # base (seconds; doubled each retry), and inter-request pacing interval
+    # (seconds; keeps bulk downloads under NCBI's 3 req/s rate limit).
+    # Promoted from respro/io/maintained_db.py module constants so they join
+    # the overridable config surface.
     genbank_timeout: int
     genbank_max_retries: int
     genbank_backoff_base: float
+    genbank_request_interval: float
 
 
 @dataclass(frozen=True)
@@ -172,6 +175,7 @@ def _bins(section: dict) -> CliAfBinsConfig:
             genbank_timeout=int(timeouts['genbank_timeout']),
             genbank_max_retries=int(timeouts['genbank_max_retries']),
             genbank_backoff_base=float(timeouts['genbank_backoff_base']),
+            genbank_request_interval=float(timeouts['genbank_request_interval']),
         ),
         urls=CliUrlConfig(
             pubchem_compound_page=str(urls['pubchem_compound_page']),

respro/config/defaults.toml

@@ -9,6 +9,10 @@ crossref = 7
 genbank_timeout = 30        # seconds; urlopen timeout for a single efetch attempt
 genbank_max_retries = 3     # max attempts before raising RuntimeError
 genbank_backoff_base = 1.0  # seconds; doubled each retry (1, 2, 4)
+# Minimum pause (seconds) between successive GenBank efetch requests during a
+# bulk database download. NCBI eutils allows at most 3 requests/second without
+# an API key; pacing prevents the download from tripping HTTP 429 rate limits.
+genbank_request_interval = 0.5
 
 [urls]
 pubchem_compound_page = 'https://pubchem.ncbi.nlm.nih.gov/compound/{cid}'

respro/io/maintained_db.py

@@ -23,13 +23,13 @@
 
 logger = logging.getLogger(__name__)
 
-_GENBANK_TIMEOUT = 30
 # NCBI eutils is rate-limited and intermittently resets connections or returns
-# truncated chunked responses. Retry transient failures a few times with
-# exponential backoff before giving up. HTTPError is NOT retried — a 4xx/5xx
+# truncated chunked responses. Transient failures (429/503 below, plus transport
+# errors) are retried with exponential backoff before giving up — the timeout,
+# retry count, and backoff base live in [timeouts] of respro/config/defaults.toml
+# (genbank_timeout, genbank_max_retries, genbank_backoff_base). Any other 4xx/5xx
 # status is a permanent response, not a transport hiccup.
-_GENBANK_MAX_RETRIES = 3
-_GENBANK_BACKOFF_BASE = 1.0  # seconds; doubled each retry (1, 2, 4)
+_RETRYABLE_HTTP_CODES = frozenset({429, 503})
 
 # NCBI nucleotide accession: letters/digits/underscore, optional ``.version`` suffix.
 # Rejects path separators, ``..``, whitespace, and other characters that could escape
@@ -187,6 +187,22 @@ def list_output_files(db_name: str) -> list[dict]:
 # Helpers
 # ──────────────────────────────────────────────────────────────────────
 
+def _retry_after_seconds(exc: urllib.error.HTTPError, default: float) -> float:
+    """Return the wait time for a rate-limited response.
+
+    Honors a numeric ``Retry-After`` response header when present (NCBI sends
+    one on 429 responses); otherwise falls back to ``default``.
+    """
+    headers = getattr(exc, 'headers', None)
+    if headers is not None:
+        raw = headers.get('Retry-After')
+        if raw:
+            try:
+                return max(float(raw), 0.0)
+            except ValueError:
+                pass
+    return default
+
 def _fetch_manifest() -> dict:
     """Fetch and validate the global maintained database manifest."""
     manifest_url = f"{CLI_CONFIG.urls.github_respro_db_raw.rstrip('/')}/manifest.json"
@@ -370,12 +386,17 @@ def _fetch_genbank_records(accessions: list[str], dest_dir: Path) -> list[Path]:
     """
     Fetch GenBank records for a list of NCBI accession IDs.
 
-    Transient transport failures (``ConnectionResetError``, ``IncompleteRead``,
-    and other ``OSError`` / ``http.client.HTTPException`` subclasses) are
-    retried up to ``_GENBANK_MAX_RETRIES`` times with exponential backoff.
-    A permanent ``HTTPError`` (4xx/5xx) is raised immediately without
-    retrying. NCBI eutils is rate-limited, so a short inter-request delay is
-    also applied between successful accessions.
+    Requests are paced: a configurable inter-request delay
+    (``genbank_request_interval``) is applied between successive accessions so
+    a large database download stays under NCBI eutils' rate limit (3 requests
+    per second without an API key).
+
+    Transient failures are retried up to ``genbank_max_retries`` times with
+    exponential backoff: transport errors (``ConnectionResetError``,
+    ``IncompleteRead``, and other ``OSError`` / ``http.client.HTTPException``
+    subclasses) and rate-limit/availability HTTP responses (429, 503). A 429
+    response's ``Retry-After`` header, when present, overrides the backoff
+    delay. Any other HTTP error status is permanent and raised immediately.
 
     :param accessions: list of nucleotide accession strings
     :param dest_dir: directory where .gb files are written
@@ -390,7 +411,12 @@ def _fetch_genbank_records(accessions: list[str], dest_dir: Path) -> list[Path]:
     genbank_timeout = CLI_CONFIG.timeouts.genbank_timeout
     genbank_max_retries = CLI_CONFIG.timeouts.genbank_max_retries
     genbank_backoff_base = CLI_CONFIG.timeouts.genbank_backoff_base
-    for accession in accessions:
+    genbank_request_interval = CLI_CONFIG.timeouts.genbank_request_interval
+    for index, accession in enumerate(accessions):
+        if index > 0:
+            # Pace successive requests so bulk downloads stay under the NCBI
+            # eutils rate limit instead of triggering 429 responses.
+            time.sleep(genbank_request_interval)
         _validate_accession(accession)
         url = CLI_CONFIG.urls.ncbi_nuccore_efetch.format(accession=urllib.parse.quote(accession))
         dest = dest_dir / f'{accession}.gb'
@@ -403,10 +429,23 @@ def _fetch_genbank_records(accessions: list[str], dest_dir: Path) -> list[Path]:
                     content = resp.read()
                 break  # success
             except urllib.error.HTTPError as exc:
-                # Permanent response — do not retry.
-                raise RuntimeError(
-                    f'Failed to fetch GenBank record for {accession!r}: HTTP {exc.code}'
-                ) from exc
+                if exc.code not in _RETRYABLE_HTTP_CODES:
+                    # Permanent response — do not retry.
+                    raise RuntimeError(
+                        f'Failed to fetch GenBank record for {accession!r}: HTTP {exc.code}'
+                    ) from exc
+                # Rate limit (429) or availability (503) — retry with backoff,
+                # honoring a Retry-After header when NCBI provides one.
+                last_exc = exc
+                logger.warning(
+                    'Rate-limited/availability error fetching GenBank record %r '
+                    '(attempt %d/%d): HTTP %s',
+                    accession, attempt, genbank_max_retries, exc.code,
+                )
+                if attempt < genbank_max_retries:
+                    time.sleep(_retry_after_seconds(
+                        exc, genbank_backoff_base * (2 ** (attempt - 1)),
+                    ))
             except (OSError, http.client.HTTPException) as exc:
                 # Transient transport failure — retry with backoff.
                 last_exc = exc

respro/io/publications.py

@@ -33,6 +33,67 @@
 _DOI_BACKOFF_SECONDS = (0.5, 1.0, 2.0)
 _RE_DOI_LIKE = r'^10\.\S+/\S+$'
 
+# Publisher-deposit titles (notably from CrossRef) carry JATS/NLM inline markup
+# such as ``<scp>S</scp>`` (small caps) and ``<i>In Vitro</i>`` (italics), often
+# broken across lines with indentation. The tags are stripped and the text is
+# normalised to plain single-spaced prose. Word fragments split by a tag are
+# rejoined: a small-caps letter followed by a line break and the rest of the
+# word (``<scp>S</scp>\n  outhern``) must render as ``Southern``, while a
+# complete token followed by a real word boundary keeps its space.
+_TITLE_TAG_RE = re.compile(r'(\s*)<([A-Za-z][A-Za-z0-9]*)>(.*?)</\2>(\s*)', re.DOTALL)
+_RE_TITLE_LEFTOVER_TAG = re.compile(r'</?[A-Za-z][^>]*>')
+_RE_TITLE_WHITESPACE = re.compile(r'\s+')
+
+
+def _replace_title_tag(m: re.Match[str]) -> str:
+    """Replace one inline markup tag with plain text, preserving word integrity."""
+    leading, content_raw, trailing = m.group(1), m.group(3), m.group(4)
+    content = content_raw.strip()
+    if not content:
+        return f'{leading} '
+    if ' ' in content:
+        # Multi-word phrase (e.g. italicised "In Vitro"): keep it as separate
+        # words; surrounding whitespace runs are normalised afterwards.
+        return f'{leading} {content} '
+    rest = m.string[m.end():]
+    if not trailing:
+        # No whitespace between the closing tag and the next character: the
+        # fragment is tightly joined ("<scp>U</scp>nited", "<scp>H</scp>3N2",
+        # "<scp>A</scp>(").
+        if rest[:1].isalnum() or rest[:1] in '([':
+            return f'{leading}{content}'
+    elif rest[:1].islower() and ('\n' in trailing or len(trailing) > 1 or len(content) == 1):
+        # Whitespace run (a formatting line break with indentation) splits a
+        # word whose fragment is the tagged content, e.g.
+        # "<scp>S</scp>\n  outhern" → "Southern".
+        return f'{leading}{content}'
+    return f'{leading}{content} '
+
+
+def _clean_publication_title(raw_title: object) -> str:
+    """
+    Normalise a raw publication title to plain single-spaced text.
+
+    Removes inline JATS/NLM markup tags (e.g. ``<scp>``, ``<i>``, ``<sup>``),
+    rejoins word fragments split by tags across line breaks, and collapses all
+    whitespace runs (including newlines with indentation) to single spaces.
+
+    :param raw_title: raw title value from an API payload (any type)
+    :return: cleaned title string; empty string when the input is not a string
+    """
+    if not isinstance(raw_title, str):
+        return ''
+    title = raw_title
+    # Repeat until stable so nested markup (e.g. "<i><scp>X</scp></i>") resolves.
+    for _ in range(10):
+        cleaned = _TITLE_TAG_RE.sub(_replace_title_tag, title)
+        if cleaned == title:
+            break
+        title = cleaned
+    # Belt-and-braces: drop any leftover unclosed tags.
+    title = _RE_TITLE_LEFTOVER_TAG.sub('', title)
+    return _RE_TITLE_WHITESPACE.sub(' ', title).strip()
+
 
 def fetch_pubmed_metadata(pmid: str, timeout: int = CLI_CONFIG.timeouts.pubmed) -> dict | None:
     """
@@ -63,8 +124,7 @@ def fetch_pubmed_metadata(pmid: str, timeout: int = CLI_CONFIG.timeouts.pubmed)
             if not isinstance(result_data, dict):
                 logger.debug('NCBI PMID lookup malformed payload for %r: missing PMID result object', pmid)
                 return None
-            raw_title = result_data.get('title', '')
-            title = raw_title.strip() if isinstance(raw_title, str) else ''
+            title = _clean_publication_title(result_data.get('title', ''))
             doi = ''
             article_ids = result_data.get('articleids')
             if not isinstance(article_ids, list):
@@ -223,7 +283,7 @@ def fetch_publication_metadata(doi: str, timeout: int = CLI_CONFIG.timeouts.cros
                 logger.debug('CrossRef lookup malformed payload for DOI %r: title is not a list', normalized_doi)
                 return None
             first_title = titles[0] if titles else ''
-            title = first_title.strip() if isinstance(first_title, str) else ''
+            title = _clean_publication_title(first_title)
             first_author = _parse_crossref_first_author(message.get('author'))
             year = _parse_crossref_year(message)
             container = message.get('container-title')

tests/test_config.py

@@ -381,6 +381,12 @@ def test_genbank_backoff_base_default(self):
         assert config.timeouts.genbank_backoff_base == 1.0
         assert isinstance(config.timeouts.genbank_backoff_base, float)
 
+    def test_genbank_request_interval_default(self):
+        """Should load genbank_request_interval with documented default 0.5."""
+        config = _load_cli_config()
+        assert config.timeouts.genbank_request_interval == 0.5
+        assert isinstance(config.timeouts.genbank_request_interval, float)
+
     def test_defaults_toml_has_codon_promoted_keys(self):
         """defaults.toml should expose the promoted codon keys with inline comments."""
         defaults_path = files('respro.config').joinpath('defaults.toml')