From 4a8c7d2e5b8561a92b1b88f18b0680cfe017810d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Dec 2025 18:00:02 +0000 Subject: [PATCH 1/3] Add 30-minute timeout for MSA server PENDING state The ColabFold MSA server (api.colabfold.com) can rate-limit or drop jobs, causing them to stay in PENDING state forever. Previously, jobs would wait 12+ hours before the RQ job timeout killed them. This change: - Detects when MSA jobs are stuck in PENDING state - Fails fast after 30 minutes with a clear error message - Advises users to retry (which typically resolves the issue) Root cause: The MSA server appears to expire/deprioritize old job IDs rather than processing them when rate limits free up. Fresh job submissions (from restart) get new IDs that work. See: https://github.com/sokrypton/ColabFold/issues/606 --- backend/app/jobs/boltz_jobs.py | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/backend/app/jobs/boltz_jobs.py b/backend/app/jobs/boltz_jobs.py index 7d3bc279..ae561fd8 100644 --- a/backend/app/jobs/boltz_jobs.py +++ b/backend/app/jobs/boltz_jobs.py @@ -1,7 +1,9 @@ import io import logging +import re import string import subprocess +import time from pathlib import Path from tempfile import TemporaryDirectory @@ -21,6 +23,18 @@ ) from app.models import Fold, Invokation +# Maximum time to wait for MSA server jobs stuck in PENDING state before failing. +# The ColabFold MSA server (api.colabfold.com) can get rate-limited or drop jobs, +# causing them to stay PENDING forever. Failing fast allows users to retry sooner. +# See: https://github.com/sokrypton/ColabFold/issues/606 +MSA_PENDING_TIMEOUT_SECONDS = 30 * 60 # 30 minutes + + +class MSAServerTimeoutError(Exception): + """Raised when MSA server jobs are stuck in PENDING state too long.""" + + pass + def try_check_smiles_string_validity(smiles_string): """Try to check if a smiles string is valid.""" @@ -120,9 +134,38 @@ def run_boltz(fold_id, invokation_id): text=True, ) + # Track MSA PENDING state to detect stuck jobs + pending_start_time = None + pending_pattern = re.compile(r"PENDING.*\d+/\d+") + for line in iter(process.stdout.readline, ""): logging.info(line.strip()) + # Check if this line indicates MSA jobs stuck in PENDING + if pending_pattern.search(line): + if pending_start_time is None: + pending_start_time = time.time() + logging.info( + f"MSA server jobs entered PENDING state. " + f"Will timeout after {MSA_PENDING_TIMEOUT_SECONDS // 60} minutes." + ) + elif time.time() - pending_start_time > MSA_PENDING_TIMEOUT_SECONDS: + process.kill() + process.wait() + raise MSAServerTimeoutError( + f"MSA server jobs stuck in PENDING state for over " + f"{MSA_PENDING_TIMEOUT_SECONDS // 60} minutes. " + f"The ColabFold MSA server (api.colabfold.com) may be " + f"rate-limiting or has dropped your jobs. " + f"Please retry the job - restarting often resolves this issue. " + f"See: https://github.com/sokrypton/ColabFold/issues/606" + ) + elif "COMPLETE" in line or "RUNNING" in line: + # Reset timer if jobs start making progress + if pending_start_time is not None: + logging.info("MSA server jobs progressing, resetting PENDING timer.") + pending_start_time = None + process.stdout.close() process.wait() From b81782924b785815d799ab80efc72fefac03c4d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Dec 2025 18:02:55 +0000 Subject: [PATCH 2/3] Revert "Add 30-minute timeout for MSA server PENDING state" This reverts commit 4a8c7d2e5b8561a92b1b88f18b0680cfe017810d. --- backend/app/jobs/boltz_jobs.py | 43 ---------------------------------- 1 file changed, 43 deletions(-) diff --git a/backend/app/jobs/boltz_jobs.py b/backend/app/jobs/boltz_jobs.py index ae561fd8..7d3bc279 100644 --- a/backend/app/jobs/boltz_jobs.py +++ b/backend/app/jobs/boltz_jobs.py @@ -1,9 +1,7 @@ import io import logging -import re import string import subprocess -import time from pathlib import Path from tempfile import TemporaryDirectory @@ -23,18 +21,6 @@ ) from app.models import Fold, Invokation -# Maximum time to wait for MSA server jobs stuck in PENDING state before failing. -# The ColabFold MSA server (api.colabfold.com) can get rate-limited or drop jobs, -# causing them to stay PENDING forever. Failing fast allows users to retry sooner. -# See: https://github.com/sokrypton/ColabFold/issues/606 -MSA_PENDING_TIMEOUT_SECONDS = 30 * 60 # 30 minutes - - -class MSAServerTimeoutError(Exception): - """Raised when MSA server jobs are stuck in PENDING state too long.""" - - pass - def try_check_smiles_string_validity(smiles_string): """Try to check if a smiles string is valid.""" @@ -134,38 +120,9 @@ def run_boltz(fold_id, invokation_id): text=True, ) - # Track MSA PENDING state to detect stuck jobs - pending_start_time = None - pending_pattern = re.compile(r"PENDING.*\d+/\d+") - for line in iter(process.stdout.readline, ""): logging.info(line.strip()) - # Check if this line indicates MSA jobs stuck in PENDING - if pending_pattern.search(line): - if pending_start_time is None: - pending_start_time = time.time() - logging.info( - f"MSA server jobs entered PENDING state. " - f"Will timeout after {MSA_PENDING_TIMEOUT_SECONDS // 60} minutes." - ) - elif time.time() - pending_start_time > MSA_PENDING_TIMEOUT_SECONDS: - process.kill() - process.wait() - raise MSAServerTimeoutError( - f"MSA server jobs stuck in PENDING state for over " - f"{MSA_PENDING_TIMEOUT_SECONDS // 60} minutes. " - f"The ColabFold MSA server (api.colabfold.com) may be " - f"rate-limiting or has dropped your jobs. " - f"Please retry the job - restarting often resolves this issue. " - f"See: https://github.com/sokrypton/ColabFold/issues/606" - ) - elif "COMPLETE" in line or "RUNNING" in line: - # Reset timer if jobs start making progress - if pending_start_time is not None: - logging.info("MSA server jobs progressing, resetting PENDING timer.") - pending_start_time = None - process.stdout.close() process.wait() From 95e1b9c3156a5a5748f6ee454b25788712f3e6cd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 20 Dec 2025 19:04:22 +0000 Subject: [PATCH 3/3] Add MSA PENDING timeout detection and RQ retry for boltz jobs Problem: MSA server jobs can get stuck in PENDING state forever when the ColabFold server drops or deprioritizes jobs. The client keeps polling the same dead job ID indefinitely (PENDING is a "sink" - it never triggers resubmission like RATELIMIT does). Solution: 1. Detect PENDING state stuck for >10 minutes and fail fast with TimeoutError instead of waiting 12+ hours for RQ job timeout 2. Configure RQ to retry boltz jobs up to 3 times (1min, 2min, 5min intervals) - each retry gets a fresh MSA job ID 3. Upgrade Boltz from 2.1.1 to 2.2.1 (adds auth support, bug fixes) This allows automatic recovery from dropped MSA jobs without manual intervention. See: https://github.com/sokrypton/ColabFold/issues/606 --- backend/app/jobs/boltz_jobs.py | 35 ++++++++++++++++++++++++++++++++++ backend/app/util.py | 1 + worker/Dockerfile.boltz | 3 +-- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/backend/app/jobs/boltz_jobs.py b/backend/app/jobs/boltz_jobs.py index 7d3bc279..ce15eaa8 100644 --- a/backend/app/jobs/boltz_jobs.py +++ b/backend/app/jobs/boltz_jobs.py @@ -1,7 +1,9 @@ import io import logging +import re import string import subprocess +import time from pathlib import Path from tempfile import TemporaryDirectory @@ -21,6 +23,12 @@ ) from app.models import Fold, Invokation +# Maximum time to wait for MSA server jobs stuck in PENDING state before failing. +# The ColabFold MSA server can drop or deprioritize jobs, causing them to stay +# PENDING forever. Failing fast allows RQ retry to resubmit with a fresh job ID. +# See: https://github.com/sokrypton/ColabFold/issues/606 +MSA_PENDING_TIMEOUT_SECONDS = 10 * 60 # 10 minutes + def try_check_smiles_string_validity(smiles_string): """Try to check if a smiles string is valid.""" @@ -120,9 +128,36 @@ def run_boltz(fold_id, invokation_id): text=True, ) + # Track MSA PENDING state to detect stuck jobs + pending_start_time = None + pending_pattern = re.compile(r"PENDING.*\d+/\d+") + for line in iter(process.stdout.readline, ""): logging.info(line.strip()) + # Check if this line indicates MSA jobs stuck in PENDING + if pending_pattern.search(line): + if pending_start_time is None: + pending_start_time = time.time() + logging.info( + f"MSA server jobs entered PENDING state. " + f"Will timeout after {MSA_PENDING_TIMEOUT_SECONDS // 60} minutes." + ) + elif time.time() - pending_start_time > MSA_PENDING_TIMEOUT_SECONDS: + process.kill() + process.wait() + raise TimeoutError( + f"MSA server jobs stuck in PENDING state for over " + f"{MSA_PENDING_TIMEOUT_SECONDS // 60} minutes. " + f"The ColabFold server may have dropped this job. " + f"See: https://github.com/sokrypton/ColabFold/issues/606" + ) + elif "RUNNING" in line or "COMPLETE" in line: + # Reset timer if jobs start making progress + if pending_start_time is not None: + logging.info("MSA server jobs progressing, resetting PENDING timer.") + pending_start_time = None + process.stdout.close() process.wait() diff --git a/backend/app/util.py b/backend/app/util.py index c6468081..05a2c051 100644 --- a/backend/app/util.py +++ b/backend/app/util.py @@ -119,6 +119,7 @@ def start_stage(fold_id: int, stage: str, email_on_completion: bool) -> None: get_job_type_replacement(fold, "boltz"), job_timeout="12h", result_ttl=48 * 60 * 60, # 2 days + retry=Retry(max=3, interval=[60, 120, 300]), # Retry on MSA PENDING timeout **email_args, ) add_meta_to_job(boltz_job, fold, "boltz", fold.id) diff --git a/worker/Dockerfile.boltz b/worker/Dockerfile.boltz index 0546006f..3dd36b9f 100644 --- a/worker/Dockerfile.boltz +++ b/worker/Dockerfile.boltz @@ -44,8 +44,7 @@ RUN /scripts/conda_env_setup.sh RUN /opt/conda/bin/conda create -y -n boltzenv python=3.11 # RUN /opt/conda/envs/boltzenv/bin/python -m pip install boltz==1.0.0 -RUN /root/.local/bin/uv pip install --python /opt/conda/envs/boltzenv/bin/python boltz==2.1.1 -# RUN /opt/conda/envs/boltzenv/bin/python -m pip install boltz==2.2.0 +RUN /root/.local/bin/uv pip install --python /opt/conda/envs/boltzenv/bin/python boltz==2.2.1 # # Chat suggested this, for when on a weird network. # RUN echo "nameserver 8.8.8.8" > /etc/resolv.conf && \