fix(429): unblock manual resume + add auto-resume toggle (#133)

hydropix · claude · hydropix · commit 03732bb5f494 · 2026-05-04T23:18:08.000+02:00
- UI now handles 'rate_limited' status (was missing, leaving the
  progress bar spinning and the Resume button greyed out)
- Interrupt endpoint accepts 'rate_limited' (was 400-ing)
- New AUTO_PAUSE_ON_RATE_LIMIT toggle (env + Options checkbox):
  when off, translation waits Retry-After seconds and auto-resumes
  from the last checkpoint instead of pausing

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.env.example b/.env.example
@@ -98,6 +98,13 @@ AUTO_ADJUST_CONTEXT=true  # Automatically adjust context/chunk size if prompt to
 # Advanced
 MAX_TRANSLATION_ATTEMPTS=3
 
+# Auto-pause on HTTP 429 rate limit
+# true  (default): pause translation after retries are exhausted, user resumes manually
+# false:           wait `Retry-After` seconds (or RATE_LIMIT_AUTO_RESUME_DELAY) and auto-resume
+#                  from the last checkpoint. Useful for long novels on free-tier APIs.
+AUTO_PAUSE_ON_RATE_LIMIT=true
+RATE_LIMIT_AUTO_RESUME_DELAY=60
+
 EPUB_TOKEN_ALIGNMENT_ENABLED=true
 # Options: true (enable Phase 2 fallback), false (use old behavior with only Phase 1 + Phase 3)
 
diff --git a/src/api/blueprints/translation_routes.py b/src/api/blueprints/translation_routes.py
@@ -8,7 +8,8 @@
 
 from src.config import (
     REQUEST_TIMEOUT,
-    OLLAMA_NUM_CTX
+    OLLAMA_NUM_CTX,
+    AUTO_PAUSE_ON_RATE_LIMIT
 )
 from src.tts.tts_config import TTSConfig
 
@@ -80,6 +81,8 @@ def start_translation_request():
             'openrouter_api_key': _resolve_api_key(data.get('openrouter_api_key'), 'OPENROUTER_API_KEY'),
             # Prompt options (optional instructions to include in the system prompt)
             'prompt_options': data.get('prompt_options', {}),
+            # Auto-pause on rate limit toggle (request overrides .env default)
+            'auto_pause_on_rate_limit': data.get('auto_pause_on_rate_limit', AUTO_PAUSE_ON_RATE_LIMIT),
             # Bilingual output (original + translation interleaved)
             'bilingual_output': data.get('bilingual_output', False),
             # TTS configuration
@@ -152,12 +155,22 @@ def interrupt_translation_job(translation_id):
             return jsonify({"error": "Translation not found"}), 404
 
         job_data = state_manager.get_translation(translation_id)
-        if job_data.get('status') == 'running' or job_data.get('status') == 'queued':
+        status = job_data.get('status')
+        if status in ('running', 'queued'):
             state_manager.set_interrupted(translation_id, True)
             return jsonify({
                 "message": "Interruption signal sent. Translation will stop after the current segment."
             }), 200
 
+        if status == 'rate_limited':
+            # Cancels any in-flight auto-resume sleep and stops the UI from treating
+            # the job as still-active.
+            state_manager.set_interrupted(translation_id, True)
+            state_manager.set_translation_field(translation_id, 'status', 'interrupted')
+            return jsonify({
+                "message": "Auto-resume cancelled. Translation marked interrupted; you can resume manually later."
+            }), 200
+
         return jsonify({
             "message": "The translation is not in an interruptible state (e.g., already completed or failed)."
         }), 400
diff --git a/src/api/handlers.py b/src/api/handlers.py
@@ -14,6 +14,7 @@
 from src.utils.file_utils import get_unique_output_path, generate_tts_for_translation
 from src.core.llm import OpenRouterProvider
 from src.core.llm.exceptions import RateLimitError
+from src.config import AUTO_PAUSE_ON_RATE_LIMIT, RATE_LIMIT_AUTO_RESUME_DELAY
 from src.core.adapters import translate_file
 from src.tts.tts_config import TTSConfig
 from .websocket import emit_update
@@ -440,44 +441,82 @@ def _openrouter_cost_callback(cost_data):
             }, namespace='/')
 
     except RateLimitError as e:
-        # Auto-pause on rate limit — save checkpoint so user can resume later
+        auto_pause = config.get('auto_pause_on_rate_limit', AUTO_PAUSE_ON_RATE_LIMIT)
         retry_msg = f" Retry suggested after ~{e.retry_after}s." if e.retry_after else ""
         provider_name = e.provider or config.get('llm_provider', 'API')
-        pause_msg = f"⏸️ Rate limited by {provider_name}.{retry_msg} Translation auto-paused — you can resume when ready."
 
-        _log_message_callback("rate_limit_auto_pause", pause_msg)
-
-        if state_manager.exists(translation_id):
-            state_manager.set_translation_field(translation_id, 'status', 'rate_limited')
-            state_manager.set_translation_field(translation_id, 'interrupted', True)
-
-            # Mark checkpoint as interrupted so it appears in resumable jobs
-            checkpoint_manager.mark_interrupted(translation_id)
+        if not state_manager.exists(translation_id):
+            return
 
-            stats = state_manager.get_translation_field(translation_id, 'stats') or {}
-            elapsed_time = time.time() - stats.get('start_time', time.time())
-            _update_translation_stats_callback({'elapsed_time': elapsed_time})
+        # Auto-resume mode keeps the job running: wait, then re-enter from the checkpoint.
+        if not auto_pause:
+            wait_seconds = e.retry_after or RATE_LIMIT_AUTO_RESUME_DELAY
+            wait_msg = (f"⏳ Rate limited by {provider_name}.{retry_msg} "
+                        f"Auto-resume in {wait_seconds}s (auto-pause disabled).")
+            _log_message_callback("rate_limit_auto_resume", wait_msg)
 
-            # Emit rate_limited status + checkpoint_created for UI
+            # Surface 'rate_limited' transiently so the UI shows what's happening.
+            state_manager.set_translation_field(translation_id, 'status', 'rate_limited')
             emit_update(socketio, translation_id, {
                 'status': 'rate_limited',
-                'log': pause_msg,
-                'result': state_manager.get_translation_field(translation_id, 'result') or f"Translation paused (rate limited)"
+                'log': wait_msg
             }, state_manager)
 
-            socketio.emit('checkpoint_created', {
-                'translation_id': translation_id,
-                'status': 'rate_limited',
-                'message': pause_msg
-            }, namespace='/')
+            await asyncio.sleep(wait_seconds)
 
-            # Trigger file list refresh for partial output
-            output_filepath = state_manager.get_translation_field(translation_id, 'output_filepath')
-            if output_filepath and os.path.exists(output_filepath):
-                socketio.emit('file_list_changed', {
-                    'reason': 'rate_limited',
-                    'filename': config.get('output_filename', 'unknown')
-                }, namespace='/')
+            # Honor an interrupt that arrived during the wait by falling through to pause.
+            if state_manager.get_translation_field(translation_id, 'interrupted'):
+                _log_message_callback("rate_limit_auto_resume_cancelled",
+                    "🛑 Auto-resume cancelled by user, pausing instead.")
+            else:
+                cp_data = checkpoint_manager.load_checkpoint(translation_id)
+                if cp_data:
+                    new_config = dict(config)
+                    new_config['is_resume'] = True
+                    new_config['resume_from_index'] = cp_data['resume_from_index']
+                    checkpoint_manager.mark_running(translation_id)
+                    state_manager.set_translation_field(translation_id, 'status', 'running')
+                    emit_update(socketio, translation_id, {
+                        'status': 'running',
+                        'log': f"▶️ Auto-resuming from chunk {cp_data['resume_from_index']}..."
+                    }, state_manager)
+                    await perform_actual_translation(
+                        translation_id, new_config, state_manager, output_dir, socketio
+                    )
+                    return
+                # No checkpoint available, fall through to the pause path below.
+                _log_message_callback("rate_limit_no_checkpoint",
+                    "⚠️ Auto-resume requested but no checkpoint found, falling back to pause.")
+
+        pause_msg = f"⏸️ Rate limited by {provider_name}.{retry_msg} Translation auto-paused, you can resume when ready."
+        _log_message_callback("rate_limit_auto_pause", pause_msg)
+
+        state_manager.set_translation_field(translation_id, 'status', 'rate_limited')
+        state_manager.set_translation_field(translation_id, 'interrupted', True)
+        checkpoint_manager.mark_interrupted(translation_id)
+
+        stats = state_manager.get_translation_field(translation_id, 'stats') or {}
+        elapsed_time = time.time() - stats.get('start_time', time.time())
+        _update_translation_stats_callback({'elapsed_time': elapsed_time})
+
+        emit_update(socketio, translation_id, {
+            'status': 'rate_limited',
+            'log': pause_msg,
+            'result': state_manager.get_translation_field(translation_id, 'result') or f"Translation paused (rate limited)"
+        }, state_manager)
+
+        socketio.emit('checkpoint_created', {
+            'translation_id': translation_id,
+            'status': 'rate_limited',
+            'message': pause_msg
+        }, namespace='/')
+
+        output_filepath = state_manager.get_translation_field(translation_id, 'output_filepath')
+        if output_filepath and os.path.exists(output_filepath):
+            socketio.emit('file_list_changed', {
+                'reason': 'rate_limited',
+                'filename': config.get('output_filename', 'unknown')
+            }, namespace='/')
 
     except Exception as e:
         critical_error_msg = f"Critical error during translation task ({translation_id}): {str(e)}"
diff --git a/src/config.py b/src/config.py
@@ -135,6 +135,13 @@
 THINKING_MODELS = UNCONTROLLABLE_THINKING_MODELS + CONTROLLABLE_THINKING_MODELS
 MAX_TRANSLATION_ATTEMPTS = int(os.getenv('MAX_TRANSLATION_ATTEMPTS', '2'))
 
+# Auto-pause on HTTP 429 rate limit
+# When True (default): translation pauses after retries are exhausted; user resumes manually.
+# When False: translation auto-resumes from the last checkpoint after waiting `retry_after`
+# seconds (or RATE_LIMIT_AUTO_RESUME_DELAY if no Retry-After header).
+AUTO_PAUSE_ON_RATE_LIMIT = os.getenv('AUTO_PAUSE_ON_RATE_LIMIT', 'true').lower() == 'true'
+RATE_LIMIT_AUTO_RESUME_DELAY = int(os.getenv('RATE_LIMIT_AUTO_RESUME_DELAY', '60'))
+
 # Adaptive context optimization settings
 # The new strategy starts at a small context and grows as needed based on actual token usage
 AUTO_ADJUST_CONTEXT = os.getenv("AUTO_ADJUST_CONTEXT", "true").lower() == "true"
diff --git a/src/web/static/js/core/settings-manager.js b/src/web/static/js/core/settings-manager.js
@@ -66,6 +66,7 @@ const LOCAL_SETTINGS = [
     'textCleanup',
     'refineTranslation',
     'bilingualMode',
+    'disableAutoPause',
     'customInstructionFile',
     'apiEndpointCustomized',  // Track if user manually changed endpoint
     'openaiEndpointCustomized'
@@ -174,6 +175,7 @@ export const SettingsManager = {
             { id: 'textCleanup', event: 'change' },
             { id: 'refineTranslation', event: 'change' },
             { id: 'bilingualMode', event: 'change' },
+            { id: 'disableAutoPause', event: 'change' },
             { id: 'customInstructionSelect', event: 'change' }
         ];
 
@@ -357,14 +359,20 @@ export const SettingsManager = {
                 bilingualCheckbox.checked = prefs.bilingualMode;
             }
         }
+        if (prefs.disableAutoPause !== undefined) {
+            const disableAutoPauseCheckbox = DomHelpers.getElement('disableAutoPause');
+            if (disableAutoPauseCheckbox) {
+                disableAutoPauseCheckbox.checked = prefs.disableAutoPause;
+            }
+        }
 
         // Store custom instruction file for later application (after loadCustomInstructions completes)
         if (prefs.customInstructionFile) {
             window.__pendingCustomInstructionSelection = prefs.customInstructionFile;
         }
 
         // Keep Prompt Options section open if any option is active
-        const hasAnyPromptOption = prefs.textCleanup || prefs.refineTranslation || prefs.bilingualMode || prefs.customInstructionFile;
+        const hasAnyPromptOption = prefs.textCleanup || prefs.refineTranslation || prefs.bilingualMode || prefs.disableAutoPause || prefs.customInstructionFile;
         if (hasAnyPromptOption) {
             const promptOptionsSection = DomHelpers.getElement('promptOptionsSection');
             const promptOptionsIcon = DomHelpers.getElement('promptOptionsIcon');
@@ -419,6 +427,7 @@ export const SettingsManager = {
         const textCleanupCheckbox = DomHelpers.getElement('textCleanup');
         const refineTranslationCheckbox = DomHelpers.getElement('refineTranslation');
         const bilingualModeCheckbox = DomHelpers.getElement('bilingualMode');
+        const disableAutoPauseCheckbox = DomHelpers.getElement('disableAutoPause');
 
         const prefs = {
             lastProvider: DomHelpers.getValue('llmProvider'),
@@ -432,6 +441,7 @@ export const SettingsManager = {
             textCleanup: textCleanupCheckbox ? textCleanupCheckbox.checked : false,
             refineTranslation: refineTranslationCheckbox ? refineTranslationCheckbox.checked : false,
             bilingualMode: bilingualModeCheckbox ? bilingualModeCheckbox.checked : false,
+            disableAutoPause: disableAutoPauseCheckbox ? disableAutoPauseCheckbox.checked : false,
             customInstructionFile: DomHelpers.getValue('customInstructionSelect') || ''
         };
 
diff --git a/src/web/static/js/translation/batch-controller.js b/src/web/static/js/translation/batch-controller.js
@@ -87,6 +87,7 @@ function getTranslationConfig(file) {
         file_type: file.fileType,
         prompt_options: promptOptions,
         bilingual_output: DomHelpers.getElement('bilingualMode')?.checked || false,
+        auto_pause_on_rate_limit: !(DomHelpers.getElement('disableAutoPause')?.checked || false),
         tts_enabled: ttsEnabled,
         tts_voice: ttsEnabled ? (DomHelpers.getValue('ttsVoice') || '') : '',
         tts_rate: ttsEnabled ? (DomHelpers.getValue('ttsRate') || '+0%') : '+0%',
diff --git a/src/web/static/js/translation/translation-tracker.js b/src/web/static/js/translation/translation-tracker.js
@@ -191,7 +191,8 @@ export const TranslationTracker = {
 
                     if (serverState.status === 'completed' ||
                         serverState.status === 'error' ||
-                        serverState.status === 'interrupted') {
+                        serverState.status === 'interrupted' ||
+                        serverState.status === 'rate_limited') {
 
                         MessageLogger.addLog(`🔄 Syncing state: translation ${serverState.status} on server`);
                         this.resetUIToIdle();
@@ -392,7 +393,7 @@ export const TranslationTracker = {
 
         if (!currentJob || data.translation_id !== currentJob.translationId) {
             if (data.translation_id && !currentJob) {
-                if (data.status === 'completed' || data.status === 'error' || data.status === 'interrupted') {
+                if (data.status === 'completed' || data.status === 'error' || data.status === 'interrupted' || data.status === 'rate_limited') {
                     this.resetUIToIdle();
                 }
             }
@@ -432,6 +433,14 @@ export const TranslationTracker = {
                 data
             );
             this.updateActiveTranslationsState();
+        } else if (data.status === 'rate_limited') {
+            MessageLogger.resetProgressTracking();
+            this.finishCurrentFileTranslation(
+                `⏸️ ${currentFile.name}: Rate limited, translation auto-paused. Use the Resume button below when ready.`,
+                'info',
+                data
+            );
+            this.updateActiveTranslationsState();
         } else if (data.status === 'error') {
             MessageLogger.resetProgressTracking();
             this.finishCurrentFileTranslation(
@@ -699,7 +708,8 @@ export const TranslationTracker = {
         this.updateFileStatusInList(
             currentFile.name,
             resultData.status === 'completed' ? 'Completed' :
-            (resultData.status === 'interrupted' ? 'Interrupted' : 'Error')
+            resultData.status === 'interrupted' ? 'Interrupted' :
+            resultData.status === 'rate_limited' ? 'Rate Limited' : 'Error'
         );
 
         StateManager.setState('translation.currentJob', null);
@@ -709,6 +719,9 @@ export const TranslationTracker = {
         } else if (resultData.status === 'interrupted') {
             MessageLogger.addLog('🛑 Batch processing stopped by user.');
             this.resetUIToIdle();
+        } else if (resultData.status === 'rate_limited') {
+            MessageLogger.addLog('⏸️ Batch processing paused (rate limited). Resume from the section below.');
+            this.resetUIToIdle();
         } else {
             this.processNextFileInQueue();
         }
diff --git a/src/web/static/js/ui/form-manager.js b/src/web/static/js/ui/form-manager.js
@@ -688,6 +688,8 @@ export const FormManager = {
             },
             // Bilingual output (original + translation interleaved)
             bilingual_output: DomHelpers.getElement('bilingualMode')?.checked || false,
+            // Disable auto-pause on rate limit (auto-resume after Retry-After)
+            auto_pause_on_rate_limit: !(DomHelpers.getElement('disableAutoPause')?.checked || false),
             // TTS configuration
             tts_enabled: ttsEnabled,
             tts_voice: ttsEnabled ? (DomHelpers.getValue('ttsVoice') || '') : '',
diff --git a/src/web/static/js/utils/lifecycle-manager.js b/src/web/static/js/utils/lifecycle-manager.js
@@ -221,7 +221,7 @@ export const LifecycleManager = {
             const data = await ApiClient.getTranslationStatus(tidToCheck);
             const serverStatus = data.status;
 
-            if (serverStatus === 'completed' || serverStatus === 'error' || serverStatus === 'interrupted') {
+            if (serverStatus === 'completed' || serverStatus === 'error' || serverStatus === 'interrupted' || serverStatus === 'rate_limited') {
                 MessageLogger.addLog(`🔄 Detected state desync: job ${serverStatus} on server but UI still active. Syncing...`);
 
                 window.dispatchEvent(new CustomEvent('translationUpdate', {
diff --git a/src/web/templates/translation_interface.html b/src/web/templates/translation_interface.html
@@ -387,7 +387,7 @@ <h3>Drop files to translate</h3>
                     </div>
 
                     <!-- Text Cleanup (OCR Correction) -->
-                    <div class="form-group" style="margin-bottom: 0;">
+                    <div class="form-group" style="margin-bottom: 15px;">
                         <label style="display: flex; align-items: flex-start; gap: 10px; cursor: pointer;">
                             <input type="checkbox" id="textCleanup">
                             <div>
@@ -399,6 +399,19 @@ <h3>Drop files to translate</h3>
                         </label>
                     </div>
 
+                    <!-- Disable Auto-Pause on Rate Limit -->
+                    <div class="form-group" style="margin-bottom: 0;">
+                        <label style="display: flex; align-items: flex-start; gap: 10px; cursor: pointer;">
+                            <input type="checkbox" id="disableAutoPause">
+                            <div>
+                                <span style="font-weight: 600; color: var(--text-dark);">Don't auto-pause on rate limit (HTTP 429)</span>
+                                <p style="margin: 5px 0 0 0; font-size: 0.8125rem; color: var(--text-muted-light);">
+                                    By default, the translation pauses after the provider rate-limits and asks you to resume manually. Enable this to keep the translation running: it will wait for <code>Retry-After</code> seconds (or 60s default) and auto-resume from the last checkpoint. Useful for long novels on free-tier APIs. Click "Interrupt" anytime to cancel the auto-resume.
+                                </p>
+                            </div>
+                        </label>
+                    </div>
+
                     <!-- Custom Instructions -->
                     <div class="form-group" style="margin-bottom: 0; margin-top: 15px;">
                         <label style="font-weight: 600; color: var(--text-dark); margin-bottom: 5px; display: block;">
