From 496ac4cf59834f23b767c8fc641b4e93e8fbad11 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 12:21:26 -0400 Subject: [PATCH 1/8] structured output retries basic middleware --- libs/langchain_v1/langchain/agents/factory.py | 16 +- .../langchain/agents/middleware/__init__.py | 2 + .../middleware/structured_output_retry.py | 47 ++++ .../langchain/agents/structured_output.py | 12 +- .../test_structured_output_retry.py | 228 ++++++++++++++++++ libs/langchain_v1/uv.lock | 2 +- 6 files changed, 300 insertions(+), 7 deletions(-) create mode 100644 libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py create mode 100644 libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index 2f9962759fc7a..4dad7411d70ba 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -797,8 +797,16 @@ def _handle_model_output( provider_strategy_binding = ProviderStrategyBinding.from_schema_spec( effective_response_format.schema_spec ) - structured_response = provider_strategy_binding.parse(output) - return {"messages": [output], "structured_response": structured_response} + try: + structured_response = provider_strategy_binding.parse(output) + except Exception as exc: # noqa: BLE001 + schema_name = getattr( + effective_response_format.schema_spec.schema, "__name__", "response_format" + ) + exception = StructuredOutputValidationError(schema_name, exc, output) + raise exception + else: + return {"messages": [output], "structured_response": structured_response} return {"messages": [output]} # Handle structured output with tool strategy @@ -816,7 +824,7 @@ def _handle_model_output( if len(structured_tool_calls) > 1: # Handle multiple structured outputs error tool_names = [tc["name"] for tc in structured_tool_calls] - exception = MultipleStructuredOutputsError(tool_names) + exception = MultipleStructuredOutputsError(tool_names, output) should_retry, error_message = _handle_structured_output_error( exception, effective_response_format ) @@ -858,7 +866,7 @@ def _handle_model_output( "structured_response": structured_response, } except Exception as exc: # noqa: BLE001 - exception = StructuredOutputValidationError(tool_call["name"], exc) + exception = StructuredOutputValidationError(tool_call["name"], exc, output) should_retry, error_message = _handle_structured_output_error( exception, effective_response_format ) diff --git a/libs/langchain_v1/langchain/agents/middleware/__init__.py b/libs/langchain_v1/langchain/agents/middleware/__init__.py index 8ed35aafcd5c4..2d8e8912ec12b 100644 --- a/libs/langchain_v1/langchain/agents/middleware/__init__.py +++ b/libs/langchain_v1/langchain/agents/middleware/__init__.py @@ -24,6 +24,7 @@ RedactionRule, ShellToolMiddleware, ) +from .structured_output_retry import StructuredOutputRetryMiddleware from .summarization import SummarizationMiddleware from .todo import TodoListMiddleware from .tool_call_limit import ToolCallLimitMiddleware @@ -65,6 +66,7 @@ "PIIMiddleware", "RedactionRule", "ShellToolMiddleware", + "StructuredOutputRetryMiddleware", "SummarizationMiddleware", "TodoListMiddleware", "ToolCallLimitMiddleware", diff --git a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py new file mode 100644 index 0000000000000..9062f9ccaa8ce --- /dev/null +++ b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py @@ -0,0 +1,47 @@ +"""Minimal structured output retry middleware example.""" + +from langchain_core.messages import HumanMessage +from langchain.agents.middleware.types import AgentMiddleware, ModelRequest, ModelResponse +from langchain.agents.structured_output import StructuredOutputError + + +class StructuredOutputRetryMiddleware(AgentMiddleware): + """Retries model calls when structured output parsing fails.""" + + def __init__(self, max_retries: int = 2): + self.max_retries = max_retries + + def wrap_model_call(self, request: ModelRequest, handler) -> ModelResponse: + for attempt in range(self.max_retries + 1): + try: + return handler(request) + except StructuredOutputError as exc: + if attempt >= self.max_retries: + raise + + # Add error feedback for retry + if exc.ai_message: + request.messages.append(exc.ai_message) + + request.messages.append( + HumanMessage( + content=f"Error: {exc}. Please try again with a valid response." + ) + ) + + async def awrap_model_call(self, request: ModelRequest, handler) -> ModelResponse: + for attempt in range(self.max_retries + 1): + try: + return await handler(request) + except StructuredOutputError as exc: + if attempt >= self.max_retries: + raise + + if exc.ai_message: + request.messages.append(exc.ai_message) + + request.messages.append( + HumanMessage( + content=f"Error: {exc}. Please try again with a valid response." + ) + ) diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index cd6a2fd9aed31..858cb2068d0fd 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -34,17 +34,21 @@ class StructuredOutputError(Exception): """Base class for structured output errors.""" + ai_message: AIMessage + class MultipleStructuredOutputsError(StructuredOutputError): """Raised when model returns multiple structured output tool calls when only one is expected.""" - def __init__(self, tool_names: list[str]) -> None: + def __init__(self, tool_names: list[str], ai_message: AIMessage) -> None: """Initialize `MultipleStructuredOutputsError`. Args: tool_names: The names of the tools called for structured output. + ai_message: The AI message that contained the invalid multiple tool calls. """ self.tool_names = tool_names + self.ai_message = ai_message super().__init__( "Model incorrectly returned multiple structured responses " @@ -55,15 +59,19 @@ def __init__(self, tool_names: list[str]) -> None: class StructuredOutputValidationError(StructuredOutputError): """Raised when structured output tool call arguments fail to parse according to the schema.""" - def __init__(self, tool_name: str, source: Exception) -> None: + def __init__( + self, tool_name: str, source: Exception, ai_message: AIMessage + ) -> None: """Initialize `StructuredOutputValidationError`. Args: tool_name: The name of the tool that failed. source: The exception that occurred. + ai_message: The AI message that contained the invalid structured output. """ self.tool_name = tool_name self.source = source + self.ai_message = ai_message super().__init__(f"Failed to parse structured output for tool '{tool_name}': {source}.") diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py new file mode 100644 index 0000000000000..4a3c5b6aa3d69 --- /dev/null +++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py @@ -0,0 +1,228 @@ +"""Tests for StructuredOutputRetryMiddleware.""" + +from unittest.mock import MagicMock + +import pytest +from langchain_core.messages import AIMessage, HumanMessage +from pydantic import BaseModel, ValidationError + +from langchain.agents.middleware import ( + ModelRequest, + ModelResponse, + StructuredOutputRetryMiddleware, +) +from langchain.agents.structured_output import ( + MultipleStructuredOutputsError, + StructuredOutputError, + StructuredOutputValidationError, +) + + +class WeatherReport(BaseModel): + """Weather report schema.""" + + temperature: float + conditions: str + + +def test_structured_output_retry_initialization() -> None: + """Test that StructuredOutputRetryMiddleware initializes correctly.""" + middleware = StructuredOutputRetryMiddleware() + assert middleware.max_retries == 2 + + middleware_custom = StructuredOutputRetryMiddleware(max_retries=5) + assert middleware_custom.max_retries == 5 + + +def test_structured_output_retry_validation_errors() -> None: + """Test that middleware validates parameters.""" + with pytest.raises(ValueError, match="max_retries must be >= 0"): + StructuredOutputRetryMiddleware(max_retries=-1) + + +def test_structured_output_retry_success_first_attempt() -> None: + """Test that successful calls on first attempt don't trigger retry.""" + middleware = StructuredOutputRetryMiddleware(max_retries=2) + + # Create mock request and handler + mock_request = MagicMock(spec=ModelRequest) + mock_request.messages = [] + + expected_response = ModelResponse( + result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], + structured_response=WeatherReport(temperature=72.5, conditions="sunny"), + ) + + handler = MagicMock(return_value=expected_response) + + # Execute + result = middleware.wrap_model_call(mock_request, handler) + + # Verify + assert result == expected_response + assert handler.call_count == 1 + + +def test_structured_output_retry_with_validation_error() -> None: + """Test that validation errors trigger retry with error feedback.""" + middleware = StructuredOutputRetryMiddleware(max_retries=2) + + # Create mock request + mock_request = MagicMock(spec=ModelRequest) + mock_request.messages = [] + + # First call fails, second succeeds + ai_msg_with_error = AIMessage(content='{"temperature": "hot", "conditions": "sunny"}') + validation_error = ValidationError.from_exception_data( + "WeatherReport", [{"type": "float_parsing", "loc": ("temperature",), "input": "hot"}] + ) + error = StructuredOutputValidationError("WeatherReport", validation_error, ai_msg_with_error) + + success_response = ModelResponse( + result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], + structured_response=WeatherReport(temperature=72.5, conditions="sunny"), + ) + + handler = MagicMock(side_effect=[error, success_response]) + + # Execute + result = middleware.wrap_model_call(mock_request, handler) + + # Verify + assert result == success_response + assert handler.call_count == 2 + + # Check that error feedback was added to messages + assert len(mock_request.messages) == 2 + assert mock_request.messages[0] == ai_msg_with_error + assert isinstance(mock_request.messages[1], HumanMessage) + assert "errors" in mock_request.messages[1].content.lower() + + +def test_structured_output_retry_exhausted() -> None: + """Test that exception is raised when retries are exhausted.""" + middleware = StructuredOutputRetryMiddleware(max_retries=2) + + # Create mock request + mock_request = MagicMock(spec=ModelRequest) + mock_request.messages = [] + + # All calls fail + ai_msg = AIMessage(content='{"temperature": "hot", "conditions": "sunny"}') + validation_error = ValidationError.from_exception_data( + "WeatherReport", [{"type": "float_parsing", "loc": ("temperature",), "input": "hot"}] + ) + error = StructuredOutputValidationError("WeatherReport", validation_error, ai_msg) + + handler = MagicMock(side_effect=error) + + # Execute and verify it raises + with pytest.raises(StructuredOutputValidationError): + middleware.wrap_model_call(mock_request, handler) + + # Verify handler was called 3 times (initial + 2 retries) + assert handler.call_count == 3 + + +def test_structured_output_retry_multiple_outputs_error() -> None: + """Test that MultipleStructuredOutputsError triggers retry.""" + middleware = StructuredOutputRetryMiddleware(max_retries=1) + + # Create mock request + mock_request = MagicMock(spec=ModelRequest) + mock_request.messages = [] + + # First call has multiple outputs error + ai_msg = AIMessage( + content="", + tool_calls=[ + {"name": "WeatherReport", "args": {}, "id": "1"}, + {"name": "WeatherReport", "args": {}, "id": "2"}, + ], + ) + error = MultipleStructuredOutputsError(["WeatherReport", "WeatherReport"], ai_msg) + + # Second call succeeds + success_response = ModelResponse( + result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], + structured_response=WeatherReport(temperature=72.5, conditions="sunny"), + ) + + handler = MagicMock(side_effect=[error, success_response]) + + # Execute + result = middleware.wrap_model_call(mock_request, handler) + + # Verify + assert result == success_response + assert handler.call_count == 2 + + + + + + +def test_structured_output_retry_ai_message_preserved() -> None: + """Test that AI message from exception is preserved in retry messages.""" + middleware = StructuredOutputRetryMiddleware(max_retries=1) + + # Create mock request + mock_request = MagicMock(spec=ModelRequest) + mock_request.messages = [] + + # Create specific AI message with error + ai_msg_with_error = AIMessage( + content='{"temperature": "invalid", "conditions": "sunny"}', + id="test-id-123", + ) + error = StructuredOutputValidationError("WeatherReport", ValueError("Invalid"), ai_msg_with_error) + + success_response = ModelResponse( + result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], + structured_response=WeatherReport(temperature=72.5, conditions="sunny"), + ) + + handler = MagicMock(side_effect=[error, success_response]) + + # Execute + result = middleware.wrap_model_call(mock_request, handler) + + # Verify the original AI message with error was added to messages + assert len(mock_request.messages) == 2 + assert mock_request.messages[0] == ai_msg_with_error + assert mock_request.messages[0].id == "test-id-123" + + +async def test_structured_output_retry_async() -> None: + """Test async version of retry middleware.""" + middleware = StructuredOutputRetryMiddleware(max_retries=1) + + # Create mock request + mock_request = MagicMock(spec=ModelRequest) + mock_request.messages = [] + + # First call fails + ai_msg = AIMessage(content='{"temperature": "hot"}') + error = StructuredOutputValidationError("WeatherReport", ValueError("Invalid"), ai_msg) + + # Second call succeeds + success_response = ModelResponse( + result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], + structured_response=WeatherReport(temperature=72.5, conditions="sunny"), + ) + + async def async_handler(request: ModelRequest) -> ModelResponse: + if not hasattr(async_handler, "call_count"): + async_handler.call_count = 0 # type: ignore[attr-defined] + async_handler.call_count += 1 # type: ignore[attr-defined] + + if async_handler.call_count == 1: # type: ignore[attr-defined] + raise error + return success_response + + # Execute + result = await middleware.awrap_model_call(mock_request, async_handler) + + # Verify + assert result == success_response + assert async_handler.call_count == 2 # type: ignore[attr-defined] diff --git a/libs/langchain_v1/uv.lock b/libs/langchain_v1/uv.lock index 41d6066b33e10..714b6ec057f6c 100644 --- a/libs/langchain_v1/uv.lock +++ b/libs/langchain_v1/uv.lock @@ -1743,7 +1743,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.0.0" +version = "1.0.1" source = { editable = "../core" } dependencies = [ { name = "jsonpatch" }, From dbf9a62099042a762435973ece876ece2b3b2256 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:10:29 -0400 Subject: [PATCH 2/8] linting --- .../middleware/structured_output_retry.py | 30 ++++++++++++------- .../langchain/agents/structured_output.py | 4 +-- .../test_structured_output_retry.py | 8 ++--- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py index 9062f9ccaa8ce..db8e82632dc49 100644 --- a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py +++ b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py @@ -1,17 +1,28 @@ """Minimal structured output retry middleware example.""" from langchain_core.messages import HumanMessage -from langchain.agents.middleware.types import AgentMiddleware, ModelRequest, ModelResponse + +from langchain.agents.middleware.types import ( + AgentMiddleware, + Awaitable, + Callable, + ModelRequest, + ModelResponse, +) from langchain.agents.structured_output import StructuredOutputError class StructuredOutputRetryMiddleware(AgentMiddleware): """Retries model calls when structured output parsing fails.""" - def __init__(self, max_retries: int = 2): + def __init__(self, max_retries: int = 2) -> None: + """Initialize the structured output retry middleware.""" self.max_retries = max_retries - def wrap_model_call(self, request: ModelRequest, handler) -> ModelResponse: + def wrap_model_call( + self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse] + ) -> ModelResponse: + """Intercept and control model execution via handler callback.""" for attempt in range(self.max_retries + 1): try: return handler(request) @@ -24,12 +35,13 @@ def wrap_model_call(self, request: ModelRequest, handler) -> ModelResponse: request.messages.append(exc.ai_message) request.messages.append( - HumanMessage( - content=f"Error: {exc}. Please try again with a valid response." - ) + HumanMessage(content=f"Error: {exc}. Please try again with a valid response.") ) - async def awrap_model_call(self, request: ModelRequest, handler) -> ModelResponse: + async def awrap_model_call( + self, request: ModelRequest, handler: Callable[[ModelRequest], Awaitable[ModelResponse]] + ) -> ModelResponse: + """Intercept and control async model execution via handler callback.""" for attempt in range(self.max_retries + 1): try: return await handler(request) @@ -41,7 +53,5 @@ async def awrap_model_call(self, request: ModelRequest, handler) -> ModelRespons request.messages.append(exc.ai_message) request.messages.append( - HumanMessage( - content=f"Error: {exc}. Please try again with a valid response." - ) + HumanMessage(content=f"Error: {exc}. Please try again with a valid response.") ) diff --git a/libs/langchain_v1/langchain/agents/structured_output.py b/libs/langchain_v1/langchain/agents/structured_output.py index 858cb2068d0fd..750386758077f 100644 --- a/libs/langchain_v1/langchain/agents/structured_output.py +++ b/libs/langchain_v1/langchain/agents/structured_output.py @@ -59,9 +59,7 @@ def __init__(self, tool_names: list[str], ai_message: AIMessage) -> None: class StructuredOutputValidationError(StructuredOutputError): """Raised when structured output tool call arguments fail to parse according to the schema.""" - def __init__( - self, tool_name: str, source: Exception, ai_message: AIMessage - ) -> None: + def __init__(self, tool_name: str, source: Exception, ai_message: AIMessage) -> None: """Initialize `StructuredOutputValidationError`. Args: diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py index 4a3c5b6aa3d69..4a71c1dac5e56 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py +++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py @@ -158,10 +158,6 @@ def test_structured_output_retry_multiple_outputs_error() -> None: assert handler.call_count == 2 - - - - def test_structured_output_retry_ai_message_preserved() -> None: """Test that AI message from exception is preserved in retry messages.""" middleware = StructuredOutputRetryMiddleware(max_retries=1) @@ -175,7 +171,9 @@ def test_structured_output_retry_ai_message_preserved() -> None: content='{"temperature": "invalid", "conditions": "sunny"}', id="test-id-123", ) - error = StructuredOutputValidationError("WeatherReport", ValueError("Invalid"), ai_msg_with_error) + error = StructuredOutputValidationError( + "WeatherReport", ValueError("Invalid"), ai_msg_with_error + ) success_response = ModelResponse( result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], From 1d61f0d505452f41e8c5d339665b4b3bf2da0d2f Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:23:58 -0400 Subject: [PATCH 3/8] example --- libs/langchain_v1/langchain/agents/factory.py | 7 +-- .../middleware/structured_output_retry.py | 45 ++++++++++++------- .../test_structured_output_retry.py | 26 +++++------ 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/factory.py b/libs/langchain_v1/langchain/agents/factory.py index 4dad7411d70ba..59ac049b13fe9 100644 --- a/libs/langchain_v1/langchain/agents/factory.py +++ b/libs/langchain_v1/langchain/agents/factory.py @@ -33,6 +33,7 @@ ProviderStrategy, ProviderStrategyBinding, ResponseFormat, + StructuredOutputError, StructuredOutputValidationError, ToolStrategy, ) @@ -803,8 +804,8 @@ def _handle_model_output( schema_name = getattr( effective_response_format.schema_spec.schema, "__name__", "response_format" ) - exception = StructuredOutputValidationError(schema_name, exc, output) - raise exception + validation_error = StructuredOutputValidationError(schema_name, exc, output) + raise validation_error else: return {"messages": [output], "structured_response": structured_response} return {"messages": [output]} @@ -820,7 +821,7 @@ def _handle_model_output( ] if structured_tool_calls: - exception: Exception | None = None + exception: StructuredOutputError | None = None if len(structured_tool_calls) > 1: # Handle multiple structured outputs error tool_names = [tc["name"] for tc in structured_tool_calls] diff --git a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py index db8e82632dc49..26cdd3bccb8ac 100644 --- a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py +++ b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py @@ -1,11 +1,11 @@ """Minimal structured output retry middleware example.""" +from collections.abc import Awaitable, Callable + from langchain_core.messages import HumanMessage from langchain.agents.middleware.types import ( AgentMiddleware, - Awaitable, - Callable, ModelRequest, ModelResponse, ) @@ -16,7 +16,11 @@ class StructuredOutputRetryMiddleware(AgentMiddleware): """Retries model calls when structured output parsing fails.""" def __init__(self, max_retries: int = 2) -> None: - """Initialize the structured output retry middleware.""" + """Initialize the structured output retry middleware. + + Args: + max_retries: Maximum number of retry attempts. + """ self.max_retries = max_retries def wrap_model_call( @@ -27,16 +31,20 @@ def wrap_model_call( try: return handler(request) except StructuredOutputError as exc: - if attempt >= self.max_retries: + if attempt == self.max_retries: raise - # Add error feedback for retry - if exc.ai_message: - request.messages.append(exc.ai_message) - - request.messages.append( - HumanMessage(content=f"Error: {exc}. Please try again with a valid response.") + # Include both the AI message and error in a single human message + # to maintain valid chat history alternation + ai_content = exc.ai_message.content + error_message = ( + f"Your previous response was:\n{ai_content}\n\n" + f"Error: {exc}. Please try again with a valid response." ) + request.messages.append(HumanMessage(content=error_message)) + + # This should never be reached, but satisfies type checker + return handler(request) async def awrap_model_call( self, request: ModelRequest, handler: Callable[[ModelRequest], Awaitable[ModelResponse]] @@ -46,12 +54,17 @@ async def awrap_model_call( try: return await handler(request) except StructuredOutputError as exc: - if attempt >= self.max_retries: + if attempt == self.max_retries: raise - if exc.ai_message: - request.messages.append(exc.ai_message) - - request.messages.append( - HumanMessage(content=f"Error: {exc}. Please try again with a valid response.") + # Include both the AI message and error in a single human message + # to maintain valid chat history alternation + ai_content = exc.ai_message.content + error_message = ( + f"Your previous response was:\n{ai_content}\n\n" + f"Error: {exc}. Please try again with a valid response." ) + request.messages.append(HumanMessage(content=error_message)) + + # This should never be reached, but satisfies type checker + return await handler(request) diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py index 4a71c1dac5e56..749547a19c8c0 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py +++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py @@ -34,12 +34,6 @@ def test_structured_output_retry_initialization() -> None: assert middleware_custom.max_retries == 5 -def test_structured_output_retry_validation_errors() -> None: - """Test that middleware validates parameters.""" - with pytest.raises(ValueError, match="max_retries must be >= 0"): - StructuredOutputRetryMiddleware(max_retries=-1) - - def test_structured_output_retry_success_first_attempt() -> None: """Test that successful calls on first attempt don't trigger retry.""" middleware = StructuredOutputRetryMiddleware(max_retries=2) @@ -93,10 +87,11 @@ def test_structured_output_retry_with_validation_error() -> None: assert handler.call_count == 2 # Check that error feedback was added to messages - assert len(mock_request.messages) == 2 - assert mock_request.messages[0] == ai_msg_with_error - assert isinstance(mock_request.messages[1], HumanMessage) - assert "errors" in mock_request.messages[1].content.lower() + assert len(mock_request.messages) == 1 + assert isinstance(mock_request.messages[0], HumanMessage) + # The human message should contain both the AI's previous response and the error + assert "your previous response" in mock_request.messages[0].content.lower() + assert "error" in mock_request.messages[0].content.lower() def test_structured_output_retry_exhausted() -> None: @@ -159,7 +154,7 @@ def test_structured_output_retry_multiple_outputs_error() -> None: def test_structured_output_retry_ai_message_preserved() -> None: - """Test that AI message from exception is preserved in retry messages.""" + """Test that AI message content from exception is embedded in retry message.""" middleware = StructuredOutputRetryMiddleware(max_retries=1) # Create mock request @@ -185,10 +180,11 @@ def test_structured_output_retry_ai_message_preserved() -> None: # Execute result = middleware.wrap_model_call(mock_request, handler) - # Verify the original AI message with error was added to messages - assert len(mock_request.messages) == 2 - assert mock_request.messages[0] == ai_msg_with_error - assert mock_request.messages[0].id == "test-id-123" + # Verify the AI message content was embedded in the human message + assert len(mock_request.messages) == 1 + assert isinstance(mock_request.messages[0], HumanMessage) + # Check that the AI's previous content is included + assert '{"temperature": "invalid", "conditions": "sunny"}' in mock_request.messages[0].content async def test_structured_output_retry_async() -> None: From d6a4577081c5631abc115d1370ec0a93051a114e Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:25:18 -0400 Subject: [PATCH 4/8] removing actual --- .../middleware/structured_output_retry.py | 70 ------ .../test_structured_output_retry.py | 222 ------------------ 2 files changed, 292 deletions(-) delete mode 100644 libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py delete mode 100644 libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py diff --git a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py b/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py deleted file mode 100644 index 26cdd3bccb8ac..0000000000000 --- a/libs/langchain_v1/langchain/agents/middleware/structured_output_retry.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Minimal structured output retry middleware example.""" - -from collections.abc import Awaitable, Callable - -from langchain_core.messages import HumanMessage - -from langchain.agents.middleware.types import ( - AgentMiddleware, - ModelRequest, - ModelResponse, -) -from langchain.agents.structured_output import StructuredOutputError - - -class StructuredOutputRetryMiddleware(AgentMiddleware): - """Retries model calls when structured output parsing fails.""" - - def __init__(self, max_retries: int = 2) -> None: - """Initialize the structured output retry middleware. - - Args: - max_retries: Maximum number of retry attempts. - """ - self.max_retries = max_retries - - def wrap_model_call( - self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse] - ) -> ModelResponse: - """Intercept and control model execution via handler callback.""" - for attempt in range(self.max_retries + 1): - try: - return handler(request) - except StructuredOutputError as exc: - if attempt == self.max_retries: - raise - - # Include both the AI message and error in a single human message - # to maintain valid chat history alternation - ai_content = exc.ai_message.content - error_message = ( - f"Your previous response was:\n{ai_content}\n\n" - f"Error: {exc}. Please try again with a valid response." - ) - request.messages.append(HumanMessage(content=error_message)) - - # This should never be reached, but satisfies type checker - return handler(request) - - async def awrap_model_call( - self, request: ModelRequest, handler: Callable[[ModelRequest], Awaitable[ModelResponse]] - ) -> ModelResponse: - """Intercept and control async model execution via handler callback.""" - for attempt in range(self.max_retries + 1): - try: - return await handler(request) - except StructuredOutputError as exc: - if attempt == self.max_retries: - raise - - # Include both the AI message and error in a single human message - # to maintain valid chat history alternation - ai_content = exc.ai_message.content - error_message = ( - f"Your previous response was:\n{ai_content}\n\n" - f"Error: {exc}. Please try again with a valid response." - ) - request.messages.append(HumanMessage(content=error_message)) - - # This should never be reached, but satisfies type checker - return await handler(request) diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py deleted file mode 100644 index 749547a19c8c0..0000000000000 --- a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Tests for StructuredOutputRetryMiddleware.""" - -from unittest.mock import MagicMock - -import pytest -from langchain_core.messages import AIMessage, HumanMessage -from pydantic import BaseModel, ValidationError - -from langchain.agents.middleware import ( - ModelRequest, - ModelResponse, - StructuredOutputRetryMiddleware, -) -from langchain.agents.structured_output import ( - MultipleStructuredOutputsError, - StructuredOutputError, - StructuredOutputValidationError, -) - - -class WeatherReport(BaseModel): - """Weather report schema.""" - - temperature: float - conditions: str - - -def test_structured_output_retry_initialization() -> None: - """Test that StructuredOutputRetryMiddleware initializes correctly.""" - middleware = StructuredOutputRetryMiddleware() - assert middleware.max_retries == 2 - - middleware_custom = StructuredOutputRetryMiddleware(max_retries=5) - assert middleware_custom.max_retries == 5 - - -def test_structured_output_retry_success_first_attempt() -> None: - """Test that successful calls on first attempt don't trigger retry.""" - middleware = StructuredOutputRetryMiddleware(max_retries=2) - - # Create mock request and handler - mock_request = MagicMock(spec=ModelRequest) - mock_request.messages = [] - - expected_response = ModelResponse( - result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], - structured_response=WeatherReport(temperature=72.5, conditions="sunny"), - ) - - handler = MagicMock(return_value=expected_response) - - # Execute - result = middleware.wrap_model_call(mock_request, handler) - - # Verify - assert result == expected_response - assert handler.call_count == 1 - - -def test_structured_output_retry_with_validation_error() -> None: - """Test that validation errors trigger retry with error feedback.""" - middleware = StructuredOutputRetryMiddleware(max_retries=2) - - # Create mock request - mock_request = MagicMock(spec=ModelRequest) - mock_request.messages = [] - - # First call fails, second succeeds - ai_msg_with_error = AIMessage(content='{"temperature": "hot", "conditions": "sunny"}') - validation_error = ValidationError.from_exception_data( - "WeatherReport", [{"type": "float_parsing", "loc": ("temperature",), "input": "hot"}] - ) - error = StructuredOutputValidationError("WeatherReport", validation_error, ai_msg_with_error) - - success_response = ModelResponse( - result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], - structured_response=WeatherReport(temperature=72.5, conditions="sunny"), - ) - - handler = MagicMock(side_effect=[error, success_response]) - - # Execute - result = middleware.wrap_model_call(mock_request, handler) - - # Verify - assert result == success_response - assert handler.call_count == 2 - - # Check that error feedback was added to messages - assert len(mock_request.messages) == 1 - assert isinstance(mock_request.messages[0], HumanMessage) - # The human message should contain both the AI's previous response and the error - assert "your previous response" in mock_request.messages[0].content.lower() - assert "error" in mock_request.messages[0].content.lower() - - -def test_structured_output_retry_exhausted() -> None: - """Test that exception is raised when retries are exhausted.""" - middleware = StructuredOutputRetryMiddleware(max_retries=2) - - # Create mock request - mock_request = MagicMock(spec=ModelRequest) - mock_request.messages = [] - - # All calls fail - ai_msg = AIMessage(content='{"temperature": "hot", "conditions": "sunny"}') - validation_error = ValidationError.from_exception_data( - "WeatherReport", [{"type": "float_parsing", "loc": ("temperature",), "input": "hot"}] - ) - error = StructuredOutputValidationError("WeatherReport", validation_error, ai_msg) - - handler = MagicMock(side_effect=error) - - # Execute and verify it raises - with pytest.raises(StructuredOutputValidationError): - middleware.wrap_model_call(mock_request, handler) - - # Verify handler was called 3 times (initial + 2 retries) - assert handler.call_count == 3 - - -def test_structured_output_retry_multiple_outputs_error() -> None: - """Test that MultipleStructuredOutputsError triggers retry.""" - middleware = StructuredOutputRetryMiddleware(max_retries=1) - - # Create mock request - mock_request = MagicMock(spec=ModelRequest) - mock_request.messages = [] - - # First call has multiple outputs error - ai_msg = AIMessage( - content="", - tool_calls=[ - {"name": "WeatherReport", "args": {}, "id": "1"}, - {"name": "WeatherReport", "args": {}, "id": "2"}, - ], - ) - error = MultipleStructuredOutputsError(["WeatherReport", "WeatherReport"], ai_msg) - - # Second call succeeds - success_response = ModelResponse( - result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], - structured_response=WeatherReport(temperature=72.5, conditions="sunny"), - ) - - handler = MagicMock(side_effect=[error, success_response]) - - # Execute - result = middleware.wrap_model_call(mock_request, handler) - - # Verify - assert result == success_response - assert handler.call_count == 2 - - -def test_structured_output_retry_ai_message_preserved() -> None: - """Test that AI message content from exception is embedded in retry message.""" - middleware = StructuredOutputRetryMiddleware(max_retries=1) - - # Create mock request - mock_request = MagicMock(spec=ModelRequest) - mock_request.messages = [] - - # Create specific AI message with error - ai_msg_with_error = AIMessage( - content='{"temperature": "invalid", "conditions": "sunny"}', - id="test-id-123", - ) - error = StructuredOutputValidationError( - "WeatherReport", ValueError("Invalid"), ai_msg_with_error - ) - - success_response = ModelResponse( - result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], - structured_response=WeatherReport(temperature=72.5, conditions="sunny"), - ) - - handler = MagicMock(side_effect=[error, success_response]) - - # Execute - result = middleware.wrap_model_call(mock_request, handler) - - # Verify the AI message content was embedded in the human message - assert len(mock_request.messages) == 1 - assert isinstance(mock_request.messages[0], HumanMessage) - # Check that the AI's previous content is included - assert '{"temperature": "invalid", "conditions": "sunny"}' in mock_request.messages[0].content - - -async def test_structured_output_retry_async() -> None: - """Test async version of retry middleware.""" - middleware = StructuredOutputRetryMiddleware(max_retries=1) - - # Create mock request - mock_request = MagicMock(spec=ModelRequest) - mock_request.messages = [] - - # First call fails - ai_msg = AIMessage(content='{"temperature": "hot"}') - error = StructuredOutputValidationError("WeatherReport", ValueError("Invalid"), ai_msg) - - # Second call succeeds - success_response = ModelResponse( - result=[AIMessage(content='{"temperature": 72.5, "conditions": "sunny"}')], - structured_response=WeatherReport(temperature=72.5, conditions="sunny"), - ) - - async def async_handler(request: ModelRequest) -> ModelResponse: - if not hasattr(async_handler, "call_count"): - async_handler.call_count = 0 # type: ignore[attr-defined] - async_handler.call_count += 1 # type: ignore[attr-defined] - - if async_handler.call_count == 1: # type: ignore[attr-defined] - raise error - return success_response - - # Execute - result = await middleware.awrap_model_call(mock_request, async_handler) - - # Verify - assert result == success_response - assert async_handler.call_count == 2 # type: ignore[attr-defined] From 2bfaf2bd43a55e378358650906b97bff85641733 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:32:13 -0400 Subject: [PATCH 5/8] updates --- libs/langchain_v1/langchain/agents/middleware/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/langchain_v1/langchain/agents/middleware/__init__.py b/libs/langchain_v1/langchain/agents/middleware/__init__.py index 2d8e8912ec12b..8ed35aafcd5c4 100644 --- a/libs/langchain_v1/langchain/agents/middleware/__init__.py +++ b/libs/langchain_v1/langchain/agents/middleware/__init__.py @@ -24,7 +24,6 @@ RedactionRule, ShellToolMiddleware, ) -from .structured_output_retry import StructuredOutputRetryMiddleware from .summarization import SummarizationMiddleware from .todo import TodoListMiddleware from .tool_call_limit import ToolCallLimitMiddleware @@ -66,7 +65,6 @@ "PIIMiddleware", "RedactionRule", "ShellToolMiddleware", - "StructuredOutputRetryMiddleware", "SummarizationMiddleware", "TodoListMiddleware", "ToolCallLimitMiddleware", From 8c2f7f0e9dc9ceae93f73635bc81959e0a8bab70 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:39:55 -0400 Subject: [PATCH 6/8] tester --- .../unit_tests/agents/test_response_format.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py index a7963ced16f57..7df5c23463b36 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_response_format.py @@ -610,6 +610,35 @@ def test_retry_with_custom_string_message(self) -> None: ) assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC + def test_validation_error_with_invalid_response(self) -> None: + """Test that StructuredOutputValidationError is raised when tool strategy receives invalid response.""" + tool_calls = [ + [ + { + "name": "WeatherBaseModel", + "id": "1", + "args": {"invalid_field": "wrong_data", "another_bad_field": 123}, + }, + ], + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + + agent = create_agent( + model, + [], + response_format=ToolStrategy( + WeatherBaseModel, + handle_errors=False, # Disable retry to ensure error is raised + ), + ) + + with pytest.raises( + StructuredOutputValidationError, + match=".*WeatherBaseModel.*", + ): + agent.invoke({"messages": [HumanMessage("What's the weather?")]}) + class TestResponseFormatAsProviderStrategy: def test_pydantic_model(self) -> None: @@ -630,6 +659,28 @@ def test_pydantic_model(self) -> None: assert response["structured_response"] == EXPECTED_WEATHER_PYDANTIC assert len(response["messages"]) == 4 + def test_validation_error_with_invalid_response(self) -> None: + """Test that StructuredOutputValidationError is raised when provider strategy receives invalid response.""" + tool_calls = [ + [{"args": {}, "id": "1", "name": "get_weather"}], + ] + + # But we're using WeatherBaseModel which has different field requirements + model = FakeToolCallingModel[dict]( + tool_calls=tool_calls, + structured_response={"invalid": "data"}, # Wrong structure + ) + + agent = create_agent( + model, [get_weather], response_format=ProviderStrategy(WeatherBaseModel) + ) + + with pytest.raises( + StructuredOutputValidationError, + match=".*WeatherBaseModel.*", + ): + agent.invoke({"messages": [HumanMessage("What's the weather?")]}) + def test_dataclass(self) -> None: """Test response_format as ProviderStrategy with dataclass.""" tool_calls = [ From bd561a212927aeb3ca9a45c9a38945084ed47eb6 Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:53:39 -0400 Subject: [PATCH 7/8] test retry middleware --- .../test_structured_output_retry.py | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py new file mode 100644 index 0000000000000..031c332c9ee89 --- /dev/null +++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py @@ -0,0 +1,297 @@ +"""Tests for StructuredOutputRetryMiddleware functionality.""" + +from collections.abc import Callable + +import pytest +from langchain_core.messages import HumanMessage +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from pydantic import BaseModel + +from langchain.agents import create_agent +from langchain.agents.middleware.types import ( + AgentMiddleware, + ModelRequest, + ModelResponse, +) +from langchain.agents.structured_output import StructuredOutputError, ToolStrategy +from tests.unit_tests.agents.model import FakeToolCallingModel + + +class StructuredOutputRetryMiddleware(AgentMiddleware): + """Retries model calls when structured output parsing fails.""" + + def __init__(self, max_retries: int) -> None: + """Initialize the structured output retry middleware. + + Args: + max_retries: Maximum number of retry attempts. + """ + self.max_retries = max_retries + + def wrap_model_call( + self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse] + ) -> ModelResponse: + """Intercept and control model execution via handler callback. + + Args: + request: The model request containing messages and configuration. + handler: The function to call the model. + + Returns: + The model response. + + Raises: + StructuredOutputError: If max retries exceeded without success. + """ + for attempt in range(self.max_retries + 1): + try: + return handler(request) + except StructuredOutputError as exc: + if attempt == self.max_retries: + raise + + # Include both the AI message and error in a single human message + # to maintain valid chat history alternation + ai_content = exc.ai_message.content + error_message = ( + f"Your previous response was:\n{ai_content}\n\n" + f"Error: {exc}. Please try again with a valid response." + ) + request.messages.append(HumanMessage(content=error_message)) + + # This should never be reached, but satisfies type checker + return handler(request) + + +class WeatherReport(BaseModel): + """Weather report schema for testing.""" + + temperature: float + conditions: str + + +@tool +def get_weather(city: str) -> str: + """Get the weather for a given city. + + Args: + city: The city to get weather for. + + Returns: + Weather information for the city. + """ + return f"The weather in {city} is sunny and 72 degrees." + + +def test_structured_output_retry_first_attempt_invalid() -> None: + """Test structured output retry when first two attempts have invalid output.""" + # First two attempts have invalid tool arguments, third attempt succeeds + # The model will call the WeatherReport structured output tool + tool_calls = [ + # First attempt - invalid: wrong type for temperature + [{"name": "WeatherReport", "id": "1", "args": {"temperature": "not-a-float", "conditions": "sunny"}}], + # Second attempt - invalid: missing required field + [{"name": "WeatherReport", "id": "2", "args": {"temperature": 72.5}}], + # Third attempt - valid + [{"name": "WeatherReport", "id": "3", "args": {"temperature": 72.5, "conditions": "sunny"}}], + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + retry_middleware = StructuredOutputRetryMiddleware(max_retries=2) + + agent = create_agent( + model=model, + tools=[get_weather], + middleware=[retry_middleware], + response_format=ToolStrategy(schema=WeatherReport, handle_errors=False), + checkpointer=InMemorySaver(), + ) + + result = agent.invoke( + {"messages": [HumanMessage("What's the weather in Tokyo?")]}, + {"configurable": {"thread_id": "test"}}, + ) + + # Verify we got a structured response + assert "structured_response" in result + structured = result["structured_response"] + assert isinstance(structured, WeatherReport) + assert structured.temperature == 72.5 + assert structured.conditions == "sunny" + + # Verify the model was called 3 times (initial + 2 retries) + assert model.index == 3 + + +def test_structured_output_retry_exceeds_max_retries() -> None: + """Test structured output retry raises error when max retries exceeded.""" + # All three attempts return invalid arguments + tool_calls = [ + [{"name": "WeatherReport", "id": "1", "args": {"temperature": "invalid", "conditions": "sunny"}}], + [{"name": "WeatherReport", "id": "2", "args": {"temperature": "also-invalid", "conditions": "cloudy"}}], + [{"name": "WeatherReport", "id": "3", "args": {"temperature": "still-invalid", "conditions": "rainy"}}], + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + retry_middleware = StructuredOutputRetryMiddleware(max_retries=2) + + agent = create_agent( + model=model, + tools=[get_weather], + middleware=[retry_middleware], + response_format=ToolStrategy(schema=WeatherReport, handle_errors=False), + # No checkpointer - we expect this to fail + ) + + # Should raise StructuredOutputError after exhausting retries + with pytest.raises(StructuredOutputError): + agent.invoke( + {"messages": [HumanMessage("What's the weather in Tokyo?")]}, + ) + + # Verify the model was called 3 times (initial + 2 retries) + assert model.index == 3 + + +def test_structured_output_retry_succeeds_first_attempt() -> None: + """Test structured output retry when first attempt succeeds (no retry needed).""" + # First attempt returns valid structured output + tool_calls = [ + [{"name": "WeatherReport", "id": "1", "args": {"temperature": 68.0, "conditions": "cloudy"}}], + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + retry_middleware = StructuredOutputRetryMiddleware(max_retries=2) + + agent = create_agent( + model=model, + tools=[get_weather], + middleware=[retry_middleware], + response_format=ToolStrategy(schema=WeatherReport, handle_errors=False), + checkpointer=InMemorySaver(), + ) + + result = agent.invoke( + {"messages": [HumanMessage("What's the weather in Paris?")]}, + {"configurable": {"thread_id": "test"}}, + ) + + # Verify we got a structured response + assert "structured_response" in result + structured = result["structured_response"] + assert isinstance(structured, WeatherReport) + assert structured.temperature == 68.0 + assert structured.conditions == "cloudy" + + # Verify the model was called only once + assert model.index == 1 + + +def test_structured_output_retry_validation_error() -> None: + """Test structured output retry with schema validation errors.""" + # First attempt has wrong type, second has missing field, third succeeds + tool_calls = [ + [{"name": "WeatherReport", "id": "1", "args": {"temperature": "seventy-two", "conditions": "sunny"}}], + [{"name": "WeatherReport", "id": "2", "args": {"temperature": 72.5}}], + [{"name": "WeatherReport", "id": "3", "args": {"temperature": 72.5, "conditions": "partly cloudy"}}], + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + retry_middleware = StructuredOutputRetryMiddleware(max_retries=2) + + agent = create_agent( + model=model, + tools=[get_weather], + middleware=[retry_middleware], + response_format=ToolStrategy(schema=WeatherReport, handle_errors=False), + checkpointer=InMemorySaver(), + ) + + result = agent.invoke( + {"messages": [HumanMessage("What's the weather in London?")]}, + {"configurable": {"thread_id": "test"}}, + ) + + # Verify we got a structured response + assert "structured_response" in result + structured = result["structured_response"] + assert isinstance(structured, WeatherReport) + assert structured.temperature == 72.5 + assert structured.conditions == "partly cloudy" + + # Verify the model was called 3 times + assert model.index == 3 + + +def test_structured_output_retry_zero_retries() -> None: + """Test structured output retry with max_retries=0 (no retries allowed).""" + # First attempt returns invalid arguments + tool_calls = [ + [{"name": "WeatherReport", "id": "1", "args": {"temperature": "invalid", "conditions": "sunny"}}], + [{"name": "WeatherReport", "id": "2", "args": {"temperature": 72.5, "conditions": "sunny"}}], # Would succeed if retried + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + retry_middleware = StructuredOutputRetryMiddleware(max_retries=0) + + agent = create_agent( + model=model, + tools=[get_weather], + middleware=[retry_middleware], + response_format=ToolStrategy(schema=WeatherReport, handle_errors=False), + checkpointer=InMemorySaver(), + ) + + # Should fail immediately without retrying + with pytest.raises(StructuredOutputError): + agent.invoke( + {"messages": [HumanMessage("What's the weather in Berlin?")]}, + {"configurable": {"thread_id": "test"}}, + ) + + # Verify the model was called only once (no retries) + assert model.index == 1 + + +def test_structured_output_retry_preserves_messages() -> None: + """Test structured output retry preserves error feedback in messages.""" + # First attempt invalid, second succeeds + tool_calls = [ + [{"name": "WeatherReport", "id": "1", "args": {"temperature": "invalid", "conditions": "rainy"}}], + [{"name": "WeatherReport", "id": "2", "args": {"temperature": 75.0, "conditions": "rainy"}}], + ] + + model = FakeToolCallingModel(tool_calls=tool_calls) + retry_middleware = StructuredOutputRetryMiddleware(max_retries=1) + + agent = create_agent( + model=model, + tools=[get_weather], + middleware=[retry_middleware], + response_format=ToolStrategy(schema=WeatherReport, handle_errors=False), + checkpointer=InMemorySaver(), + ) + + result = agent.invoke( + {"messages": [HumanMessage("What's the weather in Seattle?")]}, + {"configurable": {"thread_id": "test"}}, + ) + + # Verify structured response is correct + assert "structured_response" in result + structured = result["structured_response"] + assert structured.temperature == 75.0 + assert structured.conditions == "rainy" + + # Verify messages include the retry feedback + messages = result["messages"] + human_messages = [m for m in messages if isinstance(m, HumanMessage)] + + # Should have at least 2 human messages: initial + retry feedback + assert len(human_messages) >= 2 + + # The retry feedback message should contain error information + retry_message = human_messages[-1] + assert "Error:" in retry_message.content + assert "Please try again" in retry_message.content From 1312cf330f990295c2a8d4bc6881a41032b7d95d Mon Sep 17 00:00:00 2001 From: Sydney Runkle Date: Fri, 24 Oct 2025 13:56:42 -0400 Subject: [PATCH 8/8] linting and formatting --- .../test_structured_output_retry.py | 96 ++++++++++++++++--- 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py index 031c332c9ee89..a04f670ad4a06 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py +++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/test_structured_output_retry.py @@ -90,11 +90,23 @@ def test_structured_output_retry_first_attempt_invalid() -> None: # The model will call the WeatherReport structured output tool tool_calls = [ # First attempt - invalid: wrong type for temperature - [{"name": "WeatherReport", "id": "1", "args": {"temperature": "not-a-float", "conditions": "sunny"}}], + [ + { + "name": "WeatherReport", + "id": "1", + "args": {"temperature": "not-a-float", "conditions": "sunny"}, + } + ], # Second attempt - invalid: missing required field [{"name": "WeatherReport", "id": "2", "args": {"temperature": 72.5}}], # Third attempt - valid - [{"name": "WeatherReport", "id": "3", "args": {"temperature": 72.5, "conditions": "sunny"}}], + [ + { + "name": "WeatherReport", + "id": "3", + "args": {"temperature": 72.5, "conditions": "sunny"}, + } + ], ] model = FakeToolCallingModel(tool_calls=tool_calls) @@ -128,9 +140,27 @@ def test_structured_output_retry_exceeds_max_retries() -> None: """Test structured output retry raises error when max retries exceeded.""" # All three attempts return invalid arguments tool_calls = [ - [{"name": "WeatherReport", "id": "1", "args": {"temperature": "invalid", "conditions": "sunny"}}], - [{"name": "WeatherReport", "id": "2", "args": {"temperature": "also-invalid", "conditions": "cloudy"}}], - [{"name": "WeatherReport", "id": "3", "args": {"temperature": "still-invalid", "conditions": "rainy"}}], + [ + { + "name": "WeatherReport", + "id": "1", + "args": {"temperature": "invalid", "conditions": "sunny"}, + } + ], + [ + { + "name": "WeatherReport", + "id": "2", + "args": {"temperature": "also-invalid", "conditions": "cloudy"}, + } + ], + [ + { + "name": "WeatherReport", + "id": "3", + "args": {"temperature": "still-invalid", "conditions": "rainy"}, + } + ], ] model = FakeToolCallingModel(tool_calls=tool_calls) @@ -158,7 +188,13 @@ def test_structured_output_retry_succeeds_first_attempt() -> None: """Test structured output retry when first attempt succeeds (no retry needed).""" # First attempt returns valid structured output tool_calls = [ - [{"name": "WeatherReport", "id": "1", "args": {"temperature": 68.0, "conditions": "cloudy"}}], + [ + { + "name": "WeatherReport", + "id": "1", + "args": {"temperature": 68.0, "conditions": "cloudy"}, + } + ], ] model = FakeToolCallingModel(tool_calls=tool_calls) @@ -192,9 +228,21 @@ def test_structured_output_retry_validation_error() -> None: """Test structured output retry with schema validation errors.""" # First attempt has wrong type, second has missing field, third succeeds tool_calls = [ - [{"name": "WeatherReport", "id": "1", "args": {"temperature": "seventy-two", "conditions": "sunny"}}], + [ + { + "name": "WeatherReport", + "id": "1", + "args": {"temperature": "seventy-two", "conditions": "sunny"}, + } + ], [{"name": "WeatherReport", "id": "2", "args": {"temperature": 72.5}}], - [{"name": "WeatherReport", "id": "3", "args": {"temperature": 72.5, "conditions": "partly cloudy"}}], + [ + { + "name": "WeatherReport", + "id": "3", + "args": {"temperature": 72.5, "conditions": "partly cloudy"}, + } + ], ] model = FakeToolCallingModel(tool_calls=tool_calls) @@ -228,8 +276,20 @@ def test_structured_output_retry_zero_retries() -> None: """Test structured output retry with max_retries=0 (no retries allowed).""" # First attempt returns invalid arguments tool_calls = [ - [{"name": "WeatherReport", "id": "1", "args": {"temperature": "invalid", "conditions": "sunny"}}], - [{"name": "WeatherReport", "id": "2", "args": {"temperature": 72.5, "conditions": "sunny"}}], # Would succeed if retried + [ + { + "name": "WeatherReport", + "id": "1", + "args": {"temperature": "invalid", "conditions": "sunny"}, + } + ], + [ + { + "name": "WeatherReport", + "id": "2", + "args": {"temperature": 72.5, "conditions": "sunny"}, + } + ], # Would succeed if retried ] model = FakeToolCallingModel(tool_calls=tool_calls) @@ -258,8 +318,20 @@ def test_structured_output_retry_preserves_messages() -> None: """Test structured output retry preserves error feedback in messages.""" # First attempt invalid, second succeeds tool_calls = [ - [{"name": "WeatherReport", "id": "1", "args": {"temperature": "invalid", "conditions": "rainy"}}], - [{"name": "WeatherReport", "id": "2", "args": {"temperature": 75.0, "conditions": "rainy"}}], + [ + { + "name": "WeatherReport", + "id": "1", + "args": {"temperature": "invalid", "conditions": "rainy"}, + } + ], + [ + { + "name": "WeatherReport", + "id": "2", + "args": {"temperature": 75.0, "conditions": "rainy"}, + } + ], ] model = FakeToolCallingModel(tool_calls=tool_calls)