fix(MBT): update auto gen mbt evals

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluator.py

@@ -18,6 +18,12 @@
 def _validate_evaluator_input(slug: str, input: Dict[str, str]) -> None:
     """Validate input against the evaluator's request model if available.
 
+    The request model has a nested structure:
+    - input: Contains the actual input fields (required)
+    - config: Contains optional configuration (optional)
+
+    The flat input dict is wrapped in the nested structure for validation.
+
     Args:
         slug: The evaluator slug (e.g., "pii-detector")
         input: Dictionary of input field names to values
@@ -28,7 +34,9 @@ def _validate_evaluator_input(slug: str, input: Dict[str, str]) -> None:
     request_model = get_request_model(slug)
     if request_model:
         try:
-            request_model(**input)
+            # The request model expects nested structure: {input: {...}, config: {...}}
+            # Wrap the flat input dict in the nested 'input' field
+            request_model(input=input)
         except ValidationError as e:
             raise ValueError(f"Invalid input for '{slug}': {e}") from e
 

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluators_made_by_traceloop.py

@@ -11,22 +11,60 @@
 
 
 def _get_required_fields(slug: str) -> List[str]:
-    """Get required input fields for an evaluator from its request model."""
+    """Get required input fields for an evaluator from its request model.
+
+    The request model has a nested `input` field containing the actual input fields.
+    """
     model = REQUEST_MODELS.get(slug)
     if not model:
         return []
-    return [name for name, field in model.model_fields.items() if field.is_required()]
+
+    # Get the 'input' field's annotation (the nested Input model)
+    input_field = model.model_fields.get("input")
+    if not input_field or not input_field.annotation:
+        return []
+
+    input_model = input_field.annotation
+    if not hasattr(input_model, "model_fields"):
+        return []
+
+    return [
+        name for name, field in input_model.model_fields.items()
+        if field.is_required()
+    ]
 
 
 def _get_config_fields(slug: str) -> dict:
-    """Get config fields (non-required) with their defaults from the request model."""
+    """Get config fields with their defaults from the request model.
+
+    The request model has a nested optional `config` field containing config options.
+    """
     model = REQUEST_MODELS.get(slug)
     if not model:
         return {}
+
+    # Get the 'config' field's annotation (the nested ConfigRequest model)
+    config_field = model.model_fields.get("config")
+    if not config_field or not config_field.annotation:
+        return {}
+
+    # Handle Optional[ConfigModel] - extract the inner type
+    config_annotation = config_field.annotation
+    origin = getattr(config_annotation, "__origin__", None)
+    if origin is not None:
+        # It's a generic type like Optional[X] = Union[X, None]
+        args = getattr(config_annotation, "__args__", ())
+        # Get the non-None type from Union
+        config_model = next((a for a in args if a is not type(None)), None)
+    else:
+        config_model = config_annotation
+
+    if not config_model or not hasattr(config_model, "model_fields"):
+        return {}
+
     config_fields = {}
-    for name, field in model.model_fields.items():
-        if not field.is_required():
-            config_fields[name] = field.default
+    for name, field in config_model.model_fields.items():
+        config_fields[name] = field.default
     return config_fields
 
 

packages/traceloop-sdk/traceloop/sdk/generated/evaluators/__init__.py

@@ -5,37 +5,82 @@
 #   ./scripts/generate-models.sh /path/to/swagger.json
 
 from .request import (
+    AgentEfficiencyInput,
     AgentEfficiencyRequest,
+    AgentFlowQualityConfigRequest,
+    AgentFlowQualityInput,
     AgentFlowQualityRequest,
+    AgentGoalAccuracyInput,
     AgentGoalAccuracyRequest,
+    AgentGoalCompletenessConfigRequest,
+    AgentGoalCompletenessInput,
     AgentGoalCompletenessRequest,
+    AgentToolErrorDetectorInput,
     AgentToolErrorDetectorRequest,
+    AnswerCompletenessInput,
     AnswerCompletenessRequest,
+    AnswerCorrectnessInput,
     AnswerCorrectnessRequest,
+    AnswerRelevancyInput,
     AnswerRelevancyRequest,
+    CharCountInput,
+    CharCountRatioInput,
     CharCountRatioRequest,
     CharCountRequest,
+    ContextRelevanceConfigRequest,
+    ContextRelevanceInput,
     ContextRelevanceRequest,
+    ConversationQualityConfigRequest,
+    ConversationQualityInput,
     ConversationQualityRequest,
+    FaithfulnessInput,
     FaithfulnessRequest,
+    InstructionAdherenceInput,
     InstructionAdherenceRequest,
+    IntentChangeConfigRequest,
+    IntentChangeInput,
     IntentChangeRequest,
+    JSONValidatorConfigRequest,
+    JSONValidatorInput,
     JSONValidatorRequest,
+    PIIDetectorConfigRequest,
+    PIIDetectorInput,
     PIIDetectorRequest,
+    PerplexityInput,
     PerplexityRequest,
+    PlaceholderRegexConfigRequest,
+    PlaceholderRegexInput,
     PlaceholderRegexRequest,
+    ProfanityDetectorInput,
     ProfanityDetectorRequest,
+    PromptInjectionConfigRequest,
+    PromptInjectionInput,
     PromptInjectionRequest,
+    PromptPerplexityInput,
     PromptPerplexityRequest,
+    RegexValidatorConfigRequest,
+    RegexValidatorInput,
     RegexValidatorRequest,
+    SQLValidatorInput,
     SQLValidatorRequest,
+    SecretsDetectorInput,
     SecretsDetectorRequest,
+    SemanticSimilarityInput,
     SemanticSimilarityRequest,
+    SexismDetectorConfigRequest,
+    SexismDetectorInput,
     SexismDetectorRequest,
+    ToneDetectionInput,
     ToneDetectionRequest,
+    TopicAdherenceInput,
     TopicAdherenceRequest,
+    ToxicityDetectorConfigRequest,
+    ToxicityDetectorInput,
     ToxicityDetectorRequest,
+    UncertaintyDetectorInput,
     UncertaintyDetectorRequest,
+    WordCountInput,
+    WordCountRatioInput,
     WordCountRatioRequest,
     WordCountRequest,
 )
@@ -91,37 +136,82 @@
     "get_request_model",
     "get_response_model",
     # Evaluator request models
+    "AgentEfficiencyInput",
     "AgentEfficiencyRequest",
+    "AgentFlowQualityConfigRequest",
+    "AgentFlowQualityInput",
     "AgentFlowQualityRequest",
+    "AgentGoalAccuracyInput",
     "AgentGoalAccuracyRequest",
+    "AgentGoalCompletenessConfigRequest",
+    "AgentGoalCompletenessInput",
     "AgentGoalCompletenessRequest",
+    "AgentToolErrorDetectorInput",
     "AgentToolErrorDetectorRequest",
+    "AnswerCompletenessInput",
     "AnswerCompletenessRequest",
+    "AnswerCorrectnessInput",
     "AnswerCorrectnessRequest",
+    "AnswerRelevancyInput",
     "AnswerRelevancyRequest",
+    "CharCountInput",
+    "CharCountRatioInput",
     "CharCountRatioRequest",
     "CharCountRequest",
+    "ContextRelevanceConfigRequest",
+    "ContextRelevanceInput",
     "ContextRelevanceRequest",
+    "ConversationQualityConfigRequest",
+    "ConversationQualityInput",
     "ConversationQualityRequest",
+    "FaithfulnessInput",
     "FaithfulnessRequest",
+    "InstructionAdherenceInput",
     "InstructionAdherenceRequest",
+    "IntentChangeConfigRequest",
+    "IntentChangeInput",
     "IntentChangeRequest",
+    "JSONValidatorConfigRequest",
+    "JSONValidatorInput",
     "JSONValidatorRequest",
+    "PIIDetectorConfigRequest",
+    "PIIDetectorInput",
     "PIIDetectorRequest",
+    "PerplexityInput",
     "PerplexityRequest",
+    "PlaceholderRegexConfigRequest",
+    "PlaceholderRegexInput",
     "PlaceholderRegexRequest",
+    "ProfanityDetectorInput",
     "ProfanityDetectorRequest",
+    "PromptInjectionConfigRequest",
+    "PromptInjectionInput",
     "PromptInjectionRequest",
+    "PromptPerplexityInput",
     "PromptPerplexityRequest",
+    "RegexValidatorConfigRequest",
+    "RegexValidatorInput",
     "RegexValidatorRequest",
+    "SQLValidatorInput",
     "SQLValidatorRequest",
+    "SecretsDetectorInput",
     "SecretsDetectorRequest",
+    "SemanticSimilarityInput",
     "SemanticSimilarityRequest",
+    "SexismDetectorConfigRequest",
+    "SexismDetectorInput",
     "SexismDetectorRequest",
+    "ToneDetectionInput",
     "ToneDetectionRequest",
+    "TopicAdherenceInput",
     "TopicAdherenceRequest",
+    "ToxicityDetectorConfigRequest",
+    "ToxicityDetectorInput",
     "ToxicityDetectorRequest",
+    "UncertaintyDetectorInput",
     "UncertaintyDetectorRequest",
+    "WordCountInput",
+    "WordCountRatioInput",
     "WordCountRatioRequest",
     "WordCountRequest",
     # Evaluator response models