fix(MBT): update auto gen mbt evals

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluator.py

@@ -16,19 +16,11 @@
 
 
 def _validate_evaluator_input(slug: str, input: Dict[str, str]) -> None:
-    """Validate input against the evaluator's request model if available.
-
-    Args:
-        slug: The evaluator slug (e.g., "pii-detector")
-        input: Dictionary of input field names to values
-
-    Raises:
-        ValueError: If input fails validation against the request model
-    """
+    """Validate input against the evaluator's request model if available."""
     request_model = get_request_model(slug)
     if request_model:
         try:
-            request_model(**input)
+            request_model(input=input)
         except ValidationError as e:
             raise ValueError(f"Invalid input for '{slug}': {e}") from e
 

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluators_made_by_traceloop.py

@@ -15,18 +15,45 @@ def _get_required_fields(slug: str) -> List[str]:
     model = REQUEST_MODELS.get(slug)
     if not model:
         return []
-    return [name for name, field in model.model_fields.items() if field.is_required()]
+
+    input_field = model.model_fields.get("input")
+    if not input_field or not input_field.annotation:
+        return []
+
+    input_model = input_field.annotation
+    if not hasattr(input_model, "model_fields"):
+        return []
+
+    return [
+        name for name, field in input_model.model_fields.items()
+        if field.is_required()
+    ]
 
 
 def _get_config_fields(slug: str) -> dict:
-    """Get config fields (non-required) with their defaults from the request model."""
+    """Get config fields with their defaults from the request model."""
     model = REQUEST_MODELS.get(slug)
     if not model:
         return {}
+
+    config_field = model.model_fields.get("config")
+    if not config_field or not config_field.annotation:
+        return {}
+
+    config_annotation = config_field.annotation
+    origin = getattr(config_annotation, "__origin__", None)
+    if origin is not None:
+        args = getattr(config_annotation, "__args__", ())
+        config_model = next((a for a in args if a is not type(None)), None)
+    else:
+        config_model = config_annotation
+
+    if not config_model or not hasattr(config_model, "model_fields"):
+        return {}
+
     config_fields = {}
-    for name, field in model.model_fields.items():
-        if not field.is_required():
-            config_fields[name] = field.default
+    for name, field in config_model.model_fields.items():
+        config_fields[name] = field.default
     return config_fields
 
 

packages/traceloop-sdk/traceloop/sdk/generated/evaluators/__init__.py

@@ -5,37 +5,82 @@
 #   ./scripts/generate-models.sh /path/to/swagger.json
 
 from .request import (
+    AgentEfficiencyInput,
     AgentEfficiencyRequest,
+    AgentFlowQualityConfigRequest,
+    AgentFlowQualityInput,
     AgentFlowQualityRequest,
+    AgentGoalAccuracyInput,
     AgentGoalAccuracyRequest,
+    AgentGoalCompletenessConfigRequest,
+    AgentGoalCompletenessInput,
     AgentGoalCompletenessRequest,
+    AgentToolErrorDetectorInput,
     AgentToolErrorDetectorRequest,
+    AnswerCompletenessInput,
     AnswerCompletenessRequest,
+    AnswerCorrectnessInput,
     AnswerCorrectnessRequest,
+    AnswerRelevancyInput,
     AnswerRelevancyRequest,
+    CharCountInput,
+    CharCountRatioInput,
     CharCountRatioRequest,
     CharCountRequest,
+    ContextRelevanceConfigRequest,
+    ContextRelevanceInput,
     ContextRelevanceRequest,
+    ConversationQualityConfigRequest,
+    ConversationQualityInput,
     ConversationQualityRequest,
+    FaithfulnessInput,
     FaithfulnessRequest,
+    InstructionAdherenceInput,
     InstructionAdherenceRequest,
+    IntentChangeConfigRequest,
+    IntentChangeInput,
     IntentChangeRequest,
+    JSONValidatorConfigRequest,
+    JSONValidatorInput,
     JSONValidatorRequest,
+    PIIDetectorConfigRequest,
+    PIIDetectorInput,
     PIIDetectorRequest,
+    PerplexityInput,
     PerplexityRequest,
+    PlaceholderRegexConfigRequest,
+    PlaceholderRegexInput,
     PlaceholderRegexRequest,
+    ProfanityDetectorInput,
     ProfanityDetectorRequest,
+    PromptInjectionConfigRequest,
+    PromptInjectionInput,
     PromptInjectionRequest,
+    PromptPerplexityInput,
     PromptPerplexityRequest,
+    RegexValidatorConfigRequest,
+    RegexValidatorInput,
     RegexValidatorRequest,
+    SQLValidatorInput,
     SQLValidatorRequest,
+    SecretsDetectorInput,
     SecretsDetectorRequest,
+    SemanticSimilarityInput,
     SemanticSimilarityRequest,
+    SexismDetectorConfigRequest,
+    SexismDetectorInput,
     SexismDetectorRequest,
+    ToneDetectionInput,
     ToneDetectionRequest,
+    TopicAdherenceInput,
     TopicAdherenceRequest,
+    ToxicityDetectorConfigRequest,
+    ToxicityDetectorInput,
     ToxicityDetectorRequest,
+    UncertaintyDetectorInput,
     UncertaintyDetectorRequest,
+    WordCountInput,
+    WordCountRatioInput,
     WordCountRatioRequest,
     WordCountRequest,
 )
@@ -91,37 +136,82 @@
     "get_request_model",
     "get_response_model",
     # Evaluator request models
+    "AgentEfficiencyInput",
     "AgentEfficiencyRequest",
+    "AgentFlowQualityConfigRequest",
+    "AgentFlowQualityInput",
     "AgentFlowQualityRequest",
+    "AgentGoalAccuracyInput",
     "AgentGoalAccuracyRequest",
+    "AgentGoalCompletenessConfigRequest",
+    "AgentGoalCompletenessInput",
     "AgentGoalCompletenessRequest",
+    "AgentToolErrorDetectorInput",
     "AgentToolErrorDetectorRequest",
+    "AnswerCompletenessInput",
     "AnswerCompletenessRequest",
+    "AnswerCorrectnessInput",
     "AnswerCorrectnessRequest",
+    "AnswerRelevancyInput",
     "AnswerRelevancyRequest",
+    "CharCountInput",
+    "CharCountRatioInput",
     "CharCountRatioRequest",
     "CharCountRequest",
+    "ContextRelevanceConfigRequest",
+    "ContextRelevanceInput",
     "ContextRelevanceRequest",
+    "ConversationQualityConfigRequest",
+    "ConversationQualityInput",
     "ConversationQualityRequest",
+    "FaithfulnessInput",
     "FaithfulnessRequest",
+    "InstructionAdherenceInput",
     "InstructionAdherenceRequest",
+    "IntentChangeConfigRequest",
+    "IntentChangeInput",
     "IntentChangeRequest",
+    "JSONValidatorConfigRequest",
+    "JSONValidatorInput",
     "JSONValidatorRequest",
+    "PIIDetectorConfigRequest",
+    "PIIDetectorInput",
     "PIIDetectorRequest",
+    "PerplexityInput",
     "PerplexityRequest",
+    "PlaceholderRegexConfigRequest",
+    "PlaceholderRegexInput",
     "PlaceholderRegexRequest",
+    "ProfanityDetectorInput",
     "ProfanityDetectorRequest",
+    "PromptInjectionConfigRequest",
+    "PromptInjectionInput",
     "PromptInjectionRequest",
+    "PromptPerplexityInput",
     "PromptPerplexityRequest",
+    "RegexValidatorConfigRequest",
+    "RegexValidatorInput",
     "RegexValidatorRequest",
+    "SQLValidatorInput",
     "SQLValidatorRequest",
+    "SecretsDetectorInput",
     "SecretsDetectorRequest",
+    "SemanticSimilarityInput",
     "SemanticSimilarityRequest",
+    "SexismDetectorConfigRequest",
+    "SexismDetectorInput",
     "SexismDetectorRequest",
+    "ToneDetectionInput",
     "ToneDetectionRequest",
+    "TopicAdherenceInput",
     "TopicAdherenceRequest",
+    "ToxicityDetectorConfigRequest",
+    "ToxicityDetectorInput",
     "ToxicityDetectorRequest",
+    "UncertaintyDetectorInput",
     "UncertaintyDetectorRequest",
+    "WordCountInput",
+    "WordCountRatioInput",
     "WordCountRatioRequest",
     "WordCountRequest",
     # Evaluator response models