Added unit tests and fixed null expected retrieve data

Author: Am1n3e

.github/workflows/lint.yml

@@ -3,8 +3,6 @@ name: Lint
 on:
   pull_request:
     branches: [main]
-  push:
-    branches: [main]
 
 jobs:
   lint:

.github/workflows/test-env-ctrl.yml

@@ -7,12 +7,6 @@ on:
       - "packages/environment_control/**"
       - "tests/integration/environment_control/**"
       - "src/webarena_verified/environments/env_ctrl_client/**"
-  push:
-    branches: [main]
-    paths:
-      - "packages/environment_control/**"
-      - "tests/integration/environment_control/**"
-      - "src/webarena_verified/environments/env_ctrl_client/**"
 
 jobs:
   test:

src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py

@@ -96,6 +96,10 @@ def _normalized_actual_value(
                 k = "performed_operation"  # Support legacy field name
 
             if k not in value:
+                if k == "retrieved_data":
+                    # When retrieved_data key is missing, treat it as None for comparison
+                    # This allows missing key to match expected None (e.g., NOT_FOUND_ERROR tasks)
+                    _normalized_values[k] = None
                 continue
 
             if k == "retrieved_data":
@@ -179,11 +183,22 @@ def _compare_values(  # type: ignore[override]
             return assertions
 
         expected_retrieved_data = expected_normalized.get("retrieved_data", None)
-        if expected_retrieved_data is None:
-            raise ValueError("Expected retrieved_data must be set in config for retrieve tasks.")
 
-        # Handle None actual_retrieved_data - should fail if expected is not None
-        if actual_retrieved_data is None and expected_retrieved_data is not None:
+        if expected_retrieved_data is None and actual_retrieved_data is None:
+            # Both None - success, no data expected and none provided
+            return assertions
+        elif expected_retrieved_data is None and actual_retrieved_data is not None:
+            # Expected None but got data - failure
+            assertions.append(
+                EvalAssertion.create(
+                    assertion_name="retrieved_data_unexpected",
+                    status=EvalStatus.FAILURE,
+                    assertion_msgs=[f"Expected no retrieved_data, but got {actual_retrieved_data}"],
+                )
+            )
+            return assertions
+        elif actual_retrieved_data is None:
+            # Expected data but got None - failure
             assertions.append(
                 EvalAssertion.create(
                     assertion_name="retrieved_data_missing_or_null",
@@ -192,14 +207,15 @@ def _compare_values(  # type: ignore[override]
                 )
             )
             return assertions
-
-        assertions.extend(
-            self.value_comparator.compare(
-                expected=tuple(expected_retrieved_data),
-                actual=tuple(actual_retrieved_data),
-                value_name="retrieved_data",
-                ordered=config.ordered,
+        else:
+            # Both have data - compare them
+            assertions.extend(
+                self.value_comparator.compare(
+                    expected=tuple(expected_retrieved_data),
+                    actual=tuple(actual_retrieved_data),
+                    value_name="retrieved_data",
+                    ordered=config.ordered,
+                )
             )
-        )
 
         return assertions

tests/api/__init__.py

@@ -0,0 +1 @@
+"""Tests for webarena_verified.api module."""