fix tests

Co-authored-by: Yahya Mouman <[email protected]>

Author: gary-huang

ddtrace/llmobs/_experiment.py

@@ -142,7 +142,9 @@ def push(self) -> None:
 
     def update(self, index: int, record: DatasetRecordRaw) -> None:
         if "input_data" not in record and "expected_output" not in record and "metadata" not in record:
-            raise ValueError("invalid update, at least one of input_data, expected_output, or metadata must be updated")
+            raise ValueError(
+                "invalid update, record should contain at least one of input_data, expected_output, or metadata to update"
+            )
         record_id = self._records[index]["record_id"]
         self._updated_record_ids.append(record_id)
         self._records[index] = {**record, "record_id": record_id}

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_c1129a9c-35c8-41fb-acbb-b01b72e672b3_batch_update_post_7577c52c.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_02299014-fa9b-409d-af64-266f9b077c5d_batch_update_post_6bb82ff6.yaml

@@ -1,8 +1,8 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "c1129a9c-35c8-41fb-acbb-b01b72e672b3",
+    body: '{"data": {"type": "datasets", "id": "02299014-fa9b-409d-af64-266f9b077c5d",
       "attributes": {"insert_records": [{"input": {"prompt": "What is the capital
-      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": {}}], "update_records":
+      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": null}], "update_records":
       [], "delete_records": []}}}'
     headers:
       Accept:
@@ -13,27 +13,27 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '269'
+      - '271'
       ? !!python/object/apply:multidict._multidict.istr
       - Content-Type
       : - application/json
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/c1129a9c-35c8-41fb-acbb-b01b72e672b3/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/02299014-fa9b-409d-af64-266f9b077c5d/batch_update
   response:
     body:
-      string: '{"data":[{"id":"106302a9-678e-4465-9a54-9c158cdd55b0","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T03:00:20.452940856Z","dataset_id":"c1129a9c-35c8-41fb-acbb-b01b72e672b3","expected_output":{"answer":"Paris"},"input":{"prompt":"What
-        is the capital of France?"},"metadata":{},"updated_at":"2025-08-06T03:00:20.452940856Z","version":1}}]}'
+      string: '{"data":[{"id":"2468b370-ff5f-40fd-ace3-497468c32f52","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:20:22.123409391Z","dataset_id":"02299014-fa9b-409d-af64-266f9b077c5d","expected_output":{"answer":"Paris"},"input":{"prompt":"What
+        is the capital of France?"},"updated_at":"2025-08-06T16:20:22.123409391Z","version":1}}]}'
     headers:
       content-length:
-      - '403'
+      - '389'
       content-security-policy:
       - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 06 Aug 2025 03:00:20 GMT
+      - Wed, 06 Aug 2025 16:20:22 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary:

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_c1129a9c-35c8-41fb-acbb-b01b72e672b3_batch_update_post_4eb969d0.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_02299014-fa9b-409d-af64-266f9b077c5d_batch_update_post_9369a307.yaml

@@ -1,8 +1,8 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "c1129a9c-35c8-41fb-acbb-b01b72e672b3",
+    body: '{"data": {"type": "datasets", "id": "02299014-fa9b-409d-af64-266f9b077c5d",
       "attributes": {"insert_records": [], "update_records": [{"input": null, "expected_output":
-      "", "metadata": {}, "id": "106302a9-678e-4465-9a54-9c158cdd55b0"}], "delete_records":
+      "", "metadata": null, "id": "2468b370-ff5f-40fd-ace3-497468c32f52"}], "delete_records":
       []}}}'
     headers:
       Accept:
@@ -13,27 +13,27 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '258'
+      - '260'
       ? !!python/object/apply:multidict._multidict.istr
       - Content-Type
       : - application/json
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/c1129a9c-35c8-41fb-acbb-b01b72e672b3/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/02299014-fa9b-409d-af64-266f9b077c5d/batch_update
   response:
     body:
-      string: '{"data":[{"id":"784ff4b0-dc28-4766-8e20-33d3b9c122de","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T03:00:23.232545193Z","dataset_id":"c1129a9c-35c8-41fb-acbb-b01b72e672b3","expected_output":"","input":{"prompt":"What
-        is the capital of France?"},"metadata":{},"updated_at":"2025-08-06T03:00:23.232545266Z","version":2}}]}'
+      string: '{"data":[{"id":"85c4c2fa-d901-4d11-a7a7-b868bb77deaa","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:20:24.804599279Z","dataset_id":"02299014-fa9b-409d-af64-266f9b077c5d","expected_output":"","input":{"prompt":"What
+        is the capital of France?"},"updated_at":"2025-08-06T16:20:24.804599378Z","version":2}}]}'
     headers:
       content-length:
-      - '387'
+      - '373'
       content-security-policy:
       - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 06 Aug 2025 03:00:23 GMT
+      - Wed, 06 Aug 2025 16:20:24 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary:

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_2f376001-f964-4f1e-897d-4585ccff39ea_records_get_15f721f9.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_02299014-fa9b-409d-af64-266f9b077c5d_records_get_8e8ecd31.yaml

@@ -18,20 +18,20 @@ interactions:
       User-Agent:
       - python-requests/2.32.3
     method: GET
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/2f376001-f964-4f1e-897d-4585ccff39ea/records
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/02299014-fa9b-409d-af64-266f9b077c5d/records
   response:
     body:
-      string: '{"data":[{"id":"8d55ad5a-528e-46c4-bd58-76e3d2ea4ddf","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T03:00:50.046982Z","dataset_id":"2f376001-f964-4f1e-897d-4585ccff39ea","expected_output":"","input":{"prompt":"What
-        is the capital of Nauru?"},"metadata":{},"updated_at":"2025-08-06T03:00:50.046982Z"}}],"meta":{"after":""}}'
+      string: '{"data":[{"id":"85c4c2fa-d901-4d11-a7a7-b868bb77deaa","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:20:24.804599Z","dataset_id":"02299014-fa9b-409d-af64-266f9b077c5d","expected_output":"","input":{"prompt":"What
+        is the capital of France?"},"updated_at":"2025-08-06T16:20:24.804599Z"}}],"meta":{"after":""}}'
     headers:
       content-length:
-      - '388'
+      - '375'
       content-security-policy:
       - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 06 Aug 2025 03:00:57 GMT
+      - Wed, 06 Aug 2025 16:20:27 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary:

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_05c49e7d-1f10-4f4f-a660-114b00674e13_records_get_c5eb7aac.yaml

@@ -1,8 +1,8 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "05c49e7d-1f10-4f4f-a660-114b00674e13",
+    body: '{"data": {"type": "datasets", "id": "15903f83-b554-4a54-8448-0741b1c0f2aa",
       "attributes": {"insert_records": [{"input": {"prompt": "What is the capital
-      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": {}}], "update_records":
+      of France?"}, "expected_output": {"answer": "Paris"}, "metadata": null}], "update_records":
       [], "delete_records": []}}}'
     headers:
       Accept:
@@ -13,27 +13,27 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '269'
+      - '271'
       ? !!python/object/apply:multidict._multidict.istr
       - Content-Type
       : - application/json
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/05c49e7d-1f10-4f4f-a660-114b00674e13/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/15903f83-b554-4a54-8448-0741b1c0f2aa/batch_update
   response:
     body:
-      string: '{"data":[{"id":"3f4b5c92-27d5-40f7-a655-a9557af891ce","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T03:00:34.501998598Z","dataset_id":"05c49e7d-1f10-4f4f-a660-114b00674e13","expected_output":{"answer":"Paris"},"input":{"prompt":"What
-        is the capital of France?"},"metadata":{},"updated_at":"2025-08-06T03:00:34.501998598Z","version":1}}]}'
+      string: '{"data":[{"id":"23bc4211-8eb9-4579-81ef-2605554d78fe","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:20:59.771284671Z","dataset_id":"15903f83-b554-4a54-8448-0741b1c0f2aa","expected_output":{"answer":"Paris"},"input":{"prompt":"What
+        is the capital of France?"},"updated_at":"2025-08-06T16:20:59.771284671Z","version":1}}]}'
     headers:
       content-length:
-      - '403'
+      - '389'
       content-security-policy:
       - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 06 Aug 2025 03:00:34 GMT
+      - Wed, 06 Aug 2025 16:21:00 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary:

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_05c49e7d-1f10-4f4f-a660-114b00674e13_batch_update_post_c6b80f5f.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_15903f83-b554-4a54-8448-0741b1c0f2aa_batch_update_post_29d77066.yaml

@@ -1,9 +1,9 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "05c49e7d-1f10-4f4f-a660-114b00674e13",
+    body: '{"data": {"type": "datasets", "id": "15903f83-b554-4a54-8448-0741b1c0f2aa",
       "attributes": {"insert_records": [{"input": {"prompt": "What is the capital
-      of Sealand?"}, "expected_output": "", "metadata": {}}], "update_records": [],
-      "delete_records": []}}}'
+      of Sealand?"}, "expected_output": null, "metadata": null}], "update_records":
+      [], "delete_records": []}}}'
     headers:
       Accept:
       - '*/*'
@@ -13,27 +13,27 @@ interactions:
       Connection:
       - keep-alive
       Content-Length:
-      - '253'
+      - '257'
       ? !!python/object/apply:multidict._multidict.istr
       - Content-Type
       : - application/json
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/05c49e7d-1f10-4f4f-a660-114b00674e13/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/15903f83-b554-4a54-8448-0741b1c0f2aa/batch_update
   response:
     body:
-      string: '{"data":[{"id":"ff37d6e6-98cf-45c6-b042-cc603f065858","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-06T03:00:38.997580069Z","dataset_id":"05c49e7d-1f10-4f4f-a660-114b00674e13","expected_output":"","input":{"prompt":"What
-        is the capital of Sealand?"},"metadata":{},"updated_at":"2025-08-06T03:00:38.997580069Z","version":2}}]}'
+      string: '{"data":[{"id":"19a8e177-430b-4384-81ed-10b419f76eca","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:21:04.184059658Z","dataset_id":"15903f83-b554-4a54-8448-0741b1c0f2aa","input":{"prompt":"What
+        is the capital of Sealand?"},"updated_at":"2025-08-06T16:21:04.184059658Z","version":2}}]}'
     headers:
       content-length:
-      - '388'
+      - '353'
       content-security-policy:
       - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 06 Aug 2025 03:00:39 GMT
+      - Wed, 06 Aug 2025 16:21:04 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary:

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_05c49e7d-1f10-4f4f-a660-114b00674e13_batch_update_post_1bec44ba.yaml renamed to tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_15903f83-b554-4a54-8448-0741b1c0f2aa_batch_update_post_2d9f6dd8.yaml

@@ -0,0 +1,47 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Accept-Encoding
+      : - identity
+      Connection:
+      - keep-alive
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Length
+      : - '0'
+      ? !!python/object/apply:multidict._multidict.istr
+      - Content-Type
+      : - application/json
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/15903f83-b554-4a54-8448-0741b1c0f2aa/records
+  response:
+    body:
+      string: '{"data":[{"id":"19a8e177-430b-4384-81ed-10b419f76eca","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:21:04.184059Z","dataset_id":"15903f83-b554-4a54-8448-0741b1c0f2aa","input":{"prompt":"What
+        is the capital of Sealand?"},"updated_at":"2025-08-06T16:21:04.184059Z"}},{"id":"23bc4211-8eb9-4579-81ef-2605554d78fe","type":"datasets","attributes":{"author":{"id":"a7cd01e3-f412-11ed-a144-0aa89e224034"},"created_at":"2025-08-06T16:20:59.771284Z","dataset_id":"15903f83-b554-4a54-8448-0741b1c0f2aa","expected_output":{"answer":"Paris"},"input":{"prompt":"What
+        is the capital of France?"},"updated_at":"2025-08-06T16:20:59.771284Z"}}],"meta":{"after":""}}'
+    headers:
+      content-length:
+      - '716'
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      content-type:
+      - application/vnd.api+json
+      date:
+      - Wed, 06 Aug 2025 16:21:07 GMT
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1

tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_15903f83-b554-4a54-8448-0741b1c0f2aa_records_get_6dc85ec9.yaml

@@ -1,8 +1,8 @@
 interactions:
 - request:
-    body: '{"data": {"type": "datasets", "id": "2f376001-f964-4f1e-897d-4585ccff39ea",
+    body: '{"data": {"type": "datasets", "id": "55c9fa1e-61ec-49c9-98c8-89611f0728b5",
       "attributes": {"insert_records": [], "update_records": [], "delete_records":
-      ["f859c072-c30a-497b-be53-a66d14fedd57"]}}}'
+      ["3dcb6183-0f7f-43dd-9436-f57908bc5ed9"]}}}'
     headers:
       Accept:
       - '*/*'
@@ -19,7 +19,7 @@ interactions:
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/2f376001-f964-4f1e-897d-4585ccff39ea/batch_update
+    uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/55c9fa1e-61ec-49c9-98c8-89611f0728b5/batch_update
   response:
     body:
       string: '{"data":[]}'
@@ -31,7 +31,7 @@ interactions:
       content-type:
       - application/vnd.api+json
       date:
-      - Wed, 06 Aug 2025 03:00:54 GMT
+      - Wed, 06 Aug 2025 16:21:17 GMT
       strict-transport-security:
       - max-age=31536000; includeSubDomains; preload
       vary: