Skip to content

Commit 6ab758e

Browse files
gary-huanggary-huang
committed
make expected output optional
fix case where updating a record with expected_output to no expected_output (null)
1 parent 0ec2962 commit 6ab758e

File tree

27 files changed

+1226
-21
lines changed

27 files changed

+1226
-21
lines changed

‎ddtrace/llmobs/_llmobs.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@ def create_dataset_from_csv(
616616
csv_path: str,
617617
dataset_name: str,
618618
input_data_columns: List[str],
619-
expected_output_columns: List[str],
619+
expected_output_columns: List[str] = [],
620620
metadata_columns: List[str] = [],
621621
csv_delimiter: str = ",",
622622
description="",

‎ddtrace/llmobs/_writer.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ class LLMObsExperimentsClient(BaseLLMObsWriter):
297297
EVP_SUBDOMAIN_HEADER_VALUE = EXP_SUBDOMAIN_NAME
298298
AGENTLESS_BASE_URL = AGENTLESS_EXP_BASE_URL
299299
ENDPOINT = ""
300+
TIMEOUT = 5.0
300301

301302
def request(self, method: str, path: str, body: JSONType = None) -> Response:
302303
headers = {
@@ -308,7 +309,7 @@ def request(self, method: str, path: str, body: JSONType = None) -> Response:
308309
headers[EVP_SUBDOMAIN_HEADER_NAME] = self.EVP_SUBDOMAIN_HEADER_VALUE
309310

310311
encoded_body = json.dumps(body).encode("utf-8") if body else b""
311-
conn = get_connection(self._intake)
312+
conn = get_connection(url=self._intake, timeout=self.TIMEOUT)
312313
try:
313314
url = self._intake + self._endpoint + path
314315
logger.debug("requesting %s", url)
@@ -363,15 +364,19 @@ def dataset_batch_update(
363364
irs: JSONType = [
364365
{
365366
"input": cast(Dict[str, JSONType], r["input_data"]),
366-
"expected_output": r["expected_output"],
367+
"expected_output": r.get("expected_output", None),
367368
"metadata": r.get("metadata", {}),
368369
}
369370
for r in insert_records
370371
]
371372
urs: JSONType = [
372373
{
373374
"input": cast(Dict[str, JSONType], r["input_data"]),
374-
"expected_output": r["expected_output"],
375+
# if we default to None, the API will treat None (null in JSON)
376+
# as if the field was not in the JSON, and treat it as not being updated,
377+
# despite going from expected_output having a value to no value being a
378+
# valid use case
379+
"expected_output": r.get("expected_output", None),
375380
"metadata": r.get("metadata", {}),
376381
"id": r["record_id"],
377382
}
@@ -428,7 +433,7 @@ def dataset_get_with_records(self, name: str) -> Dataset:
428433
{
429434
"record_id": record["id"],
430435
"input_data": attrs["input"],
431-
"expected_output": attrs["expected_output"],
436+
"expected_output": attrs.get("expected_output", None),
432437
"metadata": attrs.get("metadata", {}),
433438
}
434439
)

‎setup.cfg‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,5 @@ skip = *.json,*.h,*.cpp,*.c,.riot,.tox,.mypy_cache,.git,*ddtrace/vendor,tests/co
66
exclude-file = .codespellignorelines
77
ignore-words-list = asend,dne,fo,medias,ment,nin,ot,setttings,statics,ba,spawnve,doas
88

9-
# DEV: We use `conftest.py` as a local pytest plugin to configure hooks for collection
10-
[tool:pytest]
11-
# --cov-report is intentionally empty else pytest-cov will default to generating a report
12-
addopts =
13-
--cov=ddtrace/
14-
--cov=tests/
15-
--cov-append
16-
--cov-report=
17-
--durations=10
18-
--junitxml=test-results/junit.xml
19-
# DEV: The default is `test_*\.py` which will miss `test.py` files
20-
python_files = test*\.py
21-
asyncio_mode = auto
22-
239
[flake8]
2410
max-line-length = 120

‎tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_6234ef1d-6fb4-409a-884e-184e485db40f_batch_update_post_11fd5c86.yaml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
interactions:
2+
- request:
3+
body: '{"data": {"type": "datasets", "id": "6234ef1d-6fb4-409a-884e-184e485db40f",
4+
"attributes": {"insert_records": [], "update_records": [], "delete_records":
5+
["6a9d834b-9d7c-4186-837b-76dd2c61c6d1"]}}}'
6+
headers:
7+
Accept:
8+
- '*/*'
9+
? !!python/object/apply:multidict._multidict.istr
10+
- Accept-Encoding
11+
: - identity
12+
Connection:
13+
- keep-alive
14+
Content-Length:
15+
- '196'
16+
? !!python/object/apply:multidict._multidict.istr
17+
- Content-Type
18+
: - application/json
19+
User-Agent:
20+
- python-requests/2.32.3
21+
method: POST
22+
uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/6234ef1d-6fb4-409a-884e-184e485db40f/batch_update
23+
response:
24+
body:
25+
string: '{"data":[]}'
26+
headers:
27+
content-length:
28+
- '11'
29+
content-security-policy:
30+
- frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
31+
content-type:
32+
- application/vnd.api+json
33+
date:
34+
- Mon, 04 Aug 2025 20:07:13 GMT
35+
strict-transport-security:
36+
- max-age=31536000; includeSubDomains; preload
37+
vary:
38+
- Accept-Encoding
39+
x-content-type-options:
40+
- nosniff
41+
x-frame-options:
42+
- SAMEORIGIN
43+
status:
44+
code: 200
45+
message: OK
46+
version: 1

‎tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_6234ef1d-6fb4-409a-884e-184e485db40f_batch_update_post_53d841e8.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
interactions:
2+
- request:
3+
body: '{"data": {"type": "datasets", "id": "6234ef1d-6fb4-409a-884e-184e485db40f",
4+
"attributes": {"insert_records": [{"input": {"prompt": "What is the capital
5+
of Nauru?"}, "expected_output": null, "metadata": {}}, {"input": {"prompt":
6+
"What is the capital of Sealand?"}, "expected_output": null, "metadata": {}}],
7+
"update_records": [], "delete_records": []}}}'
8+
headers:
9+
Accept:
10+
- '*/*'
11+
? !!python/object/apply:multidict._multidict.istr
12+
- Accept-Encoding
13+
: - identity
14+
Connection:
15+
- keep-alive
16+
Content-Length:
17+
- '352'
18+
? !!python/object/apply:multidict._multidict.istr
19+
- Content-Type
20+
: - application/json
21+
User-Agent:
22+
- python-requests/2.32.3
23+
method: POST
24+
uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/6234ef1d-6fb4-409a-884e-184e485db40f/batch_update
25+
response:
26+
body:
27+
string: '{"data":[{"id":"bf98322c-7c05-4016-9e90-e51fc8177427","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:07:08.792777594Z","dataset_id":"6234ef1d-6fb4-409a-884e-184e485db40f","input":{"prompt":"What
28+
is the capital of Nauru?"},"metadata":{},"updated_at":"2025-08-04T20:07:08.792777594Z","version":1}},{"id":"6a9d834b-9d7c-4186-837b-76dd2c61c6d1","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:07:08.792777594Z","dataset_id":"6234ef1d-6fb4-409a-884e-184e485db40f","input":{"prompt":"What
29+
is the capital of Sealand?"},"metadata":{},"updated_at":"2025-08-04T20:07:08.792777594Z","version":1}}]}'
30+
headers:
31+
content-length:
32+
- '722'
33+
content-security-policy:
34+
- frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
35+
content-type:
36+
- application/vnd.api+json
37+
date:
38+
- Mon, 04 Aug 2025 20:07:09 GMT
39+
strict-transport-security:
40+
- max-age=31536000; includeSubDomains; preload
41+
vary:
42+
- Accept-Encoding
43+
x-content-type-options:
44+
- nosniff
45+
x-frame-options:
46+
- SAMEORIGIN
47+
status:
48+
code: 200
49+
message: OK
50+
version: 1

‎tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_6234ef1d-6fb4-409a-884e-184e485db40f_records_get_5900bfef.yaml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
Accept:
6+
- '*/*'
7+
? !!python/object/apply:multidict._multidict.istr
8+
- Accept-Encoding
9+
: - identity
10+
Connection:
11+
- keep-alive
12+
? !!python/object/apply:multidict._multidict.istr
13+
- Content-Length
14+
: - '0'
15+
? !!python/object/apply:multidict._multidict.istr
16+
- Content-Type
17+
: - application/json
18+
User-Agent:
19+
- python-requests/2.32.3
20+
method: GET
21+
uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/6234ef1d-6fb4-409a-884e-184e485db40f/records
22+
response:
23+
body:
24+
string: '{"data":[{"id":"bf98322c-7c05-4016-9e90-e51fc8177427","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:07:08.792777Z","dataset_id":"6234ef1d-6fb4-409a-884e-184e485db40f","input":{"prompt":"What
25+
is the capital of Nauru?"},"metadata":{},"updated_at":"2025-08-04T20:07:08.792777Z"}}],"meta":{"after":""}}'
26+
headers:
27+
content-length:
28+
- '367'
29+
content-security-policy:
30+
- frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
31+
content-type:
32+
- application/vnd.api+json
33+
date:
34+
- Mon, 04 Aug 2025 20:07:16 GMT
35+
strict-transport-security:
36+
- max-age=31536000; includeSubDomains; preload
37+
vary:
38+
- Accept-Encoding
39+
x-content-type-options:
40+
- nosniff
41+
x-frame-options:
42+
- SAMEORIGIN
43+
status:
44+
code: 200
45+
message: OK
46+
version: 1

‎tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_7a23dfb3-4b7f-4074-888d-3b29e875d83b_batch_update_post_86c700a6.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
interactions:
2+
- request:
3+
body: '{"data": {"type": "datasets", "id": "7a23dfb3-4b7f-4074-888d-3b29e875d83b",
4+
"attributes": {"insert_records": [{"input": {"in0": "r0v1", "in1": "r0v2", "in2":
5+
"r0v3"}, "expected_output": {}, "metadata": {}}, {"input": {"in0": "r1v1", "in1":
6+
"r1v2", "in2": "r1v3"}, "expected_output": {}, "metadata": {}}], "update_records":
7+
[], "delete_records": []}}}'
8+
headers:
9+
Accept:
10+
- '*/*'
11+
? !!python/object/apply:multidict._multidict.istr
12+
- Accept-Encoding
13+
: - identity
14+
Connection:
15+
- keep-alive
16+
Content-Length:
17+
- '350'
18+
? !!python/object/apply:multidict._multidict.istr
19+
- Content-Type
20+
: - application/json
21+
User-Agent:
22+
- python-requests/2.32.3
23+
method: POST
24+
uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/7a23dfb3-4b7f-4074-888d-3b29e875d83b/batch_update
25+
response:
26+
body:
27+
string: '{"data":[{"id":"60ca1a59-e10a-4a5a-8d06-0b3790c2045e","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:06:17.911096065Z","dataset_id":"7a23dfb3-4b7f-4074-888d-3b29e875d83b","expected_output":{},"input":{"in0":"r0v1","in1":"r0v2","in2":"r0v3"},"metadata":{},"updated_at":"2025-08-04T20:06:17.911096065Z","version":1}},{"id":"79e7dcfd-5fef-4e25-87d0-5bc94d1ee65d","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:06:17.911096065Z","dataset_id":"7a23dfb3-4b7f-4074-888d-3b29e875d83b","expected_output":{},"input":{"in0":"r1v1","in1":"r1v2","in2":"r1v3"},"metadata":{},"updated_at":"2025-08-04T20:06:17.911096065Z","version":1}}]}'
28+
headers:
29+
content-length:
30+
- '758'
31+
content-security-policy:
32+
- frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
33+
content-type:
34+
- application/vnd.api+json
35+
date:
36+
- Mon, 04 Aug 2025 20:06:18 GMT
37+
strict-transport-security:
38+
- max-age=31536000; includeSubDomains; preload
39+
vary:
40+
- Accept-Encoding
41+
x-content-type-options:
42+
- nosniff
43+
x-frame-options:
44+
- SAMEORIGIN
45+
status:
46+
code: 200
47+
message: OK
48+
version: 1

‎tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_7a23dfb3-4b7f-4074-888d-3b29e875d83b_records_get_f47a58cb.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
interactions:
2+
- request:
3+
body: null
4+
headers:
5+
Accept:
6+
- '*/*'
7+
? !!python/object/apply:multidict._multidict.istr
8+
- Accept-Encoding
9+
: - identity
10+
Connection:
11+
- keep-alive
12+
? !!python/object/apply:multidict._multidict.istr
13+
- Content-Length
14+
: - '0'
15+
? !!python/object/apply:multidict._multidict.istr
16+
- Content-Type
17+
: - application/json
18+
User-Agent:
19+
- python-requests/2.32.3
20+
method: GET
21+
uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/7a23dfb3-4b7f-4074-888d-3b29e875d83b/records
22+
response:
23+
body:
24+
string: '{"data":[{"id":"79e7dcfd-5fef-4e25-87d0-5bc94d1ee65d","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:06:17.911096Z","dataset_id":"7a23dfb3-4b7f-4074-888d-3b29e875d83b","expected_output":{},"input":{"in0":"r1v1","in1":"r1v2","in2":"r1v3"},"metadata":{},"updated_at":"2025-08-04T20:06:17.911096Z"}},{"id":"60ca1a59-e10a-4a5a-8d06-0b3790c2045e","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:06:17.911096Z","dataset_id":"7a23dfb3-4b7f-4074-888d-3b29e875d83b","expected_output":{},"input":{"in0":"r0v1","in1":"r0v2","in2":"r0v3"},"metadata":{},"updated_at":"2025-08-04T20:06:17.911096Z"}}],"meta":{"after":""}}'
25+
headers:
26+
content-length:
27+
- '742'
28+
content-security-policy:
29+
- frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
30+
content-type:
31+
- application/vnd.api+json
32+
date:
33+
- Mon, 04 Aug 2025 20:06:23 GMT
34+
strict-transport-security:
35+
- max-age=31536000; includeSubDomains; preload
36+
vary:
37+
- Accept-Encoding
38+
x-content-type-options:
39+
- nosniff
40+
x-frame-options:
41+
- SAMEORIGIN
42+
status:
43+
code: 200
44+
message: OK
45+
version: 1

‎tests/llmobs/llmobs_cassettes/datadog/datadog_api_unstable_llm-obs_v1_datasets_952504e3-4abb-4f8f-b388-6d7344cfb3c9_batch_update_post_330dd00a.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
interactions:
2+
- request:
3+
body: '{"data": {"type": "datasets", "id": "952504e3-4abb-4f8f-b388-6d7344cfb3c9",
4+
"attributes": {"insert_records": [{"input": {"prompt": "What is the capital
5+
of France?"}, "expected_output": {"answer": "Paris"}, "metadata": {}}], "update_records":
6+
[], "delete_records": []}}}'
7+
headers:
8+
Accept:
9+
- '*/*'
10+
? !!python/object/apply:multidict._multidict.istr
11+
- Accept-Encoding
12+
: - identity
13+
Connection:
14+
- keep-alive
15+
Content-Length:
16+
- '269'
17+
? !!python/object/apply:multidict._multidict.istr
18+
- Content-Type
19+
: - application/json
20+
User-Agent:
21+
- python-requests/2.32.3
22+
method: POST
23+
uri: https://api.datadoghq.com/api/unstable/llm-obs/v1/datasets/952504e3-4abb-4f8f-b388-6d7344cfb3c9/batch_update
24+
response:
25+
body:
26+
string: '{"data":[{"id":"d9217350-bed1-47c2-8f91-adb1151e623c","type":"datasets","attributes":{"author":{"id":"de473b30-eb9f-11e9-a77a-c7405862b8bd"},"created_at":"2025-08-04T20:06:39.571375101Z","dataset_id":"952504e3-4abb-4f8f-b388-6d7344cfb3c9","expected_output":{"answer":"Paris"},"input":{"prompt":"What
27+
is the capital of France?"},"metadata":{},"updated_at":"2025-08-04T20:06:39.571375101Z","version":1}}]}'
28+
headers:
29+
content-length:
30+
- '403'
31+
content-security-policy:
32+
- frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
33+
content-type:
34+
- application/vnd.api+json
35+
date:
36+
- Mon, 04 Aug 2025 20:06:39 GMT
37+
strict-transport-security:
38+
- max-age=31536000; includeSubDomains; preload
39+
vary:
40+
- Accept-Encoding
41+
x-content-type-options:
42+
- nosniff
43+
x-frame-options:
44+
- SAMEORIGIN
45+
status:
46+
code: 200
47+
message: OK
48+
version: 1

0 commit comments

Comments
 (0)