Skip to content

Commit 9e70a5a

Browse files
RulaKhaledandreiborza
RulaKhaled
and
andreiborza
authored
feat(core): Add tool calls attributes for Anthropic AI (#17478)
This PR adds missing tool call attributes, we addgen_ai.response.tool_calls attribute for Anthropic AI, supporting both streaming and non-streaming requests. Core changes: Request Side - Capture available tools: - Extract tools extract from request params - Set gen_ai.request.available_tools attribute Response Side - Capture actual tool calls: - Extract from response.tool_calls - Set gen_ai.response.tool_calls attribute for both Streaming Support (in streaming.ts): - Accumulation of tool calls during streaming - Respects recordOutputs privacy setting --------- Co-authored-by: Andrei <[email protected]>
1 parent 9a37660 commit 9e70a5a

File tree

7 files changed

+380
-16
lines changed

7 files changed

+380
-16
lines changed

‎dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-tools.mjs

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { instrumentAnthropicAiClient } from '@sentry/core';
2+
import * as Sentry from '@sentry/node';
3+
4+
function createMockStreamEvents(model = 'claude-3-haiku-20240307') {
5+
async function* generator() {
6+
// initial message metadata with id/model and input tokens
7+
yield {
8+
type: 'content_block_start',
9+
message: {
10+
id: 'msg_stream_tool_1',
11+
type: 'message',
12+
role: 'assistant',
13+
model,
14+
content: [],
15+
stop_reason: 'end_turn',
16+
usage: { input_tokens: 11 },
17+
},
18+
};
19+
20+
// streamed text
21+
yield { type: 'content_block_delta', delta: { text: 'Starting tool...' } };
22+
23+
// tool_use streamed via partial json
24+
yield {
25+
type: 'content_block_start',
26+
index: 0,
27+
content_block: { type: 'tool_use', id: 'tool_weather_2', name: 'weather' },
28+
};
29+
yield { type: 'content_block_delta', index: 0, delta: { partial_json: '{"city":' } };
30+
yield { type: 'content_block_delta', index: 0, delta: { partial_json: '"Paris"}' } };
31+
yield { type: 'content_block_stop', index: 0 };
32+
33+
// more text
34+
yield { type: 'content_block_delta', delta: { text: 'Done.' } };
35+
36+
// final usage
37+
yield { type: 'message_delta', usage: { output_tokens: 9 } };
38+
}
39+
return generator();
40+
}
41+
42+
class MockAnthropic {
43+
constructor(config) {
44+
this.apiKey = config.apiKey;
45+
this.messages = {
46+
create: this._messagesCreate.bind(this),
47+
stream: this._messagesStream.bind(this),
48+
};
49+
}
50+
51+
async _messagesCreate(params) {
52+
await new Promise(resolve => setTimeout(resolve, 5));
53+
if (params?.stream) {
54+
return createMockStreamEvents(params.model);
55+
}
56+
return {
57+
id: 'msg_mock_no_stream',
58+
type: 'message',
59+
model: params.model,
60+
role: 'assistant',
61+
content: [{ type: 'text', text: 'No stream' }],
62+
usage: { input_tokens: 2, output_tokens: 3 },
63+
};
64+
}
65+
66+
async _messagesStream(params) {
67+
await new Promise(resolve => setTimeout(resolve, 5));
68+
return createMockStreamEvents(params?.model);
69+
}
70+
}
71+
72+
async function run() {
73+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
74+
const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' });
75+
const client = instrumentAnthropicAiClient(mockClient);
76+
77+
// stream via create(stream:true)
78+
const stream1 = await client.messages.create({
79+
model: 'claude-3-haiku-20240307',
80+
messages: [{ role: 'user', content: 'Need the weather' }],
81+
tools: [
82+
{
83+
name: 'weather',
84+
description: 'Get weather',
85+
input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] },
86+
},
87+
],
88+
stream: true,
89+
});
90+
for await (const _ of stream1) {
91+
void _;
92+
}
93+
94+
// stream via messages.stream
95+
const stream2 = await client.messages.stream({
96+
model: 'claude-3-haiku-20240307',
97+
messages: [{ role: 'user', content: 'Need the weather' }],
98+
tools: [
99+
{
100+
name: 'weather',
101+
description: 'Get weather',
102+
input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] },
103+
},
104+
],
105+
});
106+
for await (const _ of stream2) {
107+
void _;
108+
}
109+
});
110+
}
111+
112+
run();

‎dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-tools.mjs

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import { instrumentAnthropicAiClient } from '@sentry/core';
2+
import * as Sentry from '@sentry/node';
3+
4+
class MockAnthropic {
5+
constructor(config) {
6+
this.apiKey = config.apiKey;
7+
8+
this.messages = {
9+
create: this._messagesCreate.bind(this),
10+
};
11+
}
12+
13+
async _messagesCreate(params) {
14+
await new Promise(resolve => setTimeout(resolve, 5));
15+
16+
return {
17+
id: 'msg_mock_tool_1',
18+
type: 'message',
19+
model: params.model,
20+
role: 'assistant',
21+
content: [
22+
{ type: 'text', text: 'Let me check the weather.' },
23+
{
24+
type: 'tool_use',
25+
id: 'tool_weather_1',
26+
name: 'weather',
27+
input: { city: 'Paris' },
28+
},
29+
{ type: 'text', text: 'It is sunny.' },
30+
],
31+
stop_reason: 'end_turn',
32+
stop_sequence: null,
33+
usage: {
34+
input_tokens: 5,
35+
output_tokens: 7,
36+
},
37+
};
38+
}
39+
}
40+
41+
async function run() {
42+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
43+
const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' });
44+
const client = instrumentAnthropicAiClient(mockClient);
45+
46+
await client.messages.create({
47+
model: 'claude-3-haiku-20240307',
48+
messages: [{ role: 'user', content: 'What is the weather?' }],
49+
tools: [
50+
{
51+
name: 'weather',
52+
description: 'Get the weather by city',
53+
input_schema: {
54+
type: 'object',
55+
properties: { city: { type: 'string' } },
56+
required: ['city'],
57+
},
58+
},
59+
],
60+
});
61+
});
62+
}
63+
64+
run();

‎dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,4 +293,59 @@ describe('Anthropic integration', () => {
293293
await createRunner().ignore('event').expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE }).start().completed();
294294
});
295295
});
296+
297+
// Non-streaming tool calls + available tools (PII true)
298+
createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
299+
test('non-streaming sets available tools and tool calls with PII', async () => {
300+
const EXPECTED_TOOLS_JSON =
301+
'[{"name":"weather","description":"Get the weather by city","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]';
302+
const EXPECTED_TOOL_CALLS_JSON =
303+
'[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]';
304+
await createRunner()
305+
.ignore('event')
306+
.expect({
307+
transaction: {
308+
spans: expect.arrayContaining([
309+
expect.objectContaining({
310+
op: 'gen_ai.messages',
311+
data: expect.objectContaining({
312+
'gen_ai.request.available_tools': EXPECTED_TOOLS_JSON,
313+
'gen_ai.response.tool_calls': EXPECTED_TOOL_CALLS_JSON,
314+
}),
315+
}),
316+
]),
317+
},
318+
})
319+
.start()
320+
.completed();
321+
});
322+
});
323+
324+
// Streaming tool calls + available tools (PII true)
325+
createEsmAndCjsTests(__dirname, 'scenario-stream-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
326+
test('streaming sets available tools and tool calls with PII', async () => {
327+
const EXPECTED_TOOLS_JSON =
328+
'[{"name":"weather","description":"Get weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]';
329+
const EXPECTED_TOOL_CALLS_JSON =
330+
'[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]';
331+
await createRunner()
332+
.ignore('event')
333+
.expect({
334+
transaction: {
335+
spans: expect.arrayContaining([
336+
expect.objectContaining({
337+
description: expect.stringContaining('stream-response'),
338+
op: 'gen_ai.messages',
339+
data: expect.objectContaining({
340+
'gen_ai.request.available_tools': EXPECTED_TOOLS_JSON,
341+
'gen_ai.response.tool_calls': EXPECTED_TOOL_CALLS_JSON,
342+
}),
343+
}),
344+
]),
345+
},
346+
})
347+
.start()
348+
.completed();
349+
});
350+
});
296351
});

‎packages/core/src/utils/anthropic-ai/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ export const ANTHROPIC_AI_INSTRUMENTED_METHODS = [
99
'models.get',
1010
'completions.create',
1111
'models.retrieve',
12+
'beta.messages.create',
1213
] as const;

‎packages/core/src/utils/anthropic-ai/index.ts

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
ANTHROPIC_AI_RESPONSE_TIMESTAMP_ATTRIBUTE,
99
GEN_AI_OPERATION_NAME_ATTRIBUTE,
1010
GEN_AI_PROMPT_ATTRIBUTE,
11+
GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE,
1112
GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE,
1213
GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE,
1314
GEN_AI_REQUEST_MESSAGES_ATTRIBUTE,
@@ -19,6 +20,7 @@ import {
1920
GEN_AI_RESPONSE_ID_ATTRIBUTE,
2021
GEN_AI_RESPONSE_MODEL_ATTRIBUTE,
2122
GEN_AI_RESPONSE_TEXT_ATTRIBUTE,
23+
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
2224
GEN_AI_SYSTEM_ATTRIBUTE,
2325
} from '../ai/gen-ai-attributes';
2426
import { buildMethodPath, getFinalOperationName, getSpanOperation, setTokenUsageAttributes } from '../ai/utils';
@@ -31,6 +33,7 @@ import type {
3133
AnthropicAiOptions,
3234
AnthropicAiResponse,
3335
AnthropicAiStreamingEvent,
36+
ContentBlock,
3437
} from './types';
3538
import { shouldInstrument } from './utils';
3639

@@ -46,6 +49,9 @@ function extractRequestAttributes(args: unknown[], methodPath: string): Record<s
4649

4750
if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) {
4851
const params = args[0] as Record<string, unknown>;
52+
if (params.tools && Array.isArray(params.tools)) {
53+
attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] = JSON.stringify(params.tools);
54+
}
4955

5056
attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE] = params.model ?? 'unknown';
5157
if ('temperature' in params) attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE] = params.temperature;
@@ -96,10 +102,21 @@ function addResponseAttributes(span: Span, response: AnthropicAiResponse, record
96102
if (Array.isArray(response.content)) {
97103
span.setAttributes({
98104
[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: response.content
99-
.map((item: { text: string | undefined }) => item.text)
100-
.filter((text): text is string => text !== undefined)
105+
.map((item: ContentBlock) => item.text)
106+
.filter(text => !!text)
101107
.join(''),
102108
});
109+
110+
const toolCalls: Array<ContentBlock> = [];
111+
112+
for (const item of response.content) {
113+
if (item.type === 'tool_use' || item.type === 'server_tool_use') {
114+
toolCalls.push(item);
115+
}
116+
}
117+
if (toolCalls.length > 0) {
118+
span.setAttributes({ [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(toolCalls) });
119+
}
103120
}
104121
}
105122
// Completions.create

0 commit comments

Comments
 (0)