diff --git a/static/app/gettingStartedDocs/node/utils.tsx b/static/app/gettingStartedDocs/node/utils.tsx index 8c7081b9fac832..5060539137b5d3 100644 --- a/static/app/gettingStartedDocs/node/utils.tsx +++ b/static/app/gettingStartedDocs/node/utils.tsx @@ -7,6 +7,7 @@ import type { } from 'sentry/components/onboarding/gettingStartedDoc/types'; import {StepType} from 'sentry/components/onboarding/gettingStartedDoc/types'; import {t, tct} from 'sentry/locale'; +import {CopyLLMPromptButton} from 'sentry/views/insights/pages/agents/llmOnboardingInstructions'; function getInstallSnippet({ params, @@ -605,7 +606,7 @@ Sentry.init({ { type: 'text', text: tct( - 'Then follow the [link:manual instrumentation guide] to instrument your AI calls.', + 'Then follow the [link:manual instrumentation guide] to instrument your AI calls, or use an AI coding agent to do it for you.', { link: ( @@ -613,6 +614,10 @@ Sentry.init({ } ), }, + { + type: 'custom', + content: , + }, ]; const selected = (params.platformOptions as any)?.integration ?? 'vercel_ai'; diff --git a/static/app/gettingStartedDocs/python/agentMonitoring.tsx b/static/app/gettingStartedDocs/python/agentMonitoring.tsx index 48f890fba5506b..90683bc5351459 100644 --- a/static/app/gettingStartedDocs/python/agentMonitoring.tsx +++ b/static/app/gettingStartedDocs/python/agentMonitoring.tsx @@ -5,6 +5,7 @@ import type { } from 'sentry/components/onboarding/gettingStartedDoc/types'; import {StepType} from 'sentry/components/onboarding/gettingStartedDoc/types'; import {t, tct} from 'sentry/locale'; +import {CopyLLMPromptButton} from 'sentry/views/insights/pages/agents/llmOnboardingInstructions'; import {getPythonInstallCodeBlock} from './utils'; @@ -307,7 +308,7 @@ sentry_sdk.init( { type: 'text', text: tct( - 'Then follow the [link:manual instrumentation guide] to instrument your AI calls.', + 'Then follow the [link:manual instrumentation guide] to instrument your AI calls, or use an AI coding agent to do it for you.', { link: ( @@ -315,6 +316,10 @@ sentry_sdk.init( } ), }, + { + type: 'custom', + content: , + }, ], }; diff --git a/static/app/utils/analytics/agentMonitoringAnalyticsEvents.tsx b/static/app/utils/analytics/agentMonitoringAnalyticsEvents.tsx index 0385334b0c738a..47e77033273628 100644 --- a/static/app/utils/analytics/agentMonitoringAnalyticsEvents.tsx +++ b/static/app/utils/analytics/agentMonitoringAnalyticsEvents.tsx @@ -4,6 +4,7 @@ export type AgentMonitoringEventParameters = { direction: 'asc' | 'desc'; table: string; }; + 'agent-monitoring.copy-llm-prompt-click': Record; 'agent-monitoring.drawer.open': Record; 'agent-monitoring.drawer.span-select': Record; 'agent-monitoring.drawer.view-full-trace-click': Record; @@ -25,6 +26,7 @@ export const agentMonitoringEventMap: Record< keyof AgentMonitoringEventParameters, string > = { + 'agent-monitoring.copy-llm-prompt-click': 'Agent Monitoring: Copy LLM Prompt Click', 'agent-monitoring.page-view': 'Agent Monitoring: Page View', 'agent-monitoring.table-switch': 'Agent Monitoring: Table Switch', 'agent-monitoring.column-sort': 'Agent Monitoring: Column Sort', diff --git a/static/app/views/insights/pages/agents/llmOnboardingInstructions.tsx b/static/app/views/insights/pages/agents/llmOnboardingInstructions.tsx new file mode 100644 index 00000000000000..2f9107245f9f4c --- /dev/null +++ b/static/app/views/insights/pages/agents/llmOnboardingInstructions.tsx @@ -0,0 +1,197 @@ +import {Button} from 'sentry/components/core/button'; +import {IconCopy} from 'sentry/icons'; +import {t} from 'sentry/locale'; +import {trackAnalytics} from 'sentry/utils/analytics'; +import useCopyToClipboard from 'sentry/utils/useCopyToClipboard'; +import useOrganization from 'sentry/utils/useOrganization'; + +export function CopyLLMPromptButton() { + const {copy} = useCopyToClipboard(); + const organization = useOrganization(); + + return ( + + ); +} + +const LLM_ONBOARDING_INSTRUCTIONS = ` +# Instrument Sentry AI Agent Monitoring + +Follow this decision tree to instrument AI Agent Monitoring. + +## 1. Verify Sentry + Tracing + +**Search for Sentry initialization:** +- JS/TS: \`Sentry.init\` in entry points, \`@sentry/*\` in package.json +- Python: \`sentry_sdk.init\` in entry points, \`sentry-sdk\` in requirements + +**If not found:** Set up Sentry first following the official docs: +- JS/TS: https://docs.sentry.io/platforms/javascript/guides/node/ +- Python: https://docs.sentry.io/platforms/python/ + +**Verify tracing is enabled** (REQUIRED for AI monitoring): +\`\`\`javascript +// JS - must have tracesSampleRate > 0, min SDK version 10.28.0 +Sentry.init({ dsn: "...", tracesSampleRate: 1.0, sendDefaultPii: true }) +\`\`\` +\`\`\`python +# Python - must have traces_sample_rate > 0 +sentry_sdk.init(dsn="...", traces_sample_rate=1.0, send_default_pii=True) +\`\`\` + +If missing, add \`tracesSampleRate: 1.0\` / \`traces_sample_rate=1.0\` and \`sendDefaultPii: true\` / \`send_default_pii=True\`. + +## 2. Check for Supported AI Libraries + +Check in this order - **use the highest-level framework found** (e.g., if using Vercel AI SDK with OpenAI provider, use Vercel integration, not OpenAI): + +| Library (check in order) | JS Integration | Python Integration | Python Extra | +|--------------------------|---------------|-------------------|--------------| +| Vercel AI SDK | \`Sentry.vercelAIIntegration()\` | - | - | +| LangGraph | \`Sentry.langGraphIntegration()\` | Auto-enabled | \`sentry-sdk[langgraph]\` | +| LangChain | \`Sentry.langChainIntegration()\` | Auto-enabled | \`sentry-sdk[langchain]\` | +| OpenAI Agents | - | Auto-enabled | - | +| Pydantic AI | - | Auto-enabled | \`sentry-sdk[pydantic_ai]\` | +| LiteLLM | - | \`LiteLLMIntegration()\` | \`sentry-sdk[litellm]\` | +| OpenAI | \`Sentry.openAIIntegration()\` | Auto-enabled | - | +| Anthropic | \`Sentry.anthropicAIIntegration()\` | Auto-enabled | - | +| Google GenAI | \`Sentry.googleGenAIIntegration()\` | \`GoogleGenAIIntegration()\` | \`sentry-sdk[google_genai]\` | + +**If supported library found → Step 3A** +**If no supported library → Step 3B** + +## 3A. Enable Automatic Integration + +### JavaScript + +Add to Sentry.init integrations array with \`recordInputs\` and \`recordOutputs\`: + +\`\`\`javascript +import * as Sentry from "@sentry/node"; + +Sentry.init({ + dsn: "...", + tracesSampleRate: 1.0, + sendDefaultPii: true, + integrations: [ + Sentry.openAIIntegration({ recordInputs: true, recordOutputs: true }), + // OR other integration as needed + ], +}); +\`\`\` + +**Vercel AI SDK Extra Step:** Pass \`experimental_telemetry\` with \`functionId\` to every call: +\`\`\`javascript +const result = await generateText({ + model: openai("gpt-4o"), + prompt: "Tell me a joke", + experimental_telemetry: { + isEnabled: true, + functionId: "generate-joke", // Name your functions for better tracing + recordInputs: true, + recordOutputs: true, + }, +}); +\`\`\` + +### Python + +Install with extras if needed: +\`\`\`bash +pip install sentry-sdk[langchain] # or [langgraph], [litellm], [google_genai], [pydantic_ai] +\`\`\` + +Configure (some integrations auto-enable, some need explicit import): + +\`\`\`python +import sentry_sdk +# Only import if NOT auto-enabled (see table above) +from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration + +sentry_sdk.init( + dsn="...", + traces_sample_rate=1.0, + send_default_pii=True, # Required to capture inputs/outputs + integrations=[ + OpenAIAgentsIntegration(), # Only if explicit integration needed + ], +) +\`\`\` + +**Done.** SDK auto-instruments AI calls. + +## 3B. Manual Instrumentation + +Create spans with these exact \`op\` values and attributes: + +### AI Request (LLM call) +- **op:** \`"gen_ai.request"\` +- **name:** \`"chat "\` +- **Required:** \`gen_ai.request.model\` +- **Recommended:** \`gen_ai.usage.input_tokens\`, \`gen_ai.usage.output_tokens\` + +\`\`\`python +with sentry_sdk.start_span(op="gen_ai.request", name=f"chat {model}") as span: + span.set_data("gen_ai.request.model", model) + result = llm.generate(messages) + span.set_data("gen_ai.usage.input_tokens", result.input_tokens) + span.set_data("gen_ai.usage.output_tokens", result.output_tokens) + span.set_data("gen_ai.usage.input_tokens.cached", result.cached_tokens) + +\`\`\` + +### Invoke Agent +- **op:** \`"gen_ai.invoke_agent"\` +- **name:** \`"invoke_agent "\` +- **Required:** \`gen_ai.request.model\`, \`gen_ai.agent.name\` + +\`\`\`python +with sentry_sdk.start_span(op="gen_ai.invoke_agent", name=f"invoke_agent {agent_name}") as span: + span.set_data("gen_ai.agent.name", agent_name) + span.set_data("gen_ai.request.model", model) + result = agent.run() +\`\`\` + +### Execute Tool +- **op:** \`"gen_ai.execute_tool"\` +- **name:** \`"execute_tool "\` +- **Required:** \`gen_ai.tool.name\` + +\`\`\`python +with sentry_sdk.start_span(op="gen_ai.execute_tool", name=f"execute_tool {tool_name}") as span: + span.set_data("gen_ai.tool.name", tool_name) + span.set_data("gen_ai.tool.input", json.dumps(inputs)) + result = tool(**inputs) + span.set_data("gen_ai.tool.output", json.dumps(result)) +\`\`\` + +### Handoff (agent-to-agent) +- **op:** \`"gen_ai.handoff"\` +- **name:** \`"handoff from to "\` + +\`\`\`python +with sentry_sdk.start_span(op="gen_ai.handoff", name=f"handoff from {a} to {b}"): + pass +\`\`\` + +## Key Rules + +1. **All complex data must be JSON-stringified** - span attributes only accept primitives +2. **\`gen_ai.request.model\` is required** on \`gen_ai.request\` and \`gen_ai.invoke_agent\` spans +3. **Nest spans correctly:** \`gen_ai.invoke_agent\` spans should contain \`gen_ai.request\` and \`gen_ai.execute_tool\` spans as children +4. **JS min version:** \`@sentry/node@10.28.0\` or later +5. **Enable PII:** \`sendDefaultPii: true\` (JS) / \`send_default_pii=True\` (Python) to capture inputs/outputs +`; diff --git a/static/app/views/insights/pages/agents/onboarding.tsx b/static/app/views/insights/pages/agents/onboarding.tsx index 0dcece601187dd..96c438b6d185da 100644 --- a/static/app/views/insights/pages/agents/onboarding.tsx +++ b/static/app/views/insights/pages/agents/onboarding.tsx @@ -6,7 +6,6 @@ import emptyTraceImg from 'sentry-images/spot/profiling-empty-state.svg'; import {ExternalLink} from '@sentry/scraps/link'; import {Button} from 'sentry/components/core/button'; -import {LinkButton} from 'sentry/components/core/button/linkButton'; import {GuidedSteps} from 'sentry/components/guidedSteps/guidedSteps'; import LoadingIndicator from 'sentry/components/loadingIndicator'; import {AuthTokenGeneratorProvider} from 'sentry/components/onboarding/gettingStartedDoc/authTokenGenerator'; @@ -38,6 +37,7 @@ import useOrganization from 'sentry/utils/useOrganization'; import usePageFilters from 'sentry/utils/usePageFilters'; import useProjects from 'sentry/utils/useProjects'; import {useSpans} from 'sentry/views/insights/common/queries/useDiscover'; +import {CopyLLMPromptButton} from 'sentry/views/insights/pages/agents/llmOnboardingInstructions'; import {getHasAiSpansFilter} from 'sentry/views/insights/pages/agents/utils/query'; import {Referrer} from 'sentry/views/insights/pages/agents/utils/referrers'; @@ -264,28 +264,10 @@ export function Onboarding() { if (!agentMonitoringPlatforms.has(project.platform as PlatformKey)) { return ( - - -

- {tct( - 'Fiddlesticks. Auto instrumentation of AI Agents is not available for your [platform] project. ', - { - platform: currentPlatform?.name || project.slug, - } - )} -

-

- {tct( - 'However, you can still manually instrument your agents using the Sentry SDK tracing API. See [link:custom instrumentation docs] for details.', - { - link: ( - - ), - } - )} -

- - + ); } @@ -296,25 +278,7 @@ export function Onboarding() { const agentMonitoringDocs = docs?.agentMonitoringOnboarding; if (!agentMonitoringDocs || !dsn || !projectKeyId) { - return ( - - -

- {tct( - "The agent monitoring onboarding checklist isn't available for your [project] project yet, but you can still set up the Sentry SDK to start monitoring your AI agents.", - {project: project.slug} - )} -

- - {t('Go to Documentation')} - - - - ); + return ; } const docParams: DocsParams = { @@ -373,6 +337,66 @@ export function Onboarding() { ); } +function UnsupportedPlatformOnboarding({ + project, + platformName, +}: { + platformName: string; + project: Project; +}) { + return ( + + +

+ {tct( + 'Fiddlesticks. Auto instrumentation of AI Agents is not available for your [platform] project.', + { + platform: platformName, + } + )} +

+

+ {tct( + 'You can [link:manually instrument] your agents using the Sentry SDK tracing API, or use an AI coding agent to do it for you.', + { + link: ( + + ), + } + )} +

+ + + + ); +} + +function NoDocsOnboarding({project}: {project: Project}) { + return ( + + +

+ {tct( + "The agent monitoring onboarding checklist isn't available for your [project] project yet.", + {project: project.slug} + )} +

+

+ {tct( + 'You can set up the Sentry SDK by following our [link:documentation], or use an AI coding agent to do it for you.', + { + link: ( + + ), + } + )} +

+ + + + ); +} + const EventWaitingIndicator = styled((p: React.HTMLAttributes) => (
{t("Waiting for this project's first agent events")}