diff --git a/docs/specs/search-events.md b/docs/specs/search-events.md index e7a261124..e46fbbd0d 100644 --- a/docs/specs/search-events.md +++ b/docs/specs/search-events.md @@ -98,7 +98,7 @@ search_metrics({ - Prominent rendering directives for AI agents - Shareable Sentry Explorer URL -Dataset tools that declare an `outputSchema` (`search_errors`, `search_logs`, `search_traces`, and `search_metrics` today) return +Dataset tools that declare an `outputSchema` (`search_errors`, `search_logs`, `search_traces`, `search_metrics`, and `search_profiles` today) return `structuredContent` instead of the markdown above; legacy `search_events` keeps markdown. In that payload, `resultType` (`samples`, `aggregates`, or `timeSeries`) replaces the dataset-specific rendering, each row holds only the diff --git a/packages/mcp-core/src/toolDefinitions.json b/packages/mcp-core/src/toolDefinitions.json index 7527b08ac..374c0c473 100644 --- a/packages/mcp-core/src/toolDefinitions.json +++ b/packages/mcp-core/src/toolDefinitions.json @@ -11249,6 +11249,257 @@ }, "required": ["organizationSlug"] }, + "outputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "explanation": { + "type": ["string", "null"] + }, + "executedSearch": { + "type": "object", + "properties": { + "dataset": { + "type": "string" + }, + "query": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "sort": { + "type": ["string", "null"] + }, + "timeRange": { + "type": "object", + "properties": { + "statsPeriod": { + "type": ["string", "null"] + }, + "start": { + "type": ["string", "null"] + }, + "end": { + "type": ["string", "null"] + } + }, + "required": ["statsPeriod", "start", "end"], + "additionalProperties": false + } + }, + "required": ["dataset", "query", "fields", "sort", "timeRange"], + "additionalProperties": false + }, + "url": { + "type": "string", + "description": "Opens these results in Sentry." + }, + "presentationHint": { + "type": "string" + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "resultType": { + "type": "string", + "enum": ["samples", "aggregates", "timeSeries"] + }, + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "values": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {}, + "description": "The requested fields, keyed by field name." + }, + "links": { + "type": "object", + "properties": { + "profileUrl": { + "type": ["string", "null"] + }, + "traceUrl": { + "type": ["string", "null"] + } + }, + "required": ["profileUrl", "traceUrl"], + "additionalProperties": false + } + }, + "required": ["values", "links"], + "additionalProperties": false + } + }, + "timeSeries": { + "anyOf": [ + { + "type": "object", + "properties": { + "yAxis": { + "type": "string" + }, + "interval": { + "description": "Bucket size; null when Sentry chose it for the range.", + "type": ["string", "null"] + }, + "total": { + "description": "Sum of all buckets; null for non-additive aggregates.", + "type": ["number", "null"] + }, + "totalIncludesIncompleteBuckets": { + "type": "boolean" + }, + "peak": { + "anyOf": [ + { + "type": "object", + "properties": { + "time": { + "type": "string" + }, + "value": { + "type": ["number", "string"] + } + }, + "required": ["time", "value"], + "additionalProperties": false + }, + { + "type": "null" + } + ], + "description": "Highest bucket that is no longer receiving data." + }, + "ingestion": { + "anyOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "delaySeconds": { + "type": ["number", "null"] + }, + "completeThrough": { + "type": ["string", "null"] + } + }, + "required": ["status", "delaySeconds", "completeThrough"], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "bucketCount": { + "type": "number" + }, + "buckets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "time": { + "type": "string" + }, + "value": { + "type": ["number", "string"] + }, + "incomplete": { + "type": "boolean", + "description": "Data is still arriving, so the value may rise." + }, + "outsideRetention": { + "type": "boolean", + "description": "Starts before the retention window, so older data is missing and the value will not change." + } + }, + "required": [ + "time", + "value", + "incomplete", + "outsideRetention" + ], + "additionalProperties": false + }, + "description": "The most recent buckets, oldest first." + } + }, + "required": [ + "yAxis", + "interval", + "total", + "totalIncludesIncompleteBuckets", + "peak", + "ingestion", + "bucketCount", + "buckets" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "suggestedActions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "tool_call" + }, + "toolName": { + "type": "string" + }, + "arguments": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + }, + "reason": { + "type": "string" + } + }, + "required": ["type", "toolName", "arguments", "reason"], + "additionalProperties": false + } + } + }, + "required": [ + "query", + "explanation", + "executedSearch", + "url", + "presentationHint", + "notes", + "resultType", + "results", + "timeSeries", + "suggestedActions" + ], + "additionalProperties": false + }, "requiredScopes": ["event:read"], "skills": ["inspect", "triage", "seer"], "surface": "direct" diff --git a/packages/mcp-core/src/tools/catalog/search-profiles.structured.test.ts b/packages/mcp-core/src/tools/catalog/search-profiles.structured.test.ts new file mode 100644 index 000000000..49fe6cb0e --- /dev/null +++ b/packages/mcp-core/src/tools/catalog/search-profiles.structured.test.ts @@ -0,0 +1,279 @@ +import { mswServer } from "@sentry/mcp-server-mocks"; +import { HttpResponse, http } from "msw"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { getServerContext } from "../../test-setup.js"; +import { + assertStructuredOnlyResult, + getStructuredContent, +} from "../../test-utils/structured-content.js"; +import type { SearchEventsStructuredContent } from "../support/search-events/structured"; +import searchProfiles, { searchProfilesOutputSchema } from "./search-profiles"; + +vi.mock("../../internal/agents/provider-factory", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("../../internal/agents/provider-factory") + >()), + hasAgentProvider: () => false, +})); + +vi.mock("../support/search-events/seer", async (importOriginal) => ({ + ...(await importOriginal()), + translateWithSeer: vi.fn(async () => null), + trackAiQueryOutcome: vi.fn(), +})); + +type SearchParams = Parameters[0]; + +const baseParams: Omit = { + organizationSlug: "test-org", + regionUrl: null, + projectSlug: null, + fields: null, + sort: null, + period: "24h", + limit: 10, + includeExplanation: false, +}; + +function mockEvents(data: Record[]) { + mswServer.use( + http.get("https://sentry.io/api/0/organizations/test-org/events/", () => + HttpResponse.json({ data }), + ), + ); +} + +async function runSearch( + params: Partial & { query: string }, + context = getServerContext(), +) { + const result = await searchProfiles.handler( + { ...baseParams, ...params }, + context, + ); + assertStructuredOnlyResult(result); + const structuredContent = + getStructuredContent(result); + expect(searchProfilesOutputSchema.parse(structuredContent)).toEqual( + structuredContent, + ); + return structuredContent; +} + +describe("search_profiles structured output", () => { + beforeEach(() => { + mswServer.use( + http.get( + "https://sentry.io/api/0/organizations/test-org/events/validate/", + () => + HttpResponse.json({ + valid: true, + projects: [], + dataset: [], + environment: [], + field: [], + query: { valid: true, error: null, fields: [] }, + orderby: [], + }), + ), + ); + }); + + it("returns requested fields, links, and a follow-up", async () => { + mockEvents([ + { + project: "backend", + "profile.id": "a1b2c3d4e5f60718293a4b5c6d7e8f90", + "profiler.id": "", + "thread.id": "", + timestamp: "2025-01-15T10:30:00+00:00", + transaction: "/api/users", + "transaction.duration": 152000000, + release: "backend@1.2.3", + environment: "production", + trace: "6a477f5b0f31ef7b6b9b5e1dea66c91d", + "precise.start_ts": "", + "precise.finish_ts": "", + secretBackendField: "must-not-leak", + }, + ]); + + const structuredContent = await runSearch({ + query: "transaction:/api/users", + }); + + expect(JSON.stringify(structuredContent)).not.toContain("must-not-leak"); + expect(structuredContent).toMatchInlineSnapshot(` + { + "executedSearch": { + "dataset": "profiles", + "fields": [ + "project", + "profile.id", + "profiler.id", + "thread.id", + "timestamp", + "transaction", + "transaction.duration", + "release", + "environment", + "trace", + "precise.start_ts", + "precise.finish_ts", + ], + "query": "transaction:/api/users", + "sort": "-timestamp", + "timeRange": { + "end": null, + "start": null, + "statsPeriod": "24h", + }, + }, + "explanation": null, + "notes": [ + "transaction.duration values are in nanoseconds.", + ], + "presentationHint": "Concise summaries work well for these profiles, highlighting profile ID, transaction, duration, release, and trace context. Please tell the user this dashboard link is available if they want to open the results in Sentry.", + "query": "transaction:/api/users", + "resultType": "samples", + "results": [ + { + "links": { + "profileUrl": "https://test-org.sentry.io/explore/profiling/profile/backend/a1b2c3d4e5f60718293a4b5c6d7e8f90/flamegraph/", + "traceUrl": "https://test-org.sentry.io/explore/traces/trace/6a477f5b0f31ef7b6b9b5e1dea66c91d", + }, + "values": { + "environment": "production", + "precise.finish_ts": "", + "precise.start_ts": "", + "profile.id": "a1b2c3d4e5f60718293a4b5c6d7e8f90", + "profiler.id": "", + "project": "backend", + "release": "backend@1.2.3", + "thread.id": "", + "timestamp": "2025-01-15T10:30:00+00:00", + "trace": "6a477f5b0f31ef7b6b9b5e1dea66c91d", + "transaction": "/api/users", + "transaction.duration": 152000000, + }, + }, + ], + "suggestedActions": [ + { + "arguments": { + "profileUrl": "https://test-org.sentry.io/explore/profiling/profile/backend/a1b2c3d4e5f60718293a4b5c6d7e8f90/flamegraph/", + }, + "reason": "Analyze the flamegraph for the first matching profile.", + "toolName": "get_profile_details", + "type": "tool_call", + }, + { + "arguments": { + "organizationSlug": "test-org", + "resourceId": "6a477f5b0f31ef7b6b9b5e1dea66c91d", + "resourceType": "trace", + }, + "reason": "Inspect the trace for the first profiled request.", + "toolName": "get_sentry_resource", + "type": "tool_call", + }, + ], + "timeSeries": null, + "url": "https://test-org.sentry.io/explore/profiling/?query=transaction%3A%2Fapi%2Fusers&sort=-timestamp&field=project&field=profile.id&field=profiler.id&field=thread.id&field=timestamp&field=transaction&field=transaction.duration&field=release&field=environment&field=trace&field=precise.start_ts&field=precise.finish_ts&statsPeriod=24h", + } + `); + }); + + it("links continuous profiles and routes experimental follow-ups through get_sentry_resource", async () => { + mockEvents([ + { + project: "backend", + "profile.id": "", + "profiler.id": "041bde57b9844e36b8b7e5734efae5f7", + "thread.id": "259", + timestamp: "2025-01-15T10:30:00+00:00", + transaction: "/api/users", + "transaction.duration": 152000000, + release: "backend@1.2.3", + environment: "production", + trace: "", + "precise.start_ts": "2025-01-15T10:30:00.000Z", + "precise.finish_ts": "2025-01-15T10:30:01.000Z", + }, + ]); + + const structuredContent = await runSearch( + { query: "transaction:/api/users" }, + getServerContext({ experimentalMode: true }), + ); + + expect(structuredContent.results[0]?.links).toMatchInlineSnapshot(` + { + "profileUrl": "https://test-org.sentry.io/explore/profiling/profile/backend/flamegraph/?profilerId=041bde57b9844e36b8b7e5734efae5f7&start=2025-01-15T10%3A30%3A00.000Z&end=2025-01-15T10%3A30%3A01.000Z", + "traceUrl": null, + } + `); + expect(structuredContent.suggestedActions).toMatchInlineSnapshot(` + [ + { + "arguments": { + "url": "https://test-org.sentry.io/explore/profiling/profile/backend/flamegraph/?profilerId=041bde57b9844e36b8b7e5734efae5f7&start=2025-01-15T10%3A30%3A00.000Z&end=2025-01-15T10%3A30%3A01.000Z", + }, + "reason": "Analyze the flamegraph for the first matching profile.", + "toolName": "get_sentry_resource", + "type": "tool_call", + }, + ] + `); + }); + + it("returns aggregate rows", async () => { + mockEvents([ + { + transaction: "/api/users", + "count()": 42, + "p95()": 310000000, + }, + ]); + + const structuredContent = await runSearch({ + query: "transaction:/api/users", + fields: ["transaction", "count()", "p95()"], + sort: "-count()", + }); + + expect(structuredContent.resultType).toBe("aggregates"); + expect(structuredContent.results).toMatchInlineSnapshot(` + [ + { + "links": { + "profileUrl": null, + "traceUrl": null, + }, + "values": { + "count()": 42, + "p95()": 310000000, + "transaction": "/api/users", + }, + }, + ] + `); + expect(structuredContent.presentationHint).toMatchInlineSnapshot( + `"A compact table with readable duration units works well for these profile aggregates. Please tell the user this dashboard link is available if they want to open the results in Sentry."`, + ); + }); + + it("explains an empty result in notes", async () => { + mockEvents([]); + + const structuredContent = await runSearch({ + query: "transaction:/api/users", + }); + + expect(structuredContent.results).toEqual([]); + expect(structuredContent.notes).toEqual([ + "No results found. Try narrowing the transaction, release, platform, or profile filters.", + ]); + expect(structuredContent.suggestedActions).toEqual([]); + }); +}); diff --git a/packages/mcp-core/src/tools/catalog/search-profiles.test.ts b/packages/mcp-core/src/tools/catalog/search-profiles.test.ts index 6230739db..6d995d81e 100644 --- a/packages/mcp-core/src/tools/catalog/search-profiles.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-profiles.test.ts @@ -25,6 +25,7 @@ it("search_profiles runs the shared handler locked to profiles", async () => { ], "options": { "lockDataset": true, + "output": "structured", }, } `); diff --git a/packages/mcp-core/src/tools/catalog/search-profiles.ts b/packages/mcp-core/src/tools/catalog/search-profiles.ts index 208352ed0..3d0279489 100644 --- a/packages/mcp-core/src/tools/catalog/search-profiles.ts +++ b/packages/mcp-core/src/tools/catalog/search-profiles.ts @@ -5,6 +5,10 @@ import { runSearchEvents, searchToolBase, } from "../support/search-events/search"; +import { buildSearchEventsOutputSchema } from "../support/search-events/structured"; + +export const searchProfilesOutputSchema = + buildSearchEventsOutputSchema("profiles"); export default defineTool({ ...searchToolBase("search_profiles"), @@ -27,9 +31,11 @@ export default defineTool({ "", ].join("\n"), inputSchema: buildDatasetSearchInputSchema(), + outputSchema: searchProfilesOutputSchema, async handler(params, context: ServerContext) { return await runSearchEvents({ ...params, dataset: "profiles" }, context, { lockDataset: true, + output: "structured", }); }, }); diff --git a/packages/mcp-core/src/tools/support/search-events/formatters.ts b/packages/mcp-core/src/tools/support/search-events/formatters.ts index fdb94abbb..cdcb6a92f 100644 --- a/packages/mcp-core/src/tools/support/search-events/formatters.ts +++ b/packages/mcp-core/src/tools/support/search-events/formatters.ts @@ -125,6 +125,15 @@ export const METRIC_RESULT_HINTS: SearchResultHints = { empty: "Try being more specific about the metric name, type, or filters.", }; +export const PROFILE_RESULT_HINTS: SearchResultHints = { + samples: + "Concise summaries work well for these profiles, highlighting profile ID, transaction, duration, release, and trace context.", + aggregates: + "A compact table with readable duration units works well for these profile aggregates.", + empty: + "Try narrowing the transaction, release, platform, or profile filters.", +}; + export const LOG_RESULT_HINTS: SearchResultHints = { samples: "Console-style formatting works well for these logs, with timestamps preserved.", @@ -627,7 +636,8 @@ function getProfileTimestampValue(value: unknown): string | null { return null; } -function getProfileDetailUrl( +/** Flamegraph URL for a transaction profile or continuous profile row. */ +export function getProfileDetailUrl( apiService: SentryApiService, organizationSlug: string, event: FlexibleEventData, @@ -671,15 +681,11 @@ export function formatProfileResults(params: FormatEventResultsParams): string { let output = `# Search Results for "${inputQuery}"\n\n`; - if (isAggregateQuery(fields)) { - output += formatSearchPresentationHint( - "A compact table with readable duration units works well for these profile aggregates.", - ); - } else { - output += formatSearchPresentationHint( - "Concise summaries work well for these profiles, highlighting profile ID, transaction, duration, release, and trace context.", - ); - } + output += formatSearchPresentationHint( + isAggregateQuery(fields) + ? PROFILE_RESULT_HINTS.aggregates + : PROFILE_RESULT_HINTS.samples, + ); if (includeExplanation && explanation) { output += formatExplanation(explanation); @@ -700,8 +706,7 @@ export function formatProfileResults(params: FormatEventResultsParams): string { }, }); output += "No results found.\n\n"; - output += - "Try narrowing the transaction, release, platform, or profile filters.\n"; + output += `${PROFILE_RESULT_HINTS.empty}\n`; return output; } diff --git a/packages/mcp-core/src/tools/support/search-events/structured.ts b/packages/mcp-core/src/tools/support/search-events/structured.ts index ab289d610..46394ae46 100644 --- a/packages/mcp-core/src/tools/support/search-events/structured.ts +++ b/packages/mcp-core/src/tools/support/search-events/structured.ts @@ -21,7 +21,9 @@ import type { PublicEventsDataset } from "../../../utils/events-datasets"; import { DASHBOARD_LINK_HINT, ERROR_RESULT_HINTS, + getProfileDetailUrl, type ExecutedSearch, + PROFILE_RESULT_HINTS, METRIC_RESULT_HINTS, SPAN_RESULT_HINTS, LOG_RESULT_HINTS, @@ -259,6 +261,53 @@ const STRUCTURED_DATASETS = { "Inspect the trace for the first matching metric sample.", ), }), + profiles: defineDatasetConfig({ + hints: PROFILE_RESULT_HINTS, + fieldUnitNotes: { + "transaction.duration": "transaction.duration values are in nanoseconds.", + "profile.duration": "profile.duration values are in nanoseconds.", + }, + links: { + profileUrl: z.string().nullable(), + traceUrl: z.string().nullable(), + }, + rowLinks: (event, session) => ({ + profileUrl: getProfileDetailUrl( + session.apiService, + session.organizationSlug, + event, + ), + traceUrl: traceUrlFor(event, session), + }), + suggestedActions(results, session) { + const profileUrl = results.find((row) => row.links.profileUrl)?.links + .profileUrl; + const reason = "Analyze the flamegraph for the first matching profile."; + // Experimental sessions route profile URLs through get_sentry_resource. + const profileAction = profileUrl + ? buildSuggestedToolAction({ + ...(session.experimentalMode + ? { + toolName: "get_sentry_resource", + arguments: { url: profileUrl }, + } + : { toolName: "get_profile_details", arguments: { profileUrl } }), + reason, + experimentalMode: session.experimentalMode, + availableToolNames: session.availableToolNames, + directToolNames: session.directToolNames, + }) + : null; + return [ + ...(profileAction ? [profileAction] : []), + ...firstTraceAction( + results, + session, + "Inspect the trace for the first profiled request.", + ), + ]; + }, + }), } satisfies Partial>; export type StructuredSearchDataset = keyof typeof STRUCTURED_DATASETS;