Skip to content

feat(mcp): Add MCP server for Sentry docs with AI-powered evals #16235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sergical wants to merge 5 commits into
base: master
Choose a base branch
from
+1,792 −9
Open

feat(mcp): Add MCP server for Sentry docs with AI-powered evals #16235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0f853e6
feat(mcp): Add AI-powered evaluation tests for MCP server tools
sergical Jan 31, 2026
042911d
feat(mcp): Add MCP server implementation for Sentry docs
sergical Jan 31, 2026
3c1610f
fix(mcp): Use iterative HTML tag stripping to prevent incomplete sani…
sergical Jan 31, 2026
b8431d2
fix(mcp): Add error handling to search_docs and fix ESLint require-aw…
sergical Jan 31, 2026
5456874
[getsentry/action-github-commit] Auto commit
getsantry[bot] Jan 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions app/mcp/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import {createMcpHandler} from 'mcp-handler';

import {registerTools} from '../../src/mcp';

// Trailing slash handling is done in middleware
const handler = createMcpHandler(
server => {
registerTools(server);
},
{
serverInfo: {
name: 'sentry-docs-mcp',
version: '1.0.0',
},
},
{
basePath: '/',
maxDuration: 60,
disableSse: true, // Stateless HTTP only - no Redis required
}
);

export {handler as GET, handler as POST};
1 change: 1 addition & 0 deletions next.config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ if (process.env.NODE_ENV !== 'development' && !process.env.NEXT_PUBLIC_SENTRY_DS
const nextConfig = {
pageExtensions: ['js', 'jsx', 'mdx', 'ts', 'tsx', 'mdx'],
trailingSlash: true,
skipTrailingSlashRedirect: true,
serverExternalPackages: [
'rehype-preset-minify',
'esbuild',
Expand Down
10 changes: 9 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
"sidecar": "yarn spotlight-sidecar",
"test": "vitest",
"test:ci": "vitest run",
"eval": "vitest --config=src/mcp/evals/vitest.config.ts",
"eval:run": "vitest run --config=src/mcp/evals/vitest.config.ts",
"enforce-redirects": "node ./scripts/no-vercel-json-redirects.mjs"
},
"dependencies": {
Expand All @@ -45,6 +47,7 @@
"@google-cloud/storage": "^7.7.0",
"@mdx-js/loader": "^3.0.0",
"@mdx-js/react": "^3.0.0",
"@modelcontextprotocol/sdk": "^1.25.3",
"@pondorasti/remark-img-links": "^1.0.8",
"@popperjs/core": "^2.11.8",
"@prettier/plugin-xml": "^3.3.1",
Expand Down Expand Up @@ -73,6 +76,7 @@
"js-cookie": "^3.0.5",
"js-yaml": "^4.1.0",
"match-sorter": "^6.3.4",
"mcp-handler": "^1.0.7",
"mdx-bundler": "^10.0.1",
"mermaid": "^11.11.0",
"micromark": "^4.0.0",
Expand Down Expand Up @@ -113,9 +117,11 @@
"tailwindcss-scoped-preflight": "^3.0.4",
"textarea-markdown-editor": "^1.0.4",
"unified": "^11.0.5",
"unist-util-remove": "^4.0.0"
"unist-util-remove": "^4.0.0",
"zod": "^4.3.6"
},
"devDependencies": {
"@ai-sdk/anthropic": "^3.0.33",
"@babel/preset-typescript": "^7.15.0",
"@codecov/nextjs-webpack-plugin": "^1.9.0",
"@spotlightjs/spotlight": "^2.5.0",
Expand All @@ -126,6 +132,7 @@
"@types/react": "18.3.12",
"@types/react-dom": "18.3.1",
"@types/ws": "^8.5.10",
"ai": "^6.0.65",
"autoprefixer": "^10.4.17",
"concurrently": "^9.1.0",
"dotenv-cli": "^7.4.1",
Expand All @@ -142,6 +149,7 @@
"typescript": "^5",
"vite-tsconfig-paths": "^5.0.1",
"vitest": "^3.0.7",
"vitest-evals": "^0.5.0",
"ws": "^8.17.1"
},
"resolutions": {
Expand Down
13 changes: 13 additions & 0 deletions src/mcp/evals/setup.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import {config} from 'dotenv';

// Load environment variables from .env.local or .env
config({path: '.env.local'});
config({path: '.env'});

// Verify ANTHROPIC_API_KEY is set
if (!process.env.ANTHROPIC_API_KEY) {
console.warn(

Check warning on line 9 in src/mcp/evals/setup.ts

View workflow job for this annotation

GitHub Actions / Lint 

Unexpected console statement
'Warning: ANTHROPIC_API_KEY is not set. Evals will fail without it.\n' +
'Set it in .env.local: ANTHROPIC_API_KEY=sk-ant-...'
);
}
67 changes: 67 additions & 0 deletions src/mcp/evals/tests/get-doc-tree.eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import {describeEval} from 'vitest-evals';

import {NoOpTaskRunner, ToolPredictionScorer} from '../utils';

describeEval('get-doc-tree', {
data: () =>
Promise.resolve([
{
input: 'Show me the documentation structure',
expectedTools: [
{
name: 'get_doc_tree',
arguments: {},
},
],
},
{
input: 'What sections are available in the JavaScript documentation?',
expectedTools: [
{
name: 'get_doc_tree',
arguments: {path: 'platforms/javascript'},
},
],
},
{
input: 'Navigate the Python SDK docs, show me the table of contents',
expectedTools: [
{
name: 'get_doc_tree',
arguments: {path: 'platforms/python'},
},
],
},
{
input:
'I want to explore the configuration options, show me what subsections exist',
expectedTools: [
{
name: 'get_doc_tree',
arguments: {path: 'configuration', depth: 2},
},
],
},
{
input: 'Give me a deep overview of all pages under product/sentry-basics',
expectedTools: [
{
name: 'get_doc_tree',
arguments: {path: 'product/sentry-basics', depth: 3},
},
],
},
{
input: 'Show me just the top-level categories of documentation',
expectedTools: [
{
name: 'get_doc_tree',
arguments: {depth: 1},
},
],
},
]),
task: NoOpTaskRunner(),
scorers: [ToolPredictionScorer()],
threshold: 0.6,
});
58 changes: 58 additions & 0 deletions src/mcp/evals/tests/get-doc.eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import {describeEval} from 'vitest-evals';

import {NoOpTaskRunner, ToolPredictionScorer} from '../utils';

describeEval('get-doc', {
data: () =>
Promise.resolve([
{
input: 'Get the full documentation for /platforms/javascript/guides/nextjs',
expectedTools: [
{
name: 'get_doc',
arguments: {path: '/platforms/javascript/guides/nextjs'},
},
],
},
{
input: 'Read the Python SDK setup guide at platforms/python',
expectedTools: [
{
name: 'get_doc',
arguments: {path: 'platforms/python'},
},
],
},
{
input: 'Fetch the configuration page for the Go SDK',
expectedTools: [
{
name: 'get_doc',
arguments: {path: '/platforms/go/configuration'},
},
],
},
{
input: 'Get the developer documentation for envelope formats',
expectedTools: [
{
name: 'get_doc',
arguments: {path: 'envelope', site: 'develop'},
},
],
},
{
input:
'I found a search result for /platforms/javascript/configuration/sampling, can you get the full content?',
expectedTools: [
{
name: 'get_doc',
arguments: {path: '/platforms/javascript/configuration/sampling'},
},
],
},
]),
task: NoOpTaskRunner(),
scorers: [ToolPredictionScorer()],
threshold: 0.6,
});
66 changes: 66 additions & 0 deletions src/mcp/evals/tests/list-platforms.eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import {describeEval} from 'vitest-evals';

import {NoOpTaskRunner, ToolPredictionScorer} from '../utils';

describeEval('list-platforms', {
data: () =>
Promise.resolve([
{
input: 'What platforms does Sentry support?',
expectedTools: [
{
name: 'list_platforms',
arguments: {},
},
],
},
{
input: 'List all available SDKs',
expectedTools: [
{
name: 'list_platforms',
arguments: {},
},
],
},
{
input: 'What frameworks are available for JavaScript?',
expectedTools: [
{
name: 'list_platforms',
arguments: {platform: 'javascript'},
},
],
},
{
input: 'Show me the guides available for the Python SDK',
expectedTools: [
{
name: 'list_platforms',
arguments: {platform: 'python'},
},
],
},
{
input: 'Does Sentry support React Native?',
expectedTools: [
{
name: 'list_platforms',
arguments: {platform: 'react-native'},
},
],
},
{
input: 'What mobile SDKs does Sentry have?',
expectedTools: [
{
name: 'list_platforms',
arguments: {},
},
],
},
]),
task: NoOpTaskRunner(),
scorers: [ToolPredictionScorer()],
threshold: 0.6,
});
55 changes: 55 additions & 0 deletions src/mcp/evals/tests/search-docs.eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import {describeEval} from 'vitest-evals';

import {NoOpTaskRunner, ToolPredictionScorer} from '../utils';

describeEval('search-docs', {
data: () =>
Promise.resolve([
{
input: 'How do I set up Sentry in my Next.js app?',
expectedTools: [
{
name: 'search_docs',
arguments: {query: 'setup Next.js', guide: 'javascript/nextjs'},
},
],
},
{
input: 'Show me the error handling configuration for Python',
expectedTools: [
{name: 'search_docs', arguments: {query: 'error handling Python'}},
],
},
{
input: 'How do I configure the tracesSampleRate option?',
expectedTools: [
{name: 'search_docs', arguments: {query: 'tracesSampleRate configuration'}},
],
},
{
input: 'Find documentation about beforeSend hook',
expectedTools: [{name: 'search_docs', arguments: {query: 'beforeSend hook'}}],
},
{
input: 'How do I add breadcrumbs to my errors in React?',
expectedTools: [
{
name: 'search_docs',
arguments: {query: 'breadcrumbs', guide: 'javascript/react'},
},
],
},
{
input: 'Search the developer docs for SDK architecture',
expectedTools: [
{
name: 'search_docs',
arguments: {query: 'SDK architecture', site: 'develop'},
},
],
},
]),
task: NoOpTaskRunner(),
scorers: [ToolPredictionScorer()],
threshold: 0.6,
});
2 changes: 2 additions & 0 deletions src/mcp/evals/utils/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export {NoOpTaskRunner} from './runner';
export {ToolPredictionScorer, type ExpectedToolCall} from './toolPredictionScorer';
13 changes: 13 additions & 0 deletions src/mcp/evals/utils/runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/**
* NoOpTaskRunner - A task runner that does nothing.
*
* In tool prediction evals, we don't actually execute the tools.
* We just verify that the AI correctly predicts which tools would be called.
* The scorer handles the actual prediction logic.
*/
export function NoOpTaskRunner() {
return function runner(input: string) {
// Simply return the input - the actual evaluation happens in the scorer
return Promise.resolve(input);
};
}
Loading
Loading