diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3663fd7..a7adf9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,27 +1,43 @@ name: CI on: - push: - branches: [main] pull_request: branches: [main] + push: + branches: [main] jobs: - lint: + test: + name: Test and Lint runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v4 - - name: Install pnpm - uses: pnpm/action-setup@v4 - - name: Use Node.js 22 + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: "22" - cache: "pnpm" + cache: "npm" + - name: Install dependencies - run: pnpm install --frozen-lockfile - - run: pnpm run build - - name: Run ESLint - run: pnpm run lint - - name: Ensure no changes - run: git diff --exit-code + run: npm ci + + - name: Run linting + run: npm run lint + + - name: Check formatting + run: npm run format + + - name: Build project + run: npm run build + + - name: Run evaluation tests + env: + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + npm test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 804b2d2..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Publish -on: - release: - types: [published] -jobs: - publish-npm: - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - registry-url: https://registry.npmjs.org/ - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 8 - - run: pnpm install --frozen-lockfile - - run: pnpm publish --provenance - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.husky/pre-commit b/.husky/pre-commit index 0a85043..74e3413 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,2 +1,4 @@ #!/bin/sh -pnpm pre-commit \ No newline at end of file +npm install +npm run build +npm run pre-commit \ No newline at end of file diff --git a/README.md b/README.md index 9c9cce2..edb573b 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ git clone https://github.com/browserbase/mcp-server-browserbase.git cd mcp-server-browserbase # Install the dependencies and build the project -pnpm install && pnpm build +npm install && npm run build ``` Then in your MCP Config JSON run the server. To run locally we can use STDIO or self-host SHTTP. diff --git a/evals/mcp-eval-basic.config.json b/evals/mcp-eval-basic.config.json new file mode 100644 index 0000000..f59c29d --- /dev/null +++ b/evals/mcp-eval-basic.config.json @@ -0,0 +1,142 @@ +{ + "passThreshold": 0.7, + "server": { + "transport": "stdio", + "command": "node", + "args": ["./cli.js"], + "env": { + "BROWSERBASE_API_KEY": "${BROWSERBASE_API_KEY}", + "BROWSERBASE_PROJECT_ID": "${BROWSERBASE_PROJECT_ID}", + "GEMINI_API_KEY": "${GEMINI_API_KEY}" + } + }, + "timeout": 180000, + "llmJudge": false, + "workflows": [ + { + "name": "basic-navigation-test", + "description": "Test basic browser navigation functionality", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_session_close" + ] + }, + { + "name": "search-and-extract-test", + "description": "Test navigation, search interaction, and data extraction", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, extract the page title, and close the session", + "expectedState": "Example Domain" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_extract", + "browserbase_session_close" + ] + }, + { + "name": "observe-and-interact-test", + "description": "Test element observation and interaction capabilities", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, observe the page elements, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_observe", + "browserbase_session_close" + ] + }, + { + "name": "screenshot-test", + "description": "Test screenshot functionality", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, take a screenshot, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_screenshot", + "browserbase_session_close" + ] + }, + { + "name": "multi-session-test", + "description": "Test multi-session browser management", + "steps": [ + { + "user": "Create a multi-session browser named 'test-session', list all sessions, navigate to https://example.com in that session, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "multi_browserbase_stagehand_session_create", + "multi_browserbase_stagehand_session_list", + "multi_browserbase_stagehand_navigate_session", + "multi_browserbase_stagehand_session_close" + ] + }, + { + "name": "form-interaction-test", + "description": "Test form filling and submission capabilities", + "steps": [ + { + "user": "Create a browser session, navigate to https://httpbin.org/forms/post, fill in the customer name field with 'TestUser', and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_act", + "browserbase_session_close" + ] + }, + { + "name": "error-handling-test", + "description": "Test error handling for invalid operations", + "steps": [ + { + "user": "Create a browser session and try to navigate to an invalid URL like 'invalid-url-test'", + "expectedState": "error" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate" + ] + }, + { + "name": "url-retrieval-test", + "description": "Test URL retrieval functionality", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, get the current URL to verify navigation, and close the session", + "expectedState": "https://example.com" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_get_url", + "browserbase_session_close" + ] + } + ] +} diff --git a/evals/mcp-eval-minimal.config.json b/evals/mcp-eval-minimal.config.json new file mode 100644 index 0000000..f5251fd --- /dev/null +++ b/evals/mcp-eval-minimal.config.json @@ -0,0 +1,95 @@ +{ + "passThreshold": 0.7, + "server": { + "transport": "stdio", + "command": "node", + "args": ["./cli.js"], + "env": { + "BROWSERBASE_API_KEY": "${BROWSERBASE_API_KEY}", + "BROWSERBASE_PROJECT_ID": "${BROWSERBASE_PROJECT_ID}", + "GEMINI_API_KEY": "${GEMINI_API_KEY}" + } + }, + "timeout": 60000, + "llmJudge": false, + "workflows": [ + { + "name": "smoke-test-navigation", + "description": "Quick test to verify basic navigation works", + "steps": [ + { + "user": "Open a browser and go to example.org", + "expectedState": "session created" + }, + { + "user": "Close the browser", + "expectedState": "session closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_session_close" + ] + }, + { + "name": "smoke-test-extraction", + "description": "Quick test to verify data extraction works", + "steps": [ + { + "user": "Navigate to example.org and extract the page title", + "expectedState": "Example Domain" + }, + { + "user": "Close the session", + "expectedState": "session closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_extract", + "browserbase_session_close" + ] + }, + { + "name": "smoke-test-multi-session", + "description": "Quick test to verify multi-session functionality", + "steps": [ + { + "user": "Create a browser session named 'test-session'", + "expectedState": "session created" + }, + { + "user": "List active sessions", + "expectedState": "test-session" + }, + { + "user": "Close the test session", + "expectedState": "closed session" + } + ], + "expectTools": [ + "multi_browserbase_stagehand_session_create", + "multi_browserbase_stagehand_session_list", + "multi_browserbase_stagehand_session_close" + ] + }, + { + "name": "smoke-test-url-tools", + "description": "Quick test to verify URL retrieval tools work", + "steps": [ + { + "user": "Create a browser session, navigate to example.org, get the current URL, and close the session", + "expectedState": "example.org" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_get_url", + "browserbase_session_close" + ] + } + ] +} diff --git a/evals/mcp-eval.config.json b/evals/mcp-eval.config.json new file mode 100644 index 0000000..f59c29d --- /dev/null +++ b/evals/mcp-eval.config.json @@ -0,0 +1,142 @@ +{ + "passThreshold": 0.7, + "server": { + "transport": "stdio", + "command": "node", + "args": ["./cli.js"], + "env": { + "BROWSERBASE_API_KEY": "${BROWSERBASE_API_KEY}", + "BROWSERBASE_PROJECT_ID": "${BROWSERBASE_PROJECT_ID}", + "GEMINI_API_KEY": "${GEMINI_API_KEY}" + } + }, + "timeout": 180000, + "llmJudge": false, + "workflows": [ + { + "name": "basic-navigation-test", + "description": "Test basic browser navigation functionality", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_session_close" + ] + }, + { + "name": "search-and-extract-test", + "description": "Test navigation, search interaction, and data extraction", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, extract the page title, and close the session", + "expectedState": "Example Domain" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_extract", + "browserbase_session_close" + ] + }, + { + "name": "observe-and-interact-test", + "description": "Test element observation and interaction capabilities", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, observe the page elements, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_observe", + "browserbase_session_close" + ] + }, + { + "name": "screenshot-test", + "description": "Test screenshot functionality", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, take a screenshot, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_screenshot", + "browserbase_session_close" + ] + }, + { + "name": "multi-session-test", + "description": "Test multi-session browser management", + "steps": [ + { + "user": "Create a multi-session browser named 'test-session', list all sessions, navigate to https://example.com in that session, and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "multi_browserbase_stagehand_session_create", + "multi_browserbase_stagehand_session_list", + "multi_browserbase_stagehand_navigate_session", + "multi_browserbase_stagehand_session_close" + ] + }, + { + "name": "form-interaction-test", + "description": "Test form filling and submission capabilities", + "steps": [ + { + "user": "Create a browser session, navigate to https://httpbin.org/forms/post, fill in the customer name field with 'TestUser', and close the session", + "expectedState": "closed" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_act", + "browserbase_session_close" + ] + }, + { + "name": "error-handling-test", + "description": "Test error handling for invalid operations", + "steps": [ + { + "user": "Create a browser session and try to navigate to an invalid URL like 'invalid-url-test'", + "expectedState": "error" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate" + ] + }, + { + "name": "url-retrieval-test", + "description": "Test URL retrieval functionality", + "steps": [ + { + "user": "Create a browser session, navigate to https://example.com, get the current URL to verify navigation, and close the session", + "expectedState": "https://example.com" + } + ], + "expectTools": [ + "browserbase_session_create", + "browserbase_stagehand_navigate", + "browserbase_stagehand_get_url", + "browserbase_session_close" + ] + } + ] +} diff --git a/evals/run-evals.ts b/evals/run-evals.ts new file mode 100644 index 0000000..37b9466 --- /dev/null +++ b/evals/run-evals.ts @@ -0,0 +1,281 @@ +#!/usr/bin/env tsx + +import { Command } from "commander"; +import * as fs from "fs/promises"; +import * as path from "path"; +import { evaluate } from "mcpvals"; +import os from "os"; +import chalk from "chalk"; + +// Load environment variables from .env file +import { config } from "dotenv"; +config(); + +// Types for evaluation results +interface EvaluationResult { + workflowName: string; + passed: boolean; + overallScore: number; + results: Array<{ + metric: string; + passed: boolean; + score: number; + details: string; + metadata?: Record; + }>; +} + +interface EvaluationReport { + config: Record; + evaluations: EvaluationResult[]; + passed: boolean; + timestamp: Date; +} + +interface TestResult { + config: string; + passed: boolean; + score: number; + duration: number; + workflows: { + name: string; + passed: boolean; + score: number; + }[]; +} + +interface EvalConfig { + workflows: Array<{ name?: string }>; + passThreshold?: number; + [key: string]: unknown; +} + +const program = new Command(); + +program + .name("browserbase-mcp-evals") + .description("Run evaluation tests for Browserbase MCP Server") + .version("1.0.0"); + +program + .command("run") + .description("Run evaluation tests") + .option( + "-c, --config ", + "Config file path", + "./evals/mcp-eval.config.json", + ) + .option("-d, --debug", "Enable debug output") + .option("-j, --json", "Output results as JSON") + .option("-l, --llm", "Enable LLM judge") + .option("-o, --output ", "Save results to file") + .option( + "-p, --pass-threshold ", + "Minimum average score (0-1) required to pass. Can also be set via EVAL_PASS_THRESHOLD env var.", + ) + .option("-t, --timeout ", "Override timeout in milliseconds") + .action(async (options) => { + try { + const startTime = Date.now(); + + // Check for required environment variables + const requiredEnvVars = [ + "BROWSERBASE_API_KEY", + "BROWSERBASE_PROJECT_ID", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + ]; + const missingVars = requiredEnvVars.filter((v) => !process.env[v]); + + if (missingVars.length > 0) { + console.error( + chalk.red( + `Missing required environment variables: ${missingVars.join(", ")}`, + ), + ); + console.error( + chalk.yellow("Please set them before running the tests."), + ); + console.error(chalk.yellow("Example:")); + + for (const missingVar of missingVars) { + switch (missingVar) { + case "BROWSERBASE_API_KEY": + console.error( + chalk.yellow( + " export BROWSERBASE_API_KEY='your_api_key_here'", + ), + ); + break; + case "BROWSERBASE_PROJECT_ID": + console.error( + chalk.yellow( + " export BROWSERBASE_PROJECT_ID='your_project_id_here'", + ), + ); + break; + case "ANTHROPIC_API_KEY": + console.error( + chalk.yellow( + " export ANTHROPIC_API_KEY='sk-ant-your_key_here'", + ), + ); + break; + case "GEMINI_API_KEY": + console.error( + chalk.yellow(" export GEMINI_API_KEY='your_gemini_key_here'"), + ); + break; + } + } + process.exit(1); + } + + // Check for LLM judge requirements + if (options.llm && !process.env.OPENAI_API_KEY) { + console.error( + chalk.red("LLM judge requires OPENAI_API_KEY environment variable"), + ); + process.exit(1); + } + + // Resolve config path + const configPath = path.resolve(options.config); + + // Load config to get workflow count for display + const configContent = await fs.readFile(configPath, "utf-8"); + const config: EvalConfig = JSON.parse(configContent); + + console.log(chalk.blue(`Running evaluation tests from: ${configPath}`)); + console.log(chalk.gray(`Workflows to test: ${config.workflows.length}`)); + + // Prepare evaluation options + const evalOptions = { + debug: options.debug, + reporter: (options.json ? "json" : "console") as + | "json" + | "console" + | "junit" + | undefined, + llmJudge: options.llm, + timeout: options.timeout ? parseInt(options.timeout) : undefined, + }; + + console.log( + chalk.yellow( + "Parallel mode: splitting workflows and running concurrently", + ), + ); + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "mcp-evals-")); + + const workflowFiles: string[] = []; + for (let i = 0; i < config.workflows.length; i++) { + const wf = config.workflows[i]; + const wfConfig = { ...config, workflows: [wf] }; + const wfPath = path.join( + tmpDir, + `workflow-${i}-${(wf.name || "unnamed").replace(/[^a-z0-9_-]/gi, "_")}.json`, + ); + await fs.writeFile(wfPath, JSON.stringify(wfConfig, null, 2)); + workflowFiles.push(wfPath); + } + + const reports: EvaluationReport[] = await Promise.all( + workflowFiles.map((wfPath) => evaluate(wfPath, evalOptions)), + ); + + // Aggregate results + const allEvaluations = reports.flatMap((r) => r.evaluations); + const duration = Date.now() - startTime; + + // Determine pass/fail based on threshold instead of strict all-pass + const avgScore = + allEvaluations.length === 0 + ? 0 + : allEvaluations.reduce((sum, e) => sum + e.overallScore, 0) / + allEvaluations.length; + + const thresholdFromEnv = + (process.env.EVAL_PASS_THRESHOLD || process.env.PASS_THRESHOLD) ?? ""; + const thresholdFromCli = options.passThreshold ?? ""; + const thresholdFromConfig = + typeof config.passThreshold === "number" + ? String(config.passThreshold) + : ""; + const threshold = (() => { + const raw = String( + thresholdFromCli || thresholdFromEnv || thresholdFromConfig, + ).trim(); + const parsed = Number.parseFloat(raw); + if (!Number.isFinite(parsed)) return 0.6; // default lowered threshold + return parsed; + })(); + + const passed = avgScore >= threshold; + + const finalReport: EvaluationReport = { + config: { parallel: true, source: configPath }, + evaluations: allEvaluations, + passed, + timestamp: new Date(), + }; + + const finalResult: TestResult = { + config: configPath, + passed, + score: avgScore, + duration, + workflows: allEvaluations.map((e) => ({ + name: e.workflowName, + passed: e.passed, + score: e.overallScore, + })), + }; + + // Best-effort cleanup + try { + await Promise.all(workflowFiles.map((f) => fs.unlink(f))); + await fs.rmdir(tmpDir); + } catch { + // ignore cleanup errors + } + + // Output results + if (options.json) { + console.log(JSON.stringify(finalResult, null, 2)); + } else { + console.log( + chalk.green( + `\nTest execution completed in ${(finalResult.duration / 1000).toFixed(2)}s`, + ), + ); + console.log( + chalk.gray( + `Threshold for pass: ${threshold.toFixed(2)} | Average score: ${finalResult.score.toFixed(3)}`, + ), + ); + console.log( + chalk[finalResult.passed ? "green" : "red"]( + `Overall result: ${finalResult.passed ? "PASSED" : "FAILED"} (${(finalResult.score * 100).toFixed(1)}%)`, + ), + ); + } + + // Save to file if requested + if (options.output) { + await fs.writeFile( + options.output, + JSON.stringify(finalReport, null, 2), + ); + console.log(chalk.gray(`Results saved to: ${options.output}`)); + } + + process.exit(finalResult.passed ? 0 : 1); + } catch (error) { + console.error("Error running evaluation tests:", error); + process.exit(1); + } + }); + +program.parse(); diff --git a/package-lock.json b/package-lock.json index 7f1938d..30f4106 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@smithery/cli": "^1.2.15", "commander": "^14.0.0", "dotenv": "^16.4.6", + "mcpvals": "^0.0.3", "playwright-core": "^1.53.2", "zod": "^3.25.67" }, @@ -40,7 +41,6 @@ "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-1.2.12.tgz", "integrity": "sha512-YSzjlko7JvuiyQFmI9RN1tNZdEiZxc+6xld/0tq/VkJaHpEzGAb1yiNxxvmYVcjvfu/PcvCxAAYXmTYQQ63IHQ==", "license": "Apache-2.0", - "optional": true, "dependencies": { "@ai-sdk/provider": "1.1.3", "@ai-sdk/provider-utils": "2.2.8" @@ -162,7 +162,6 @@ "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-1.3.24.tgz", "integrity": "sha512-GYXnGJTHRTZc4gJMSmFRgEQudjqd4PUN0ZjQhPwOAYH1yOAvQoG/Ikqs+HyISRbLPCrhbZnPKCNHuRU4OfpW0Q==", "license": "Apache-2.0", - "optional": true, "dependencies": { "@ai-sdk/provider": "1.1.3", "@ai-sdk/provider-utils": "2.2.8" @@ -2107,6 +2106,12 @@ ], "license": "MIT" }, + "node_modules/basic-auth": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-0.0.1.tgz", + "integrity": "sha512-sCz6E05DMvrA9dUBGJFfnQ3qs+/lQkVr7qjOT5XMMNfpTzWbpkElpzXfnbNlBjPnDQyz0uBFJ4nELJRIdcKoNQ==", + "license": "MIT" + }, "node_modules/bignumber.js": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", @@ -2127,6 +2132,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", @@ -2417,6 +2428,32 @@ "node": ">=0.8" } }, + "node_modules/co-bluebird": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/co-bluebird/-/co-bluebird-1.1.0.tgz", + "integrity": "sha512-JuoemMXxQjYAxbfRrNpOsLyiwDiY8mXvGqJyYLM7jMySDJtnMklW3V2o8uyubpc1eN2YoRsAdfZ1lfKCd3lsrA==", + "dependencies": { + "bluebird": "^2.10.0", + "co-use": "^1.1.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/co-bluebird/node_modules/bluebird": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", + "integrity": "sha512-UfFSr22dmHPQqPP9XWHRhq+gWnHCYguQGkXQlbyPtW5qTnhFWA8/iXg765tH0cAjy7l/zPJ1aBTO0g5XgA7kvQ==", + "license": "MIT" + }, + "node_modules/co-use": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/co-use/-/co-use-1.1.0.tgz", + "integrity": "sha512-1lVRtdywv41zQO/xvI2wU8w6oFcUYT6T84YKSxN25KN4N4Kld3scLovt8FjDmD63Cm7HtyRWHjezt+IanXmkyA==", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -3286,6 +3323,80 @@ "node": ">=20.0.0" } }, + "node_modules/execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/express": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz", @@ -3787,6 +3898,18 @@ "node": ">= 0.4" } }, + "node_modules/get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-symbol-description": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz", @@ -4075,6 +4198,15 @@ "node": ">= 14" } }, + "node_modules/human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=16.17.0" + } + }, "node_modules/humanize-ms": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", @@ -4500,6 +4632,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-generator": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-generator/-/is-generator-1.0.3.tgz", + "integrity": "sha512-G56jBpbJeg7ds83HW1LuShNs8J73Fv3CPz/bmROHOHlnKkN8sWb9ujiagjmxxMUywftgq48HlBZELKKqFLk0oA==", + "license": "MIT" + }, "node_modules/is-generator-function": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.0.tgz", @@ -5421,6 +5559,48 @@ "node": ">= 0.4" } }, + "node_modules/mcpvals": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/mcpvals/-/mcpvals-0.0.3.tgz", + "integrity": "sha512-aMOsp8K6eJNId796Y1K+S2nNHiW8wuGtMuTEB1aCpMAvT57XpK2cMu/SCoKp3vqMZMPFOWX1TS2AhSdhYaOwkw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/anthropic": "^1.2.12", + "@ai-sdk/openai": "^1.3.22", + "@modelcontextprotocol/sdk": "^1.12.0", + "ai": "^4.3.16", + "chalk": "^5.3.0", + "commander": "^11.1.0", + "execa": "^8.0.1", + "node-oauth2-server": "^2.4.0", + "oauth2-server": "^3.1.1", + "zod": "^3.25.0" + }, + "bin": { + "mcpvals": "dist/cli/index.js" + } + }, + "node_modules/mcpvals/node_modules/chalk": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.5.0.tgz", + "integrity": "sha512-1tm8DTaJhPBG3bIkVeZt1iZM9GfSX2lzOeDVZH9R9ffRHpmHvxZ/QhgQH/aDTkswQVt+YHdXAdS/In/30OjCbg==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/mcpvals/node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -5442,6 +5622,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -5641,6 +5827,137 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-oauth2-server": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/node-oauth2-server/-/node-oauth2-server-2.4.0.tgz", + "integrity": "sha512-k3NUmzjEIPyKBuY1OYtHqJ2L6siIlN+oERGe1MVeUGxzeOxEq/2z5K03/P8lfW4ys0Iivbn1KlGJgBeXNZ6Z5w==", + "dependencies": { + "basic-auth": "~0.0.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "license": "MIT", + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/oauth2-server": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/oauth2-server/-/oauth2-server-3.1.1.tgz", + "integrity": "sha512-4dv+fE9hrK+xTaCygOLh/kQeFzbFr7UqSyHvBDbrQq8Hg52sAkV2vTsyH3Z42hoeaKpbhM7udhL8Y4GYbl6TGQ==", + "license": "MIT", + "dependencies": { + "basic-auth": "2.0.1", + "bluebird": "3.7.2", + "lodash": "4.17.19", + "promisify-any": "2.0.1", + "statuses": "1.5.0", + "type-is": "1.6.18" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/oauth2-server/node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/oauth2-server/node_modules/lodash": { + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "license": "MIT" + }, + "node_modules/oauth2-server/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/oauth2-server/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/oauth2-server/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/oauth2-server/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/oauth2-server/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/oauth2-server/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -6348,6 +6665,25 @@ ], "license": "MIT" }, + "node_modules/promisify-any": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promisify-any/-/promisify-any-2.0.1.tgz", + "integrity": "sha512-pVaGouFbTVxqpVJ+T5A15olNJDASAZHYq5cXz6mWdr6/X34mVWiG9MSdzHTcVBCv4aqBP7wGspi7BUSRbEmhsw==", + "dependencies": { + "bluebird": "^2.10.0", + "co-bluebird": "^1.1.0", + "is-generator": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/promisify-any/node_modules/bluebird": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", + "integrity": "sha512-UfFSr22dmHPQqPP9XWHRhq+gWnHCYguQGkXQlbyPtW5qTnhFWA8/iXg765tH0cAjy7l/zPJ1aBTO0g5XgA7kvQ==", + "license": "MIT" + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -7275,6 +7611,18 @@ "node": ">=8" } }, + "node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", diff --git a/package.json b/package.json index 62d5046..2ad24b0 100644 --- a/package.json +++ b/package.json @@ -28,15 +28,17 @@ "smithery": "npx @smithery/cli dev src/index.ts", "inspector": "npx @modelcontextprotocol/inspector build/index.js", "lint": "eslint . --ext .ts", - "prettier:check": "prettier --check .", - "prettier:fix": "prettier --write .", + "format": "prettier --write .", "clean": "rm -rf dist", - "prepublishOnly": "pnpm clean && pnpm build", - "pre-commit": "pnpm lint-staged" + "prepublishOnly": "npm run clean && npm run build", + "test": "npm run build && tsx evals/run-evals.ts run --config evals/mcp-eval-basic.config.json && tsx evals/run-evals.ts run --config evals/mcp-eval-minimal.config.json", + "test:basic": "npm run build && tsx evals/run-evals.ts run --config evals/mcp-eval-basic.config.json", + "test:minimal": "npm run build && tsx evals/run-evals.ts run --config evals/mcp-eval-minimal.config.json", + "pre-commit": "lint-staged" }, "lint-staged": { "*.{js,jsx,ts,tsx,json,css,scss,md}": [ - "prettier --write", + "prettier --write .", "eslint --fix" ] }, @@ -47,6 +49,7 @@ "@smithery/cli": "^1.2.15", "commander": "^14.0.0", "dotenv": "^16.4.6", + "mcpvals": "^0.0.3", "playwright-core": "^1.53.2", "zod": "^3.25.67" }, @@ -62,6 +65,5 @@ "tsx": "^4.20.3", "typescript": "^5.6.2", "typescript-eslint": "^8.35.0" - }, - "packageManager": "pnpm@10.12.4+sha512.5ea8b0deed94ed68691c9bad4c955492705c5eeb8a87ef86bc62c74a26b037b08ff9570f108b2e4dbd1dd1a9186fea925e527f141c648e85af45631074680184" + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eb4fc67..557c39c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,6 +25,9 @@ importers: dotenv: specifier: ^16.4.6 version: 16.6.1 + mcpvals: + specifier: ^0.0.3 + version: 0.0.3(react@19.1.0) playwright-core: specifier: ^1.53.2 version: 1.54.1 @@ -1086,6 +1089,19 @@ packages: integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==, } + basic-auth@0.0.1: + resolution: + { + integrity: sha512-sCz6E05DMvrA9dUBGJFfnQ3qs+/lQkVr7qjOT5XMMNfpTzWbpkElpzXfnbNlBjPnDQyz0uBFJ4nELJRIdcKoNQ==, + } + + basic-auth@2.0.1: + resolution: + { + integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==, + } + engines: { node: ">= 0.8" } + bignumber.js@9.3.1: resolution: { @@ -1098,6 +1114,18 @@ packages: integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==, } + bluebird@2.11.0: + resolution: + { + integrity: sha512-UfFSr22dmHPQqPP9XWHRhq+gWnHCYguQGkXQlbyPtW5qTnhFWA8/iXg765tH0cAjy7l/zPJ1aBTO0g5XgA7kvQ==, + } + + bluebird@3.7.2: + resolution: + { + integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==, + } + body-parser@2.2.0: resolution: { @@ -1240,6 +1268,20 @@ packages: } engines: { node: ">=0.8" } + co-bluebird@1.1.0: + resolution: + { + integrity: sha512-JuoemMXxQjYAxbfRrNpOsLyiwDiY8mXvGqJyYLM7jMySDJtnMklW3V2o8uyubpc1eN2YoRsAdfZ1lfKCd3lsrA==, + } + engines: { node: ">=0.12.0" } + + co-use@1.1.0: + resolution: + { + integrity: sha512-1lVRtdywv41zQO/xvI2wU8w6oFcUYT6T84YKSxN25KN4N4Kld3scLovt8FjDmD63Cm7HtyRWHjezt+IanXmkyA==, + } + engines: { node: ">=0.12.0" } + color-convert@2.0.1: resolution: { @@ -1266,6 +1308,13 @@ packages: } engines: { node: ">= 0.8" } + commander@11.1.0: + resolution: + { + integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==, + } + engines: { node: ">=16" } + commander@14.0.0: resolution: { @@ -1693,6 +1742,13 @@ packages: } engines: { node: ">=18.0.0" } + execa@8.0.1: + resolution: + { + integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==, + } + engines: { node: ">=16.17" } + express-rate-limit@7.5.1: resolution: { @@ -1943,6 +1999,13 @@ packages: } engines: { node: ">= 0.4" } + get-stream@8.0.1: + resolution: + { + integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==, + } + engines: { node: ">=16" } + get-symbol-description@1.1.0: resolution: { @@ -2100,6 +2163,13 @@ packages: } engines: { node: ">= 14" } + human-signals@5.0.0: + resolution: + { + integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==, + } + engines: { node: ">=16.17.0" } + humanize-ms@1.2.1: resolution: { @@ -2310,6 +2380,12 @@ packages: } engines: { node: ">= 0.4" } + is-generator@1.0.3: + resolution: + { + integrity: sha512-G56jBpbJeg7ds83HW1LuShNs8J73Fv3CPz/bmROHOHlnKkN8sWb9ujiagjmxxMUywftgq48HlBZELKKqFLk0oA==, + } + is-glob@4.0.3: resolution: { @@ -2393,6 +2469,13 @@ packages: } engines: { node: ">=8" } + is-stream@3.0.0: + resolution: + { + integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==, + } + engines: { node: ^12.20.0 || ^14.13.1 || >=16.0.0 } + is-string@1.1.1: resolution: { @@ -2600,6 +2683,12 @@ packages: integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==, } + lodash@4.17.19: + resolution: + { + integrity: sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==, + } + lodash@4.17.21: resolution: { @@ -2641,6 +2730,20 @@ packages: } engines: { node: ">= 0.4" } + mcpvals@0.0.3: + resolution: + { + integrity: sha512-aMOsp8K6eJNId796Y1K+S2nNHiW8wuGtMuTEB1aCpMAvT57XpK2cMu/SCoKp3vqMZMPFOWX1TS2AhSdhYaOwkw==, + } + hasBin: true + + media-typer@0.3.0: + resolution: + { + integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==, + } + engines: { node: ">= 0.6" } + media-typer@1.1.0: resolution: { @@ -2655,6 +2758,12 @@ packages: } engines: { node: ">=18" } + merge-stream@2.0.0: + resolution: + { + integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==, + } + merge2@1.4.1: resolution: { @@ -2704,6 +2813,13 @@ packages: } engines: { node: ">=6" } + mimic-fn@4.0.0: + resolution: + { + integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==, + } + engines: { node: ">=12" } + mimic-function@5.0.1: resolution: { @@ -2796,6 +2912,27 @@ packages: } hasBin: true + node-oauth2-server@2.4.0: + resolution: + { + integrity: sha512-k3NUmzjEIPyKBuY1OYtHqJ2L6siIlN+oERGe1MVeUGxzeOxEq/2z5K03/P8lfW4ys0Iivbn1KlGJgBeXNZ6Z5w==, + } + engines: { node: ">=0.8" } + + npm-run-path@5.3.0: + resolution: + { + integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==, + } + engines: { node: ^12.20.0 || ^14.13.1 || >=16.0.0 } + + oauth2-server@3.1.1: + resolution: + { + integrity: sha512-4dv+fE9hrK+xTaCygOLh/kQeFzbFr7UqSyHvBDbrQq8Hg52sAkV2vTsyH3Z42hoeaKpbhM7udhL8Y4GYbl6TGQ==, + } + engines: { node: ">=4.0" } + object-assign@4.1.1: resolution: { @@ -2890,6 +3027,13 @@ packages: } engines: { node: ">=6" } + onetime@6.0.0: + resolution: + { + integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==, + } + engines: { node: ">=12" } + onetime@7.0.0: resolution: { @@ -3002,6 +3146,13 @@ packages: } engines: { node: ">=8" } + path-key@4.0.0: + resolution: + { + integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==, + } + engines: { node: ">=12" } + path-parse@1.0.7: resolution: { @@ -3113,6 +3264,13 @@ packages: integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==, } + promisify-any@2.0.1: + resolution: + { + integrity: sha512-pVaGouFbTVxqpVJ+T5A15olNJDASAZHYq5cXz6mWdr6/X34mVWiG9MSdzHTcVBCv4aqBP7wGspi7BUSRbEmhsw==, + } + engines: { node: ">=0.10.0" } + prop-types@15.8.1: resolution: { @@ -3308,6 +3466,12 @@ packages: } engines: { node: ">=0.4" } + safe-buffer@5.1.2: + resolution: + { + integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==, + } + safe-buffer@5.2.1: resolution: { @@ -3507,6 +3671,13 @@ packages: } engines: { node: ">= 10.x" } + statuses@1.5.0: + resolution: + { + integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==, + } + engines: { node: ">= 0.6" } + statuses@2.0.1: resolution: { @@ -3610,6 +3781,13 @@ packages: } engines: { node: ">=12" } + strip-final-newline@3.0.0: + resolution: + { + integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==, + } + engines: { node: ">=12" } + strip-json-comments@3.1.1: resolution: { @@ -3742,6 +3920,13 @@ packages: } engines: { node: ">=10" } + type-is@1.6.18: + resolution: + { + integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==, + } + engines: { node: ">= 0.6" } + type-is@2.0.1: resolution: { @@ -4001,7 +4186,6 @@ snapshots: "@ai-sdk/provider": 1.1.3 "@ai-sdk/provider-utils": 2.2.8(zod@3.25.76) zod: 3.25.76 - optional: true "@ai-sdk/azure@1.3.24(zod@3.25.76)": dependencies: @@ -4060,7 +4244,6 @@ snapshots: "@ai-sdk/provider": 1.1.3 "@ai-sdk/provider-utils": 2.2.8(zod@3.25.76) zod: 3.25.76 - optional: true "@ai-sdk/perplexity@1.1.9(zod@3.25.76)": dependencies: @@ -4710,6 +4893,12 @@ snapshots: base64-js@1.5.1: {} + basic-auth@0.0.1: {} + + basic-auth@2.0.1: + dependencies: + safe-buffer: 5.1.2 + bignumber.js@9.3.1: {} bl@4.1.0: @@ -4718,6 +4907,10 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + bluebird@2.11.0: {} + + bluebird@3.7.2: {} + body-parser@2.2.0: dependencies: bytes: 3.1.2 @@ -4806,6 +4999,13 @@ snapshots: clone@1.0.4: {} + co-bluebird@1.1.0: + dependencies: + bluebird: 2.11.0 + co-use: 1.1.0 + + co-use@1.1.0: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -4818,6 +5018,8 @@ snapshots: dependencies: delayed-stream: 1.0.0 + commander@11.1.0: {} + commander@14.0.0: {} concat-map@0.0.1: {} @@ -5170,6 +5372,18 @@ snapshots: dependencies: eventsource-parser: 3.0.3 + execa@8.0.1: + dependencies: + cross-spawn: 7.0.6 + get-stream: 8.0.1 + human-signals: 5.0.0 + is-stream: 3.0.0 + merge-stream: 2.0.0 + npm-run-path: 5.3.0 + onetime: 6.0.0 + signal-exit: 4.1.0 + strip-final-newline: 3.0.0 + express-rate-limit@7.5.1(express@5.1.0): dependencies: express: 5.1.0 @@ -5362,6 +5576,8 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + get-stream@8.0.1: {} + get-symbol-description@1.1.0: dependencies: call-bound: 1.0.4 @@ -5463,6 +5679,8 @@ snapshots: transitivePeerDependencies: - supports-color + human-signals@5.0.0: {} + humanize-ms@1.2.1: dependencies: ms: 2.1.3 @@ -5595,6 +5813,8 @@ snapshots: has-tostringtag: 1.0.2 safe-regex-test: 1.1.0 + is-generator@1.0.3: {} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -5631,6 +5851,8 @@ snapshots: is-stream@2.0.1: {} + is-stream@3.0.0: {} + is-string@1.1.1: dependencies: call-bound: 1.0.4 @@ -5761,6 +5983,8 @@ snapshots: lodash.merge@4.6.2: {} + lodash@4.17.19: {} + lodash@4.17.21: {} log-symbols@4.1.0: @@ -5787,10 +6011,30 @@ snapshots: math-intrinsics@1.1.0: {} + mcpvals@0.0.3(react@19.1.0): + dependencies: + "@ai-sdk/anthropic": 1.2.12(zod@3.25.76) + "@ai-sdk/openai": 1.3.23(zod@3.25.76) + "@modelcontextprotocol/sdk": 1.15.1 + ai: 4.3.19(react@19.1.0)(zod@3.25.76) + chalk: 5.4.1 + commander: 11.1.0 + execa: 8.0.1 + node-oauth2-server: 2.4.0 + oauth2-server: 3.1.1 + zod: 3.25.76 + transitivePeerDependencies: + - react + - supports-color + + media-typer@0.3.0: {} + media-typer@1.1.0: {} merge-descriptors@2.0.0: {} + merge-stream@2.0.0: {} + merge2@1.4.1: {} micromatch@4.0.8: @@ -5812,6 +6056,8 @@ snapshots: mimic-fn@2.1.0: {} + mimic-fn@4.0.0: {} + mimic-function@5.0.1: {} minimatch@3.1.2: @@ -5845,6 +6091,23 @@ snapshots: node-gyp-build@4.8.4: optional: true + node-oauth2-server@2.4.0: + dependencies: + basic-auth: 0.0.1 + + npm-run-path@5.3.0: + dependencies: + path-key: 4.0.0 + + oauth2-server@3.1.1: + dependencies: + basic-auth: 2.0.1 + bluebird: 3.7.2 + lodash: 4.17.19 + promisify-any: 2.0.1 + statuses: 1.5.0 + type-is: 1.6.18 + object-assign@4.1.1: {} object-inspect@1.13.4: {} @@ -5906,6 +6169,10 @@ snapshots: dependencies: mimic-fn: 2.1.0 + onetime@6.0.0: + dependencies: + mimic-fn: 4.0.0 + onetime@7.0.0: dependencies: mimic-function: 5.0.1 @@ -5989,6 +6256,8 @@ snapshots: path-key@3.1.1: {} + path-key@4.0.0: {} + path-parse@1.0.7: {} path-to-regexp@8.2.0: {} @@ -6053,6 +6322,12 @@ snapshots: process-warning@5.0.0: {} + promisify-any@2.0.1: + dependencies: + bluebird: 2.11.0 + co-bluebird: 1.1.0 + is-generator: 1.0.3 + prop-types@15.8.1: dependencies: loose-envify: 1.4.0 @@ -6182,6 +6457,8 @@ snapshots: has-symbols: 1.1.0 isarray: 2.0.5 + safe-buffer@5.1.2: {} + safe-buffer@5.2.1: {} safe-push-apply@1.0.0: @@ -6321,6 +6598,8 @@ snapshots: split2@4.2.0: {} + statuses@1.5.0: {} + statuses@2.0.1: {} statuses@2.0.2: {} @@ -6402,6 +6681,8 @@ snapshots: dependencies: ansi-regex: 6.1.0 + strip-final-newline@3.0.0: {} + strip-json-comments@3.1.1: {} supports-color@7.2.0: @@ -6465,6 +6746,11 @@ snapshots: type-fest@0.21.3: {} + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + type-is@2.0.1: dependencies: content-type: 1.0.5 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml deleted file mode 100644 index ccbac80..0000000 --- a/pnpm-workspace.yaml +++ /dev/null @@ -1,2 +0,0 @@ -packages: - - "." diff --git a/src/tools/multiSession.ts b/src/tools/multiSession.ts index 0a71860..1c270a1 100644 --- a/src/tools/multiSession.ts +++ b/src/tools/multiSession.ts @@ -227,7 +227,7 @@ export const closeSessionTool = defineTool({ content: [ { type: "text", - text: `Closed session ${sessionId}`, + text: `closed session ${sessionId}`, }, ], }), diff --git a/src/tools/session.ts b/src/tools/session.ts index c944d70..af62e15 100644 --- a/src/tools/session.ts +++ b/src/tools/session.ts @@ -99,7 +99,7 @@ async function handleCreateSession( content: [ { type: "text", - text: `Browserbase Live Session View URL: https://www.browserbase.com/sessions/${session.sessionId}\nBrowserbase Live Debugger URL: ${debugUrl}`, + text: `Browserbase Live Session View URL: https://www.browserbase.com/sessions/${session.sessionId}\nBrowserbase Live Debugger URL: ${debugUrl} \n session created`, }, ], }; @@ -206,7 +206,7 @@ async function handleCloseSession(context: Context): Promise { } if (stagehandClosedSuccessfully) { - let successMessage = `Browserbase session (${previousSessionId || "default"}) closed successfully via Stagehand. Context reset to default.`; + let successMessage = `Browserbase session (${previousSessionId || "default"}) closed successfully via Stagehand. Context reset to default. session closed`; if (browserbaseSessionId && previousSessionId !== defaultSessionId) { successMessage += ` View replay at https://www.browserbase.com/sessions/${browserbaseSessionId}`; }