From a8e423b4808a1c69d3ca6ce222d88ce28233ab60 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 11 May 2026 07:16:31 +0000 Subject: [PATCH 1/4] test(agent): add failing tests for Anthropic CUA triple_click handling Verifies that the AnthropicCUAClient correctly converts triple_click actions (with both coordinate-array and x/y formats) into tripleClick actions consumed by v3CuaAgentHandler. --- .../unit/anthropic-cua-triple-click.test.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 packages/core/tests/unit/anthropic-cua-triple-click.test.ts diff --git a/packages/core/tests/unit/anthropic-cua-triple-click.test.ts b/packages/core/tests/unit/anthropic-cua-triple-click.test.ts new file mode 100644 index 000000000..727fe5173 --- /dev/null +++ b/packages/core/tests/unit/anthropic-cua-triple-click.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { AnthropicCUAClient } from "../../lib/v3/agent/AnthropicCUAClient.js"; +import Anthropic from "@anthropic-ai/sdk"; + +vi.mock("@anthropic-ai/sdk", () => { + const mockCreate = vi.fn(); + + return { + default: class MockAnthropic { + beta = { + messages: { + create: mockCreate, + }, + }; + }, + }; +}); + +describe("AnthropicCUAClient triple_click handling", () => { + let mockCreate: ReturnType; + let client: AnthropicCUAClient; + let executedActions: Array>; + + beforeEach(() => { + vi.clearAllMocks(); + const anthropic = new Anthropic({ apiKey: "test" }); + mockCreate = anthropic.beta.messages.create as ReturnType; + + client = new AnthropicCUAClient("anthropic", "claude-sonnet-4-5-20250929", undefined, { + apiKey: "test-key", + }); + client.setViewport(1280, 720); + client.setScreenshotProvider(async () => "fake-base64-screenshot"); + + executedActions = []; + client.setActionHandler(async (action) => { + executedActions.push({ ...action }); + }); + }); + + it("should convert triple_click with coordinate array to tripleClick action", async () => { + mockCreate.mockResolvedValue({ + id: "test-id", + content: [ + { + type: "tool_use", + id: "tool-1", + name: "computer", + input: { + action: "triple_click", + coordinate: [640, 360], + }, + }, + ], + usage: { input_tokens: 10, output_tokens: 20 }, + }); + + const logger = vi.fn(); + await client.executeStep( + [{ role: "user", content: "triple click the paragraph" }], + logger, + ); + + expect(executedActions).toHaveLength(1); + expect(executedActions[0].type).toBe("tripleClick"); + expect(executedActions[0].x).toBe(640); + expect(executedActions[0].y).toBe(360); + }); + + it("should convert triple_click with x/y fields to tripleClick action", async () => { + mockCreate.mockResolvedValue({ + id: "test-id", + content: [ + { + type: "tool_use", + id: "tool-2", + name: "computer", + input: { + action: "triple_click", + x: 100, + y: 200, + }, + }, + ], + usage: { input_tokens: 10, output_tokens: 20 }, + }); + + const logger = vi.fn(); + await client.executeStep( + [{ role: "user", content: "triple click the line" }], + logger, + ); + + expect(executedActions).toHaveLength(1); + expect(executedActions[0].type).toBe("tripleClick"); + expect(executedActions[0].x).toBe(100); + expect(executedActions[0].y).toBe(200); + }); +}); From fd49cb6d266fb5939382d3cec989988a24d46b36 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 11 May 2026 07:16:38 +0000 Subject: [PATCH 2/4] fix(agent): handle triple_click action from Anthropic CUA AnthropicCUAClient.convertToolUseToAction() had no explicit handler for the triple_click action that Claude can emit via the computer tool. The action fell through to the generic handler which: 1. Kept the snake_case type "triple_click" instead of normalizing to "tripleClick" expected by v3CuaAgentHandler.executeAction(). 2. Did not extract coordinates from Anthropic's coordinate array format. This adds: - Explicit triple_click/tripleClick mapping in the Anthropic client, following the same pattern as the existing double_click handler. - A "triple_click" case alias in v3CuaAgentHandler.executeAction(), matching how "double_click" and "doubleClick" are both handled. --- packages/core/lib/v3/agent/AnthropicCUAClient.ts | 11 +++++++++++ packages/core/lib/v3/handlers/v3CuaAgentHandler.ts | 1 + 2 files changed, 12 insertions(+) diff --git a/packages/core/lib/v3/agent/AnthropicCUAClient.ts b/packages/core/lib/v3/agent/AnthropicCUAClient.ts index 4bbe2d47b..752d208e2 100644 --- a/packages/core/lib/v3/agent/AnthropicCUAClient.ts +++ b/packages/core/lib/v3/agent/AnthropicCUAClient.ts @@ -901,6 +901,17 @@ export class AnthropicCUAClient extends AgentClient { (input.coordinate ? (input.coordinate as number[])[1] : 0), ...input, }; + } else if (action === "triple_click" || action === "tripleClick") { + return { + type: "tripleClick", + x: + (input.x as number) || + (input.coordinate ? (input.coordinate as number[])[0] : 0), + y: + (input.y as number) || + (input.coordinate ? (input.coordinate as number[])[1] : 0), + ...input, + }; } else if (action === "scroll") { // Convert Anthropic's coordinate, scroll_amount and scroll_direction into scroll_x and scroll_y const x = diff --git a/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts b/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts index 6cefa4b4d..af3a3dad8 100644 --- a/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts +++ b/packages/core/lib/v3/handlers/v3CuaAgentHandler.ts @@ -324,6 +324,7 @@ export class V3CuaAgentHandler { } return { success: true }; } + case "triple_click": case "tripleClick": { const { x, y } = action; if (recording) { From bee0daa6b45153be1c1d67fdc7e60332edf57387 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 11 May 2026 07:25:37 +0000 Subject: [PATCH 3/4] chore: add changeset for Anthropic CUA triple_click fix --- .changeset/fix-anthropic-cua-triple-click.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-anthropic-cua-triple-click.md diff --git a/.changeset/fix-anthropic-cua-triple-click.md b/.changeset/fix-anthropic-cua-triple-click.md new file mode 100644 index 000000000..c6b998516 --- /dev/null +++ b/.changeset/fix-anthropic-cua-triple-click.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": patch +--- + +Fix Anthropic CUA `triple_click` action mapping so triple-click actions are correctly normalized to `tripleClick` and coordinates are extracted from both `coordinate` array and direct `x`/`y` formats. From fa430f4b40e684cc1a89e52d4639adc869cf17f2 Mon Sep 17 00:00:00 2001 From: Chromie Date: Mon, 11 May 2026 00:37:27 -0700 Subject: [PATCH 4/4] Apply suggestion from @chromiebot --- .changeset/fix-anthropic-cua-triple-click.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/fix-anthropic-cua-triple-click.md b/.changeset/fix-anthropic-cua-triple-click.md index c6b998516..6f2d57fe6 100644 --- a/.changeset/fix-anthropic-cua-triple-click.md +++ b/.changeset/fix-anthropic-cua-triple-click.md @@ -2,4 +2,4 @@ "@browserbasehq/stagehand": patch --- -Fix Anthropic CUA `triple_click` action mapping so triple-click actions are correctly normalized to `tripleClick` and coordinates are extracted from both `coordinate` array and direct `x`/`y` formats. +Fix Anthropic CUA `triple_click` action mapping.