feat: add claude-3-7-sonnet-20250219 (#369)

* Add 2 new model options for config.ai.model for `anthropic`
  * `claude-3-7-sonnet-20250219 `
  * `claude-3-7-sonnet-latest `
* Add 2 new computer tools: `computer_20250124`, `bash_20250124`
* Add support for new actions: `triple_click`, `hold_key`,
`left_mouse_down`, `left_mouse_up`, `wait`, `scroll`

Author: rmarescu

packages/shortest/package.json

@@ -58,9 +58,9 @@
     "node": ">=18"
   },
   "peerDependencies": {
-    "@ai-sdk/anthropic": "^1.1.9",
-    "@ai-sdk/provider": "^1.0.8",
-    "ai": "^4.1.45",
+    "@ai-sdk/anthropic": "^1.1.15",
+    "@ai-sdk/provider": "^1.0.10",
+    "ai": "^4.1.53",
     "@babel/parser": "^7.26.9",
     "@babel/traverse": "^7.26.9",
     "@babel/types": "^7.26.9",

packages/shortest/src/ai/tools/anthropic/bash_20250124.ts

@@ -0,0 +1,18 @@
+import { anthropic } from "@ai-sdk/anthropic";
+import { BashTool } from "@/browser/core/bash-tool";
+
+/**
+ * @see https://sdk.vercel.ai/providers/ai-sdk-providers/anthropic#bash-tool
+ */
+export const createAnthropicBash20250124 = () =>
+  anthropic.tools.bash_20250124({
+    execute: async ({ command }) => await new BashTool().execute(command),
+    experimental_toToolResultContent(result) {
+      return [
+        {
+          type: "text",
+          text: result,
+        },
+      ];
+    },
+  });

packages/shortest/src/ai/tools/anthropic/computer_20241022.ts

@@ -1,6 +1,7 @@
 import { anthropic } from "@ai-sdk/anthropic";
 import { Tool } from "ai";
 import { BrowserTool } from "@/browser/core/browser-tool";
+import { getLogger } from "@/log";
 import { InternalActionEnum } from "@/types/browser";
 
 /**
@@ -17,6 +18,11 @@ export const createAnthropicComputer20241022 = (
       const { action, ...restOfInput } = input;
       const internalAction = actionMap[action];
       if (!internalAction) {
+        const log = getLogger();
+        log.error(`Computer action not supported`, {
+          tool: "anthropic.computer_20241022",
+          action,
+        });
         return { output: `Action '${action}' not supported` };
       }
       return browserTool.execute({ action: internalAction, ...restOfInput });

packages/shortest/src/ai/tools/anthropic/computer_20250124.ts

@@ -0,0 +1,55 @@
+import { anthropic } from "@ai-sdk/anthropic";
+import { Tool } from "ai";
+import { BrowserTool } from "@/browser/core/browser-tool";
+import { getLogger } from "@/log";
+import { InternalActionEnum } from "@/types/browser";
+
+/**
+ * @see https://sdk.vercel.ai/providers/ai-sdk-providers/anthropic#computer-tool
+ */
+export const createAnthropicComputer20250124 = (
+  browserTool: BrowserTool,
+): Tool =>
+  anthropic.tools.computer_20250124({
+    displayWidthPx: 1920,
+    displayHeightPx: 1080,
+    displayNumber: 0,
+    execute: async (input) => {
+      const { action, ...restOfInput } = input;
+      const internalAction = actionMap[action];
+      if (!internalAction) {
+        const log = getLogger();
+        log.error(`Computer action not supported`, {
+          tool: "anthropic.computer_20250124",
+          action,
+        });
+        return { output: `Action '${action}' not supported` };
+      }
+      return browserTool.execute({ action: internalAction, ...restOfInput });
+    },
+    experimental_toToolResultContent: browserTool.resultToToolResultContent,
+  });
+
+/**
+ * Map of Anthropic computer_20250124 actions to internal actions
+ *
+ * @see https://docs.anthropic.com/en/docs/agents-and-tools/computer-use#computer-tool
+ */
+const actionMap: Record<string, InternalActionEnum> = {
+  key: InternalActionEnum.KEY,
+  hold_key: InternalActionEnum.HOLD_KEY,
+  type: InternalActionEnum.TYPE,
+  cursor_position: InternalActionEnum.CURSOR_POSITION,
+  mouse_move: InternalActionEnum.MOUSE_MOVE,
+  left_mouse_down: InternalActionEnum.LEFT_MOUSE_DOWN,
+  left_mouse_up: InternalActionEnum.LEFT_MOUSE_UP,
+  left_click: InternalActionEnum.LEFT_CLICK,
+  left_click_drag: InternalActionEnum.LEFT_CLICK_DRAG,
+  right_click: InternalActionEnum.RIGHT_CLICK,
+  middle_click: InternalActionEnum.MIDDLE_CLICK,
+  double_click: InternalActionEnum.DOUBLE_CLICK,
+  triple_click: InternalActionEnum.TRIPLE_CLICK,
+  scroll: InternalActionEnum.SCROLL,
+  wait: InternalActionEnum.WAIT,
+  screenshot: InternalActionEnum.SCREENSHOT,
+};

packages/shortest/src/browser/actions/index.ts

@@ -67,6 +67,10 @@ export const click = async (
   page: Page,
   x: number,
   y: number,
+  options: { button?: "left" | "right" | "middle"; clickCount?: number } = {
+    button: "left",
+    clickCount: 1,
+  },
 ): Promise<void> => {
   const scaledX = Math.round(x * scaleRatio.x);
   const scaledY = Math.round(y * scaleRatio.y);
@@ -75,7 +79,7 @@ export const click = async (
   const animationPromise = showClickAnimation(page, "left");
 
   await Promise.all([
-    page.mouse.click(scaledX, scaledY, { delay: 200 }), // delay to match animation duration
+    page.mouse.click(scaledX, scaledY, options),
     animationPromise,
   ]);
 };

packages/shortest/src/browser/core/browser-tool.ts

@@ -26,7 +26,12 @@ import { TestCase } from "@/core/runner/test-case";
 import { getConfig, initializeConfig } from "@/index";
 import { getLogger, Log } from "@/log/index";
 import { TestContext, BrowserToolConfig, ShortestConfig } from "@/types";
-import { ActionInput, ToolResult, BetaToolType } from "@/types/browser";
+import {
+  ActionInput,
+  ToolResult,
+  BetaToolType,
+  InternalActionEnum,
+} from "@/types/browser";
 import { getErrorDetails, ToolError, TestError } from "@/utils/errors";
 
 export class BrowserTool extends BaseBrowserTool {
@@ -201,24 +206,58 @@ export class BrowserTool extends BaseBrowserTool {
     await this.page.click(selector);
   }
 
-  public async clickAtCoordinates(x: number, y: number): Promise<void> {
-    this.log.debug("Clicking at coordinates", { x, y });
-    await actions.click(this.page, x, y);
-  }
-
   async execute(input: ActionInput): Promise<ToolResult> {
     try {
       this.log.setGroup(`🛠️ ${input.action}`);
       let output = "";
       let metadata = {};
 
       switch (input.action) {
-        case "left_click":
-        case "right_click":
-        case "middle_click":
-        case "double_click": {
-          const clickCoords = input.coordinates || this.lastMousePosition;
-          await this.clickAtCoordinates(clickCoords[0], clickCoords[1]);
+        case InternalActionEnum.LEFT_CLICK:
+        case InternalActionEnum.RIGHT_CLICK:
+        case InternalActionEnum.MIDDLE_CLICK:
+        case InternalActionEnum.DOUBLE_CLICK:
+        case InternalActionEnum.TRIPLE_CLICK: {
+          const clickCoords =
+            input.coordinate || input.coordinates || this.lastMousePosition;
+          const x = clickCoords[0];
+          const y = clickCoords[1];
+          const button = () => {
+            switch (input.action) {
+              case InternalActionEnum.LEFT_CLICK:
+              case InternalActionEnum.DOUBLE_CLICK:
+              case InternalActionEnum.TRIPLE_CLICK:
+                return "left";
+              case InternalActionEnum.RIGHT_CLICK:
+                return "right";
+              case InternalActionEnum.MIDDLE_CLICK:
+                return "middle";
+              default:
+                throw new ToolError(
+                  `Unsupported click action: ${input.action}`,
+                );
+            }
+          };
+          const clickCount = () => {
+            switch (input.action) {
+              case InternalActionEnum.DOUBLE_CLICK:
+                return 2;
+              case InternalActionEnum.TRIPLE_CLICK:
+                return 3;
+              default:
+                return 1;
+            }
+          };
+          this.log.debug("Clicking at coordinates", {
+            x,
+            y,
+            button: button(),
+            clickCount: clickCount(),
+          });
+          await actions.click(this.page, x, y, {
+            button: button(),
+            clickCount: clickCount(),
+          });
           output = `${input.action} at (${clickCoords[0]}, ${clickCoords[1]})`;
 
           // Get initial metadata before potential navigation
@@ -245,7 +284,7 @@ export class BrowserTool extends BaseBrowserTool {
           break;
         }
 
-        case "mouse_move":
+        case InternalActionEnum.MOUSE_MOVE:
           const coords = input.coordinates || (input as any).coordinate;
           if (!coords) {
             throw new ToolError("Coordinates required for mouse_move");
@@ -255,7 +294,7 @@ export class BrowserTool extends BaseBrowserTool {
           output = `Mouse moved to (${coords[0]}, ${coords[1]})`;
           break;
 
-        case "left_click_drag":
+        case InternalActionEnum.LEFT_CLICK_DRAG:
           if (!input.coordinates) {
             throw new ToolError("Coordinates required for left_click_drag");
           }
@@ -267,15 +306,25 @@ export class BrowserTool extends BaseBrowserTool {
           output = `Dragged mouse to (${input.coordinates[0]}, ${input.coordinates[1]})`;
           break;
 
-        case "cursor_position":
+        case InternalActionEnum.LEFT_MOUSE_DOWN:
+          await this.page.mouse.down();
+          output = "Pressed left mouse button";
+          break;
+
+        case InternalActionEnum.LEFT_MOUSE_UP:
+          await this.page.mouse.up();
+          output = "Released left mouse button";
+          break;
+
+        case InternalActionEnum.CURSOR_POSITION:
           const position = await actions.getCursorPosition(this.page);
           output = `Cursor position: (${position[0]}, ${position[1]})`;
           break;
 
-        case "screenshot":
+        case InternalActionEnum.SCREENSHOT:
           return await this.takeScreenshotWithMetadata();
 
-        case "type":
+        case InternalActionEnum.TYPE:
           if (!input.text) {
             throw new ToolError("Text required for type action");
           }
@@ -285,7 +334,7 @@ export class BrowserTool extends BaseBrowserTool {
           output = `Typed: ${input.text}`;
           break;
 
-        case "key": {
+        case InternalActionEnum.KEY: {
           if (!input.text) {
             throw new ToolError("Key required for key action");
           }
@@ -313,7 +362,31 @@ export class BrowserTool extends BaseBrowserTool {
           break;
         }
 
-        case "github_login": {
+        case InternalActionEnum.HOLD_KEY: {
+          if (!input.text) {
+            throw new ToolError("Key required for hold_key action");
+          }
+
+          if (!input.duration) {
+            throw new ToolError("Duration required for hold_key action");
+          }
+
+          const seconds = input.duration;
+          const delay = seconds / 1000;
+
+          const keyText = input.text.toLowerCase();
+          const keys = Array.isArray(actions.keyboardShortcuts[keyText])
+            ? actions.keyboardShortcuts[keyText]
+            : [actions.keyboardShortcuts[keyText] || input.text];
+
+          const parsedKeys = keys.join("+");
+          await this.page.keyboard.press(parsedKeys, { delay });
+
+          output = `Held key: ${parsedKeys} for ${seconds} second${seconds !== 1 ? "s" : ""}`;
+          break;
+        }
+
+        case InternalActionEnum.GITHUB_LOGIN: {
           if (!this.githubTool) {
             this.githubTool = new GitHubTool();
           }
@@ -328,7 +401,7 @@ export class BrowserTool extends BaseBrowserTool {
           break;
         }
 
-        case "clear_session":
+        case InternalActionEnum.CLEAR_SESSION:
           const newContext = await this.browserManager.recreateContext();
           this.page = newContext.pages()[0] || (await newContext.newPage());
           await this.page.evaluate(() => {
@@ -341,7 +414,7 @@ export class BrowserTool extends BaseBrowserTool {
             metadata: {},
           };
 
-        case "run_callback": {
+        case InternalActionEnum.RUN_CALLBACK: {
           if (!this.testContext?.currentTest) {
             throw new ToolError(
               "No test context available for callback execution",
@@ -389,7 +462,7 @@ export class BrowserTool extends BaseBrowserTool {
           }
         }
 
-        case "navigate": {
+        case InternalActionEnum.NAVIGATE: {
           if (!input.url) {
             throw new ToolError("URL required for navigation");
           }
@@ -440,7 +513,37 @@ export class BrowserTool extends BaseBrowserTool {
           }
         }
 
-        case "sleep": {
+        case InternalActionEnum.WAIT:
+          if (!input.duration) {
+            throw new ToolError("Duration required for wait action");
+          }
+          const seconds = input.duration;
+          await this.page.waitForTimeout(seconds * 1000);
+          output = `Waited for ${seconds} second${seconds !== 1 ? "s" : ""}`;
+          break;
+
+        case InternalActionEnum.SCROLL:
+          if (
+            !input.coordinate ||
+            !input.scroll_amount ||
+            !input.scroll_direction
+          ) {
+            throw new ToolError("Missing args for scroll action");
+          }
+          await this.page.mouse.move(input.coordinate[0], input.coordinate[1]);
+          const deltaX =
+            (input.scroll_direction === "up"
+              ? -input.scroll_amount
+              : input.scroll_amount) || 0;
+          const deltaY =
+            (input.scroll_direction === "left"
+              ? -input.scroll_amount
+              : input.scroll_amount) || 0;
+          await this.page.mouse.wheel(deltaX, deltaY);
+          output = `Scrolled ${input.scroll_amount} clicks ${input.scroll_direction}`;
+          break;
+
+        case InternalActionEnum.SLEEP: {
           const defaultDuration = 1000;
           const maxDuration = 60000;
           let duration = input.duration ?? defaultDuration;
@@ -461,7 +564,7 @@ export class BrowserTool extends BaseBrowserTool {
           break;
         }
 
-        case "check_email": {
+        case InternalActionEnum.CHECK_EMAIL: {
           if (!this.mailosaurTool) {
             const mailosaurAPIKey =
               this.config.mailosaur?.apiKey || process.env.MAILOSAUR_API_KEY;
@@ -714,15 +817,17 @@ export class BrowserTool extends BaseBrowserTool {
 
     writeFileSync(filePath, buffer);
     const filePathWithoutCwd = filePath.replace(process.cwd() + "/", "");
-    this.log.debug("📺", "Screenshot saved", { filePath: filePathWithoutCwd });
 
-    const metadata = {
+    const browserMetadata = await this.getMetadata();
+    this.log.trace("Screenshot saved", {
+      filePath: filePathWithoutCwd,
+      ...browserMetadata["window_info"],
+    });
+    return {
       output: "Screenshot taken",
       base64_image: buffer.toString("base64"),
-      metadata: await this.getMetadata(),
+      metadata: browserMetadata,
     };
-    this.log.trace("Screenshot details", metadata);
-    return metadata;
   }
 
   toToolParameters() {