feat: observe-act flow + stagehand v4 upgrade

.changeset/browser-act-replay.md

@@ -0,0 +1,13 @@
+---
+"@upstash/box": minor
+---
+
+feat: replay a resolved browser action with `tab.act(action)` (no LLM, no key)
+
+`observe()` now returns each element's suggested `method` and `arguments`
+alongside `selector`, and `act()` accepts a pre-resolved action
+(`BrowserObserveElement` or `BrowserActAction`) in addition to a natural-language
+string. Passing an action replays it deterministically: no LLM call, no tokens,
+and no model provider key required. Resolve a step once with `observe()`, cache
+the returned action, and replay it across pages or runs. A new `BrowserAction`
+type (`BrowserObserveElement | BrowserActAction`) is exported.

packages/python-sdk/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to `upstash-box` (Python) are documented here.
 
 ## Unreleased
 
+- `tab.act(action)` — replay a pre-resolved action from `observe()`
+  deterministically, with no LLM call, no tokens, and no model provider key
+  required (pass a `BrowserObserveElement` or `BrowserActAction` instead of a
+  string; `model` is ignored in that form). `BrowserObserveElement` now also
+  carries `method` and `arguments`. Mirrors `act(action)` in `@upstash/box`.
 - `browser.recordings.download(recording_id, path=...)` — save a recording's
   video to a local file (streamed to disk, parent directories created as
   needed) and return the path written. Recordings download as MP4; recordings

packages/python-sdk/tests/_async/test_box_browser.py

@@ -261,6 +261,45 @@ async def test_act_executes_one_action():
     await box.aclose()
 
 
+@respx.mock
+async def test_act_replays_pre_resolved_action():
+    from upstash_box import BrowserActAction
+
+    box = await make_async_box(respx.mock)
+    act = respx.post(f"{BASE}/browser/act").mock(
+        return_value=httpx.Response(
+            200,
+            json={
+                "success": True,
+                "message": "done",
+                "action_description": "Sign in",
+                "actions": [],
+                "input_tokens": 0,
+                "output_tokens": 0,
+            },
+        )
+    )
+
+    action = BrowserActAction(
+        selector="xpath=/html/body/button", description="Sign in", method="click", arguments=[]
+    )
+    result = await box.browser.get_tab("tab-2").act(action)
+
+    assert result.success is True
+    assert result.input_tokens == 0
+    # Posts a pre-resolved action, never an instruction.
+    assert last_json_body(act) == {
+        "action": {
+            "selector": "xpath=/html/body/button",
+            "description": "Sign in",
+            "method": "click",
+            "arguments": [],
+        },
+        "tab": "tab-2",
+    }
+    await box.aclose()
+
+
 class Person(BaseModel):
     name: str
     headline: str

packages/python-sdk/upstash_box/_async/client.py

@@ -42,8 +42,10 @@
     BoxData,
     BoxGetOptions,
     BoxRunData,
+    BrowserActAction,
     BrowserActResult,
     BrowserContent,
+    BrowserObserveElement,
     BrowserObserveResult,
     BrowserRecording,
     BrowserRunResult,
@@ -589,11 +591,23 @@ async def observe(
         )
         return BrowserObserveResult.model_validate({"elements": resp.get("elements") or []})
 
-    async def act(self, instruction: str, *, model: Optional[str] = None) -> BrowserActResult:
-        """Resolve and execute one natural-language action on this tab (metered)."""
-        body: Dict[str, Any] = {"instruction": instruction, "tab": self.id}
-        if model:
-            body["model"] = model
+    async def act(
+        self,
+        instruction: Union[str, BrowserObserveElement, BrowserActAction],
+        *,
+        model: Optional[str] = None,
+    ) -> BrowserActResult:
+        """Resolve and execute one action on this tab.
+
+        Pass a string (LLM-resolved, metered) or a pre-resolved ``observe()``
+        action to replay it with no LLM call and no key (``model`` ignored).
+        """
+        if isinstance(instruction, str):
+            body: Dict[str, Any] = {"instruction": instruction, "tab": self.id}
+            if model:
+                body["model"] = model
+        else:
+            body = {"action": instruction.model_dump(exclude_none=True), "tab": self.id}
         resp = await self._box._request(
             "POST",
             f"/v2/box/{self._box.id}/browser/act",

packages/python-sdk/upstash_box/_sync/client.py

@@ -41,8 +41,10 @@
     BoxData,
     BoxGetOptions,
     BoxRunData,
+    BrowserActAction,
     BrowserActResult,
     BrowserContent,
+    BrowserObserveElement,
     BrowserObserveResult,
     BrowserRecording,
     BrowserRunResult,
@@ -582,11 +584,23 @@ def observe(self, instruction: str, *, model: Optional[str] = None) -> BrowserOb
         )
         return BrowserObserveResult.model_validate({"elements": resp.get("elements") or []})
 
-    def act(self, instruction: str, *, model: Optional[str] = None) -> BrowserActResult:
-        """Resolve and execute one natural-language action on this tab (metered)."""
-        body: Dict[str, Any] = {"instruction": instruction, "tab": self.id}
-        if model:
-            body["model"] = model
+    def act(
+        self,
+        instruction: Union[str, BrowserObserveElement, BrowserActAction],
+        *,
+        model: Optional[str] = None,
+    ) -> BrowserActResult:
+        """Resolve and execute one action on this tab.
+
+        Pass a string (LLM-resolved, metered) or a pre-resolved ``observe()``
+        action to replay it with no LLM call and no key (``model`` ignored).
+        """
+        if isinstance(instruction, str):
+            body: Dict[str, Any] = {"instruction": instruction, "tab": self.id}
+            if model:
+                body["model"] = model
+        else:
+            body = {"action": instruction.model_dump(exclude_none=True), "tab": self.id}
         resp = self._box._request(
             "POST",
             f"/v2/box/{self._box.id}/browser/act",

packages/python-sdk/upstash_box/types.py

@@ -783,6 +783,9 @@ class BrowserObserveElement(_Model):
     # A selector for the element (Stagehand-resolved), when available.
     selector: Optional[str] = None
     url: Optional[str] = None
+    # Suggested method and args, for replay via ``tab.act(element)``.
+    method: Optional[str] = None
+    arguments: Optional[List[str]] = None
 
 
 class BrowserObserveResult(_Model):

packages/sdk/src/__tests__/box-browser.test.ts

@@ -202,6 +202,46 @@ describe("Box browser operations", () => {
     });
   });
 
+  it("replays a pre-resolved action deterministically (posts action, not instruction)", async () => {
+    const { box, fetchMock } = await createTestBox();
+    fetchMock
+      .mockResolvedValueOnce(mockResponse({ id: "tab-2", url: "https://example.com/login" }))
+      .mockResolvedValueOnce(
+        mockResponse({
+          success: true,
+          message: "done",
+          action_description: "Sign in",
+          actions: [
+            {
+              selector: "xpath=/html/body/button",
+              description: "Sign in",
+              method: "click",
+              arguments: [],
+            },
+          ],
+          input_tokens: 0,
+          output_tokens: 0,
+        }),
+      );
+
+    const tab = await box.browser.tab.create("https://example.com/login");
+    const action = {
+      selector: "xpath=/html/body/button",
+      description: "Sign in",
+      method: "click",
+      arguments: [],
+    };
+    const result = await tab.act(action);
+
+    expect(result.success).toBe(true);
+    expect(result.inputTokens).toBe(0);
+    expect(fetchMock.mock.calls[2]?.[0]).toContain("browser/act");
+    expect(JSON.parse(fetchMock.mock.calls[2]?.[1]?.body as string)).toEqual({
+      action,
+      tab: "tab-2",
+    });
+  });
+
   it("runs a multi-step task with schema-validated structured output", async () => {
     const { box, fetchMock } = await createTestBox();
     fetchMock
@@ -586,14 +626,27 @@ describe("Box browser operations", () => {
     const { box, fetchMock } = await createTestBox();
     fetchMock.mockResolvedValueOnce(
       mockResponse({
-        elements: [{ description: "Sign in button", selector: "xpath=/html/body/button" }],
+        elements: [
+          {
+            description: "Sign in button",
+            selector: "xpath=/html/body/button",
+            method: "click",
+            arguments: [],
+          },
+        ],
       }),
     );
 
     const result = await box.browser.getTab("tab-1").observe("the sign in button");
 
+    // method/arguments pass through so the element can be replayed via act(action).
     expect(result.elements).toEqual([
-      { description: "Sign in button", selector: "xpath=/html/body/button" },
+      {
+        description: "Sign in button",
+        selector: "xpath=/html/body/button",
+        method: "click",
+        arguments: [],
+      },
     ]);
     expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string)).toEqual({
       instruction: "the sign in button",

packages/sdk/src/client.ts

@@ -55,6 +55,7 @@ import {
   type BrowserTabCreateOptions,
   type BrowserObserveResult,
   type BrowserActResult,
+  type BrowserAction,
   type BrowserRunOptions,
   type BrowserRunResult,
   type BrowserRunStep,
@@ -512,7 +513,21 @@ export class Tab {
   }
 
   /** Resolve and execute one natural-language action on this tab (metered). */
-  async act(instruction: string, options?: BrowserExtractOptions): Promise<BrowserActResult> {
+  async act(instruction: string, options?: BrowserExtractOptions): Promise<BrowserActResult>;
+  /** Replay a pre-resolved `observe()` action with no LLM call and no key (`model` ignored). */
+  async act(action: BrowserAction): Promise<BrowserActResult>;
+  async act(
+    instructionOrAction: string | BrowserAction,
+    options?: BrowserExtractOptions,
+  ): Promise<BrowserActResult> {
+    const body =
+      typeof instructionOrAction === "string"
+        ? {
+            instruction: instructionOrAction,
+            tab: this.id,
+            ...(options?.model ? { model: options.model } : {}),
+          }
+        : { action: instructionOrAction, tab: this.id };
     const resp = await this.box._request<{
       success?: boolean;
       message?: string;
@@ -522,7 +537,7 @@ export class Tab {
       input_tokens?: number;
       output_tokens?: number;
     }>("POST", `/v2/box/${this.box.id}/browser/act`, {
-      body: { instruction, tab: this.id, ...(options?.model ? { model: options.model } : {}) },
+      body,
       timeout: 180000,
     });
     return {

packages/sdk/src/index.ts

@@ -87,6 +87,7 @@ export type {
   BrowserObserveResult,
   BrowserActAction,
   BrowserActResult,
+  BrowserAction,
   BrowserRunOptions,
   BrowserRunResult,
   BrowserRunStep,

packages/sdk/src/types.ts

@@ -1146,6 +1146,9 @@ export interface BrowserObserveElement {
   /** A selector for the element (Stagehand-resolved), when available. */
   selector?: string;
   url?: string;
+  /** Suggested method and args, for replay via `act(action)`. */
+  method?: string;
+  arguments?: string[];
 }
 
 /** Result of `box.browser.observe()`. */
@@ -1161,6 +1164,9 @@ export interface BrowserActAction {
   arguments?: string[];
 }
 
+/** A pre-resolved action from `observe()`, passable to `act()` for a no-LLM replay. */
+export type BrowserAction = BrowserObserveElement | BrowserActAction;
+
 /** Result of one natural-language `tab.act()` call. */
 export interface BrowserActResult {
   success: boolean;