diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/mcp-apps-renderer.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/mcp-apps-renderer.test.tsx index 5060923e8..9957c9f85 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/mcp-apps-renderer.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/mcp-apps-renderer.test.tsx @@ -139,6 +139,9 @@ vi.mock("@/stores/widget-debug-store", () => ({ clearCspViolations: stableStoreFns.clearCspViolations, setWidgetModelContext: stableStoreFns.setWidgetModelContext, setWidgetHtml: stableStoreFns.setWidgetHtml, + setStreamingHistoryCount: vi.fn(), + setStreamingPlaybackActive: vi.fn(), + widgets: new Map(), }), })); @@ -156,9 +159,13 @@ vi.mock("@/lib/mcp-ui/mcp-apps-utils", () => ({ isVisibleToModelOnly: () => false, })); -vi.mock("lucide-react", () => ({ - X: (props: any) =>
, -})); +vi.mock("lucide-react", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + X: (props: any) =>
, + }; +}); vi.mock("../mcp-apps-modal", () => ({ McpAppsModal: () => null, diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/streaming-playback-bar.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/streaming-playback-bar.test.tsx new file mode 100644 index 000000000..146952d4e --- /dev/null +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/streaming-playback-bar.test.tsx @@ -0,0 +1,93 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { render, screen, fireEvent } from "@testing-library/react"; +import { StreamingPlaybackBar } from "../streaming-playback-bar"; +import type { PartialHistoryEntry } from "../useToolInputStreaming"; + +function createHistory(count: number): PartialHistoryEntry[] { + return Array.from({ length: count }, (_, i) => ({ + timestamp: 1000 + i * 100, + elapsedFromStart: i * 100, + input: { code: "x".repeat(i + 1) }, + isFinal: i === count - 1, + })); +} + +describe("StreamingPlaybackBar", () => { + const defaultProps = { + replayToPosition: vi.fn(), + exitReplay: vi.fn(), + isReplayActive: false, + toolCallId: "call-1", + }; + + beforeEach(() => { + vi.useFakeTimers(); + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("renders all transport control buttons", () => { + const history = createHistory(4); + render(); + + expect(screen.getByLabelText("Previous")).toBeInTheDocument(); + expect(screen.getByLabelText("Play")).toBeInTheDocument(); + expect(screen.getByLabelText("Next")).toBeInTheDocument(); + }); + + it("displays position label", () => { + const history = createHistory(4); + render(); + + // Initially at last position: "4/4" + expect(screen.getByText(/4\/4/)).toBeInTheDocument(); + }); + + it("Previous button calls replayToPosition with correct index", () => { + const history = createHistory(4); + const replayToPosition = vi.fn(); + render( + , + ); + + // First click Previous to move from position 3 to 2 + fireEvent.click(screen.getByLabelText("Previous")); + + expect(replayToPosition).toHaveBeenCalledWith(2); + }); + + it("Next button is disabled at last position", () => { + const history = createHistory(4); + render(); + + const nextButton = screen.getByLabelText("Next"); + expect(nextButton).toBeDisabled(); + }); + + it("renders speed selector with default value", () => { + const history = createHistory(4); + render(); + + expect(screen.getByLabelText("Playback speed")).toBeInTheDocument(); + }); + + it("renders timeline slider", () => { + const history = createHistory(4); + render(); + expect(screen.getByLabelText("Streaming timeline")).toBeInTheDocument(); + }); + + it("renders Raw JSON collapsible trigger", () => { + const history = createHistory(4); + render(); + + expect(screen.getByText("Raw JSON")).toBeInTheDocument(); + }); +}); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/useToolInputStreaming.test.ts b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/useToolInputStreaming.test.ts index 9e63072ea..732831faf 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/useToolInputStreaming.test.ts +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/__tests__/useToolInputStreaming.test.ts @@ -444,4 +444,221 @@ describe("useToolInputStreaming", () => { expect(bridge.sendToolCancelled).toHaveBeenCalledTimes(1); }); + + // ── Partial history tests ─────────────────────────────────────────────── + + describe("partialHistory", () => { + it("is empty during streaming, populated after transition to input-available", () => { + const props = createDefaultProps(bridge); + props.toolState = "input-streaming"; + props.toolInput = { code: "he" }; + + const { result, rerender } = renderHook(() => + useToolInputStreaming(props), + ); + + // During streaming, partialHistory state has not been snapshotted yet + expect(result.current.partialHistory).toEqual([]); + + // Transition to input-available + props.toolState = "input-available"; + props.toolInput = { code: "hello" }; + rerender(); + + // Now partialHistory should be populated (includes the streaming partial + final) + expect(result.current.partialHistory.length).toBeGreaterThan(0); + }); + + it("final entry has isFinal: true", () => { + const props = createDefaultProps(bridge); + props.toolState = "input-streaming"; + props.toolInput = { code: "he" }; + + const { result, rerender } = renderHook(() => + useToolInputStreaming(props), + ); + + // Transition to input-available + props.toolState = "input-available"; + props.toolInput = { code: "hello" }; + rerender(); + + const history = result.current.partialHistory; + expect(history.length).toBeGreaterThan(0); + expect(history[history.length - 1].isFinal).toBe(true); + }); + + it("elapsedFromStart is relative to first entry", () => { + const props = createDefaultProps(bridge); + props.toolState = "input-streaming"; + props.toolInput = { code: "h" }; + + const { result, rerender } = renderHook(() => + useToolInputStreaming(props), + ); + + // Advance time, change input + act(() => { + vi.advanceTimersByTime(PARTIAL_INPUT_THROTTLE_MS + 10); + }); + props.toolInput = { code: "he" }; + rerender(); + + // Transition to complete + props.toolState = "input-available"; + props.toolInput = { code: "hello" }; + rerender(); + + const history = result.current.partialHistory; + expect(history.length).toBeGreaterThanOrEqual(2); + expect(history[0].elapsedFromStart).toBe(0); + // Later entries should have positive elapsed time + for (let i = 1; i < history.length; i++) { + expect(history[i].elapsedFromStart).toBeGreaterThanOrEqual(0); + } + }); + + it("history resets on toolCallId change", () => { + const props = createDefaultProps(bridge); + props.toolState = "input-streaming"; + props.toolInput = { code: "hello" }; + + const { result, rerender } = renderHook(() => + useToolInputStreaming(props), + ); + + // Complete to populate history + props.toolState = "input-available"; + rerender(); + + expect(result.current.partialHistory.length).toBeGreaterThan(0); + + // Change toolCallId — history should reset + props.toolCallId = "call-2"; + props.toolState = "input-streaming"; + props.toolInput = { code: "new" }; + rerender(); + + expect(result.current.partialHistory).toEqual([]); + }); + + it("history keeps growing past 200 entries", () => { + const props = createDefaultProps(bridge); + props.toolState = "input-streaming"; + + const { result, rerender } = renderHook(() => + useToolInputStreaming(props), + ); + + // Send many distinct partials + const partialCount = 250; + for (let i = 0; i < partialCount; i++) { + act(() => { + vi.advanceTimersByTime(PARTIAL_INPUT_THROTTLE_MS + 10); + }); + props.toolInput = { code: `input-${i}` }; + rerender(); + } + + // Complete to snapshot history + props.toolState = "input-available"; + props.toolInput = { code: "final" }; + rerender(); + + // Includes all unique partials plus the final full input entry. + expect(result.current.partialHistory.length).toBeGreaterThan( + partialCount, + ); + }); + }); + + // ── Replay tests ──────────────────────────────────────────────────────── + + describe("replay", () => { + function setupWithHistory(b: ReturnType) { + const props = createDefaultProps(b); + props.toolState = "input-streaming"; + props.toolInput = { code: "h" }; + + const hookResult = renderHook(() => useToolInputStreaming(props)); + + // Advance and add another partial + act(() => { + vi.advanceTimersByTime(PARTIAL_INPUT_THROTTLE_MS + 10); + }); + props.toolInput = { code: "he" }; + hookResult.rerender(); + + // Complete + props.toolState = "input-available"; + props.toolInput = { code: "hello" }; + hookResult.rerender(); + + // Clear mock call history for cleaner assertions + b.sendToolInputPartial.mockClear(); + b.sendToolInput.mockClear(); + + return { hookResult, props }; + } + + it("replayToPosition(0) calls bridge.sendToolInputPartial", () => { + const { hookResult } = setupWithHistory(bridge); + // The first entry in history is { code: "he" } — the mount-time entry + // { code: "h" } gets cleared by the reset effect that runs in the same cycle. + const firstEntry = hookResult.result.current.partialHistory[0]; + + act(() => { + hookResult.result.current.replayToPosition(0); + }); + + expect(bridge.sendToolInputPartial).toHaveBeenCalledWith({ + arguments: firstEntry.input, + }); + }); + + it("replayToPosition(lastIndex) calls bridge.sendToolInput for final entry", () => { + const { hookResult } = setupWithHistory(bridge); + const lastIndex = hookResult.result.current.partialHistory.length - 1; + + act(() => { + hookResult.result.current.replayToPosition(lastIndex); + }); + + expect(bridge.sendToolInput).toHaveBeenCalledWith({ + arguments: { code: "hello" }, + }); + }); + + it("replayToPosition sets isReplayActive = true", () => { + const { hookResult } = setupWithHistory(bridge); + + expect(hookResult.result.current.isReplayActive).toBe(false); + + act(() => { + hookResult.result.current.replayToPosition(0); + }); + + expect(hookResult.result.current.isReplayActive).toBe(true); + }); + + it("exitReplay calls bridge.sendToolInput with final args and sets isReplayActive = false", () => { + const { hookResult } = setupWithHistory(bridge); + + act(() => { + hookResult.result.current.replayToPosition(0); + }); + expect(hookResult.result.current.isReplayActive).toBe(true); + + bridge.sendToolInput.mockClear(); + + act(() => { + hookResult.result.current.exitReplay(); + }); + + expect(hookResult.result.current.isReplayActive).toBe(false); + expect(bridge.sendToolInput).toHaveBeenCalledWith({ + arguments: { code: "hello" }, + }); + }); + }); }); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/mcp-apps-renderer.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/mcp-apps-renderer.tsx index 322c279cb..e92f09a44 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/mcp-apps-renderer.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/mcp-apps-renderer.tsx @@ -17,7 +17,11 @@ import { useCallback, type CSSProperties, } from "react"; -import { useToolInputStreaming, type ToolState } from "./useToolInputStreaming"; +import { + useToolInputStreaming, + type ToolState, + type StreamingPlaybackData, +} from "./useToolInputStreaming"; import { usePreferencesStore } from "@/stores/preferences/preferences-provider"; import { useUIPlaygroundStore, @@ -121,6 +125,8 @@ interface MCPAppsRendererProps { ) => void; /** Callback when app declares its supported display modes during ui/initialize */ onAppSupportedDisplayModesChange?: (modes: DisplayMode[] | undefined) => void; + /** Callback when streaming playback data changes (for embedding playback bar elsewhere) */ + onStreamingPlaybackDataChange?: (data: StreamingPlaybackData | null) => void; /** Whether the server is offline (for using cached content) */ isOffline?: boolean; /** URL to cached widget HTML for offline rendering */ @@ -150,6 +156,7 @@ export function MCPAppsRenderer({ onExitFullscreen, onModelContextUpdate, onAppSupportedDisplayModesChange, + onStreamingPlaybackDataChange, isOffline, cachedWidgetHtmlUrl, }: MCPAppsRendererProps) { @@ -252,6 +259,7 @@ export function MCPAppsRenderer({ const [widgetPermissive, setWidgetPermissive] = useState(false); const [prefersBorder, setPrefersBorder] = useState(true); const [loadedCspMode, setLoadedCspMode] = useState(null); + const [replayResetNonce, setReplayResetNonce] = useState(0); // Modal state const [modalOpen, setModalOpen] = useState(false); const [modalParams, setModalParams] = useState>({}); @@ -274,6 +282,11 @@ export function MCPAppsRenderer({ const hostContextRef = useRef(null); const isReadyRef = useRef(false); const lastInlineHeightRef = useRef("400px"); + const replayResetNonceRef = useRef(0); + const pendingReplayResetRef = useRef<{ + nonce: number; + resolve: () => void; + } | null>(null); const onSendFollowUpRef = useRef(onSendFollowUp); const onCallToolRef = useRef(onCallTool); @@ -302,14 +315,34 @@ export function MCPAppsRenderer({ const themeModeRef = useRef(themeMode); themeModeRef.current = themeMode; + const requestReplaySessionReset = useCallback((): Promise => { + setIsReady(false); + isReadyRef.current = false; + + const nextNonce = replayResetNonceRef.current + 1; + replayResetNonceRef.current = nextNonce; + setReplayResetNonce(nextNonce); + + return new Promise((resolve) => { + // Unblock any older in-flight replay reset request. + pendingReplayResetRef.current?.resolve(); + pendingReplayResetRef.current = { nonce: nextNonce, resolve }; + }); + }, []); + const { canRenderStreamingInput, signalStreamingRender, resetStreamingState, + partialHistory, + replayToPosition, + exitReplay, + isReplayActive, } = useToolInputStreaming({ bridgeRef, isReady, isReadyRef, + requestReplaySessionReset, toolState, toolInput, toolOutput, @@ -446,7 +479,9 @@ export function MCPAppsRenderer({ (s) => s.setWidgetModelContext, ); const setWidgetHtmlStore = useWidgetDebugStore((s) => s.setWidgetHtml); - + const setStreamingHistoryCount = useWidgetDebugStore( + (s) => s.setStreamingHistoryCount, + ); // Clear CSP violations when CSP mode changes (stale data from previous mode) useEffect(() => { if (loadedCspMode !== null && loadedCspMode !== cspMode) { @@ -520,6 +555,34 @@ export function MCPAppsRenderer({ setWidgetGlobals, ]); + // Write streaming history count to debug store + useEffect(() => { + if (partialHistory.length > 0) { + setStreamingHistoryCount(toolCallId, partialHistory.length); + } + }, [partialHistory.length, toolCallId, setStreamingHistoryCount]); + + // Push streaming playback data to parent for embedding in ToolPart + useEffect(() => { + if (!onStreamingPlaybackDataChange) return; + if (partialHistory.length > 1) { + onStreamingPlaybackDataChange({ + partialHistory, + replayToPosition, + exitReplay, + isReplayActive, + }); + } else { + onStreamingPlaybackDataChange(null); + } + }, [ + partialHistory, + replayToPosition, + exitReplay, + isReplayActive, + onStreamingPlaybackDataChange, + ]); + // CSS Variables for theming (SEP-1865 styles.variables) // These are sent via hostContext.styles.variables - the SDK should pass them through const useChatGPTStyle = isPlaygroundActive && hostStyle === "chatgpt"; @@ -621,6 +684,14 @@ export function MCPAppsRenderer({ bridge.oninitialized = () => { setIsReady(true); isReadyRef.current = true; + const pendingReset = pendingReplayResetRef.current; + if ( + pendingReset && + pendingReset.nonce === replayResetNonceRef.current + ) { + pendingReplayResetRef.current = null; + pendingReset.resolve(); + } const appCaps = bridge.getAppCapabilities(); onAppSupportedDisplayModesChangeRef.current?.( appCaps?.availableDisplayModes as DisplayMode[] | undefined, @@ -872,6 +943,11 @@ export function MCPAppsRenderer({ let isActive = true; bridge.connect(transport).catch((error) => { if (!isActive) return; + const pendingReset = pendingReplayResetRef.current; + if (pendingReset && pendingReset.nonce === replayResetNonceRef.current) { + pendingReplayResetRef.current = null; + pendingReset.resolve(); + } setLoadError( error instanceof Error ? error.message : "Failed to connect MCP App", ); @@ -884,6 +960,11 @@ export function MCPAppsRenderer({ bridge.teardownResource({}).catch(() => {}); } bridge.close().catch(() => {}); + const pendingReset = pendingReplayResetRef.current; + if (pendingReset && pendingReset.nonce === replayResetNonceRef.current) { + pendingReplayResetRef.current = null; + pendingReset.resolve(); + } // Clear model context on widget teardown setWidgetModelContext(toolCallId, null); }; @@ -892,6 +973,7 @@ export function MCPAppsRenderer({ serverId, toolCallId, widgetHtml, + replayResetNonce, registerBridgeHandlers, setWidgetModelContext, ]); @@ -1164,6 +1246,7 @@ export function MCPAppsRenderer({ )} {/* Uses SandboxedIframe for DRY double-iframe architecture */} void; + exitReplay: () => void; + isReplayActive: boolean; + toolCallId: string; +} + +const SPEED_OPTIONS = ["0.25", "0.5", "1", "2", "4"] as const; + +function getPlaybackDelay( + entries: PartialHistoryEntry[], + current: number, + next: number, + speed: number, +): number { + const gap = + entries[next].elapsedFromStart - entries[current].elapsedFromStart; + const scaled = gap / speed; + const maxGap = 2000 / speed; + return Math.max(32, Math.min(scaled, maxGap)); +} + +function findNearestEntryIndex( + entries: PartialHistoryEntry[], + timeMs: number, +): number { + let lo = 0; + let hi = entries.length - 1; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (entries[mid].elapsedFromStart < timeMs) lo = mid + 1; + else hi = mid; + } + if (lo > 0) { + const before = Math.abs(entries[lo - 1].elapsedFromStart - timeMs); + const after = Math.abs(entries[lo].elapsedFromStart - timeMs); + return before <= after ? lo - 1 : lo; + } + return lo; +} + +function TimeSlider({ + entries, + currentPosition, + onSeek, +}: { + entries: PartialHistoryEntry[]; + currentPosition: number; + onSeek: (position: number) => void; +}) { + const totalDuration = + entries.length > 1 ? entries[entries.length - 1].elapsedFromStart : 1; + const currentTime = entries[currentPosition]?.elapsedFromStart ?? 0; + + return ( + { + const index = findNearestEntryIndex(entries, time); + onSeek(index); + }} + className="flex-1 mx-1" + aria-label="Streaming timeline" + /> + ); +} + +export function StreamingPlaybackBar({ + partialHistory, + replayToPosition, + exitReplay, + isReplayActive, + toolCallId, +}: StreamingPlaybackBarProps) { + const [currentPosition, setCurrentPosition] = useState( + partialHistory.length - 1, + ); + const [isPlaying, setIsPlaying] = useState(false); + const [playbackSpeed, setPlaybackSpeed] = useState("1"); + const [jsonPanelOpen, setJsonPanelOpen] = useState(false); + const playTimerRef = useRef(null); + + const lastIndex = partialHistory.length - 1; + + const stopPlayback = useCallback(() => { + if (playTimerRef.current !== null) { + window.clearTimeout(playTimerRef.current); + playTimerRef.current = null; + } + setIsPlaying(false); + }, []); + + const goToPosition = useCallback( + (position: number) => { + const clamped = Math.max(0, Math.min(position, lastIndex)); + setCurrentPosition(clamped); + replayToPosition(clamped); + }, + [lastIndex, replayToPosition], + ); + + // Auto-play effect + useEffect(() => { + if (!isPlaying) return; + if (currentPosition >= lastIndex) { + setIsPlaying(false); + return; + } + + const speed = parseFloat(playbackSpeed); + const delay = getPlaybackDelay( + partialHistory, + currentPosition, + currentPosition + 1, + speed, + ); + + playTimerRef.current = window.setTimeout(() => { + playTimerRef.current = null; + const nextPos = currentPosition + 1; + setCurrentPosition(nextPos); + replayToPosition(nextPos); + }, delay); + + return () => { + if (playTimerRef.current !== null) { + window.clearTimeout(playTimerRef.current); + playTimerRef.current = null; + } + }; + }, [ + isPlaying, + currentPosition, + lastIndex, + playbackSpeed, + partialHistory, + replayToPosition, + ]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (playTimerRef.current !== null) { + window.clearTimeout(playTimerRef.current); + } + }; + }, []); + + const handlePlayPause = () => { + if (isPlaying) { + stopPlayback(); + } else { + // If at the end, restart from beginning + if (currentPosition >= lastIndex) { + setCurrentPosition(0); + replayToPosition(0); + } + setIsPlaying(true); + } + }; + + const currentEntry = partialHistory[currentPosition]; + const elapsedMs = currentEntry?.elapsedFromStart ?? 0; + + return ( +
+ {/* Transport controls row */} +
+ {/* Navigation buttons */} +
+ + + + + Previous + + + + + + + {isPlaying ? "Pause" : "Play"} + + + + + + + Next + +
+ + {/* Separator */} +
+ + {/* Position label */} + + {currentPosition + 1}/{partialHistory.length}{" "} + +{elapsedMs}ms + + + {/* Timeline slider */} + { + stopPlayback(); + goToPosition(pos); + }} + /> + + {/* Speed selector */} + +
+ + {/* Collapsible JSON panel */} + + + + Raw JSON + + +
+ +
+ + +
+ ); +} diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/useToolInputStreaming.ts b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/useToolInputStreaming.ts index d640278f7..97e54a718 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/useToolInputStreaming.ts +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/mcp-apps/useToolInputStreaming.ts @@ -22,6 +22,20 @@ export const SIGNATURE_STRING_EDGE_LENGTH = 24; // ── Types ──────────────────────────────────────────────────────────────────── +export interface PartialHistoryEntry { + timestamp: number; + elapsedFromStart: number; + input: Record; + isFinal?: boolean; +} + +export interface StreamingPlaybackData { + partialHistory: PartialHistoryEntry[]; + replayToPosition: (position: number) => void; + exitReplay: () => void; + isReplayActive: boolean; +} + export type ToolState = | "input-streaming" | "input-available" @@ -147,6 +161,14 @@ export interface UseToolInputStreamingReturn { signalStreamingRender: () => void; /** Called on CSP mode change (or externally) to clear all streaming state. */ resetStreamingState: () => void; + /** Recorded history of partial inputs (populated after streaming completes). */ + partialHistory: PartialHistoryEntry[]; + /** Replay the widget to a specific history position. */ + replayToPosition: (position: number) => void; + /** Exit replay mode and restore widget to final state. */ + exitReplay: () => void; + /** Whether the hook is currently in replay mode. */ + isReplayActive: boolean; } // ── Hook ───────────────────────────────────────────────────────────────────── @@ -176,11 +198,19 @@ export function useToolInputStreaming({ const toolInputSentRef = useRef(false); const previousToolStateRef = useRef(toolState); + // ── History recording refs ────────────────────────────────────────────── + const partialHistoryRef = useRef([]); + const recordingStartTimeRef = useRef(null); + // ── Internal state ─────────────────────────────────────────────────────── const [streamingRenderSignaled, setStreamingRenderSignaled] = useState(false); const [hasDeliveredStreamingInput, setHasDeliveredStreamingInput] = useState(false); + const [partialHistory, setPartialHistory] = useState( + [], + ); + const [isReplayActive, setIsReplayActive] = useState(false); // ── Derived values ─────────────────────────────────────────────────────── @@ -214,12 +244,77 @@ export function useToolInputStreaming({ toolInputSentRef.current = false; setStreamingRenderSignaled(false); setHasDeliveredStreamingInput(false); + partialHistoryRef.current = []; + recordingStartTimeRef.current = null; + setPartialHistory([]); + setIsReplayActive(false); }, []); const signalStreamingRender = useCallback(() => { setStreamingRenderSignaled(true); }, []); + const recordPartialEntry = useCallback( + (input: Record, isFinal?: boolean) => { + const now = Date.now(); + if (recordingStartTimeRef.current === null) { + recordingStartTimeRef.current = now; + } + partialHistoryRef.current.push({ + timestamp: now, + elapsedFromStart: now - recordingStartTimeRef.current, + input: structuredClone(input), + isFinal, + }); + }, + [], + ); + + // ── Replay callbacks ────────────────────────────────────────────────────── + + const replayToPosition = useCallback( + (position: number) => { + const bridge = bridgeRef.current; + if (!bridge || !isReadyRef.current) return; + const history = partialHistoryRef.current; + if (position < 0 || position >= history.length) return; + + setIsReplayActive(true); + + // Clear dedup guards so bridge accepts the replayed message + lastToolInputRef.current = null; + lastToolInputPartialRef.current = null; + + const entry = history[position]; + if (entry.isFinal) { + Promise.resolve(bridge.sendToolInput({ arguments: entry.input })).catch( + () => {}, + ); + } else { + Promise.resolve( + bridge.sendToolInputPartial({ arguments: entry.input }), + ).catch(() => {}); + } + }, + [bridgeRef, isReadyRef], + ); + + const exitReplay = useCallback(() => { + setIsReplayActive(false); + + // Restore widget to final state + const bridge = bridgeRef.current; + if (!bridge || !isReadyRef.current) return; + const history = partialHistoryRef.current; + if (history.length === 0) return; + + const finalEntry = history[history.length - 1]; + lastToolInputRef.current = null; + Promise.resolve( + bridge.sendToolInput({ arguments: finalEntry.input }), + ).catch(() => {}); + }, [bridgeRef, isReadyRef]); + // ── Effects ────────────────────────────────────────────────────────────── // 1. Clear reveal timer when signaled @@ -280,6 +375,7 @@ export function useToolInputStreaming({ lastToolInputPartialSentAtRef.current = Date.now(); setHasDeliveredStreamingInput(true); setStreamingRenderSignaled(true); + recordPartialEntry(pending); Promise.resolve( bridge.sendToolInputPartial({ arguments: pending }), ).catch(() => {}); @@ -306,7 +402,15 @@ export function useToolInputStreaming({ partialInputTimerRef.current = null; flushPartialInput(); }, PARTIAL_INPUT_THROTTLE_MS - elapsed); - }, [hasToolInputData, isReady, toolInput, toolState, bridgeRef, isReadyRef]); + }, [ + hasToolInputData, + isReady, + toolInput, + toolState, + bridgeRef, + isReadyRef, + recordPartialEntry, + ]); // 5. Complete input delivery useEffect(() => { @@ -335,13 +439,15 @@ export function useToolInputStreaming({ } lastToolInputRef.current = serialized; toolInputSentRef.current = true; + recordPartialEntry(resolvedToolInput, true); + setPartialHistory([...partialHistoryRef.current]); Promise.resolve( bridge.sendToolInput({ arguments: resolvedToolInput }), ).catch(() => { toolInputSentRef.current = false; lastToolInputRef.current = null; }); - }, [isReady, toolInput, toolState, bridgeRef]); + }, [isReady, toolInput, toolState, bridgeRef, recordPartialEntry]); // 6. Tool result delivery useEffect(() => { @@ -390,5 +496,9 @@ export function useToolInputStreaming({ canRenderStreamingInput, signalStreamingRender, resetStreamingState, + partialHistory, + replayToPosition, + exitReplay, + isReplayActive, }; } diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx index 9f3ab3f5e..4eaeee304 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx @@ -1,4 +1,5 @@ import { useState, useCallback, useMemo } from "react"; +import type { StreamingPlaybackData } from "./mcp-apps/useToolInputStreaming"; import { type ToolUIPart, type DynamicToolUIPart, type UITools } from "ai"; import { UIMessage } from "@ai-sdk/react"; import type { ContentBlock } from "@modelcontextprotocol/sdk/types.js"; @@ -94,6 +95,8 @@ export function PartSwitch({ const [appSupportedDisplayModes, setAppSupportedDisplayModes] = useState< DisplayMode[] | undefined >(); + const [streamingPlaybackData, setStreamingPlaybackData] = + useState(null); void messageParts; // Get auth and app state for saving views @@ -427,6 +430,7 @@ export function PartSwitch({ onRequestPip={onRequestPip} onExitPip={onExitPip} appSupportedDisplayModes={appSupportedDisplayModes} + streamingPlaybackData={streamingPlaybackData} onSaveView={showSaveViewButton ? handleSaveView : undefined} canSaveView={showSaveViewButton ? canSaveView : undefined} saveDisabledReason={ @@ -461,6 +465,7 @@ export function PartSwitch({ onRequestFullscreen={onRequestFullscreen} onExitFullscreen={onExitFullscreen} onAppSupportedDisplayModesChange={setAppSupportedDisplayModes} + onStreamingPlaybackDataChange={setStreamingPlaybackData} isOffline={renderOverride?.isOffline} cachedWidgetHtmlUrl={renderOverride?.cachedWidgetHtmlUrl} /> diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx index 11aa6651c..89dfdfd6e 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx @@ -5,6 +5,7 @@ import { ChevronDown, Database, Layers, + ListVideo, Loader2, Maximize2, MessageCircle, @@ -37,6 +38,8 @@ import { import { CspDebugPanel } from "../csp-debug-panel"; import { JsonEditor } from "@/components/ui/json-editor"; import { cn } from "@/lib/chat-utils"; +import type { StreamingPlaybackData } from "../mcp-apps/useToolInputStreaming"; +import { StreamingPlaybackBar } from "../mcp-apps/streaming-playback-bar"; type ApprovalVisualState = "pending" | "approved" | "denied"; const SAVE_VIEW_BUTTON_USED_KEY = "mcpjam-save-view-button-used"; @@ -61,6 +64,7 @@ export function ToolPart({ canSaveView, saveDisabledReason, isSaving, + streamingPlaybackData, }: { part: ToolUIPart | DynamicToolUIPart; uiType?: UIType | null; @@ -74,6 +78,8 @@ export function ToolPart({ onExitPip?: (toolCallId: string) => void; /** Display modes the app declared support for. If undefined, all modes are available. */ appSupportedDisplayModes?: DisplayMode[]; + /** Streaming playback data lifted from MCPAppsRenderer for embedding playback bar */ + streamingPlaybackData?: StreamingPlaybackData | null; approvalId?: string; onApprove?: (id: string) => void; onDeny?: (id: string) => void; @@ -124,7 +130,7 @@ export function ToolPart({ const [userExpanded, setUserExpanded] = useState(false); const isExpanded = needsApproval || userExpanded; const [activeDebugTab, setActiveDebugTab] = useState< - "data" | "state" | "csp" | "context" | null + "data" | "state" | "csp" | "context" | "streaming" | null >("data"); const [hasUsedSaveViewButton, setHasUsedSaveViewButton] = useState(true); @@ -138,6 +144,9 @@ export function ToolPart({ const widgetDebugInfo = useWidgetDebugStore((s) => toolCallId ? s.widgets.get(toolCallId) : undefined, ); + const streamingHistoryCount = useWidgetDebugStore((s) => + toolCallId ? (s.widgets.get(toolCallId)?.streamingHistoryCount ?? 0) : 0, + ); const hasWidgetDebug = !!widgetDebugInfo; const showDisplayModeControls = @@ -158,7 +167,7 @@ export function ToolPart({ const debugOptions = useMemo(() => { const options: { - tab: "data" | "state" | "csp" | "context"; + tab: "data" | "state" | "csp" | "context" | "streaming"; icon: typeof Database; label: string; badge?: number; @@ -184,15 +193,35 @@ export function ToolPart({ badge: widgetDebugInfo?.csp?.violations?.length, }); + if (uiType === UIType.MCP_APPS && streamingHistoryCount > 1) { + options.push({ + tab: "streaming", + icon: ListVideo, + label: "Streaming", + badge: streamingHistoryCount, + }); + } + return options; }, [ uiType, widgetDebugInfo?.csp?.violations?.length, widgetDebugInfo?.modelContext, + streamingHistoryCount, ]); - const handleDebugClick = (tab: "data" | "state" | "csp" | "context") => { + const handleDebugClick = ( + tab: "data" | "state" | "csp" | "context" | "streaming", + ) => { + // Exit streaming replay when switching away from streaming tab + if (activeDebugTab === "streaming" && tab !== "streaming") { + streamingPlaybackData?.exitReplay(); + } + if (activeDebugTab === tab) { + if (tab === "streaming") { + streamingPlaybackData?.exitReplay(); + } setActiveDebugTab(null); setUserExpanded(false); } else { @@ -303,7 +332,9 @@ export function ToolPart({ ? "State" : tab === "csp" ? "CSP" - : "Context"; + : tab === "streaming" + ? "Streaming" + : "Context"; const tooltipLabel = tab === "data" ? "Data" @@ -311,7 +342,9 @@ export function ToolPart({ ? "Widget State" : tab === "csp" ? "CSP" - : "Model Context"; + : tab === "streaming" + ? "Streaming Playback" + : "Model Context"; return ( @@ -326,7 +359,7 @@ export function ToolPart({ className={`inline-flex items-center gap-1 px-1.5 py-1 rounded transition-colors cursor-pointer relative ${ activeDebugTab === tab ? "bg-background text-foreground shadow-sm" - : badge && badge > 0 + : badge && badge > 0 && tab !== "streaming" ? "text-destructive hover:text-destructive hover:bg-destructive/10" : "text-muted-foreground/60 hover:text-muted-foreground hover:bg-background/50" }`} @@ -337,8 +370,10 @@ export function ToolPart({ {badge !== undefined && badge > 0 && ( {badge} @@ -474,7 +509,7 @@ export function ToolPart({
)} e.stopPropagation()} > {renderDebugOptionButtons()} @@ -667,6 +702,22 @@ export function ToolPart({ )}
)} + {hasWidgetDebug && + activeDebugTab === "streaming" && + (streamingPlaybackData && + streamingPlaybackData.partialHistory.length > 1 ? ( + + ) : ( +
+ {streamingHistoryCount} streaming snapshots recorded. +
+ ))} {!hasWidgetDebug && (
{hasInput && ( diff --git a/mcpjam-inspector/client/src/stores/widget-debug-store.ts b/mcpjam-inspector/client/src/stores/widget-debug-store.ts index 9fda9a6c1..5a0a27c84 100644 --- a/mcpjam-inspector/client/src/stores/widget-debug-store.ts +++ b/mcpjam-inspector/client/src/stores/widget-debug-store.ts @@ -95,6 +95,10 @@ export interface WidgetDebugInfo { } | null; /** Cached widget HTML for offline rendering */ widgetHtml?: string; + /** Number of recorded streaming history entries (for debug tab badge) */ + streamingHistoryCount?: number; + /** Whether streaming playback is currently active */ + streamingPlaybackActive?: boolean; } interface WidgetDebugStore { @@ -147,6 +151,12 @@ interface WidgetDebugStore { // Set widget HTML for offline rendering cache setWidgetHtml: (toolCallId: string, html: string) => void; + + // Set streaming history count for a widget + setStreamingHistoryCount: (toolCallId: string, count: number) => void; + + // Set streaming playback active state for a widget + setStreamingPlaybackActive: (toolCallId: string, active: boolean) => void; } export const useWidgetDebugStore = create((set, get) => ({ @@ -172,6 +182,8 @@ export const useWidgetDebugStore = create((set, get) => ({ csp: existing?.csp, // Preserve CSP violations across updates widgetHtml: existing?.widgetHtml, // Preserve cached HTML for save view feature modelContext: existing?.modelContext, // Preserve model context across updates + streamingHistoryCount: existing?.streamingHistoryCount, + streamingPlaybackActive: existing?.streamingPlaybackActive, updatedAt: Date.now(), }); return { widgets }; @@ -320,10 +332,42 @@ export const useWidgetDebugStore = create((set, get) => ({ globals: existing?.globals ?? { theme: "dark", displayMode: "inline" }, csp: existing?.csp, modelContext: existing?.modelContext, + streamingHistoryCount: existing?.streamingHistoryCount, + streamingPlaybackActive: existing?.streamingPlaybackActive, widgetHtml: html, updatedAt: Date.now(), }); return { widgets }; }); }, + + setStreamingHistoryCount: (toolCallId, count) => { + set((state) => { + const existing = state.widgets.get(toolCallId); + if (!existing) return state; + + const widgets = new Map(state.widgets); + widgets.set(toolCallId, { + ...existing, + streamingHistoryCount: count, + updatedAt: Date.now(), + }); + return { widgets }; + }); + }, + + setStreamingPlaybackActive: (toolCallId, active) => { + set((state) => { + const existing = state.widgets.get(toolCallId); + if (!existing) return state; + + const widgets = new Map(state.widgets); + widgets.set(toolCallId, { + ...existing, + streamingPlaybackActive: active, + updatedAt: Date.now(), + }); + return { widgets }; + }); + }, }));