From 37b92a4c8ddc74cb4a84683f8ba6302845f0cc2c Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Fri, 13 Feb 2026 15:20:38 +1100 Subject: [PATCH 01/15] Resolution, cropping, dragging for H264 --- .../experimental/camera-control-h264.tsx | 4 +- src/ui/experimental/camera-control.tsx | 119 ++++++++++++++--- src/ui/experimental/image-context.tsx | 16 ++- .../experimental/websocket-h264-provider.tsx | 120 ++++++++++++++++-- 4 files changed, 223 insertions(+), 36 deletions(-) diff --git a/src/docs/routes/experimental/camera-control-h264.tsx b/src/docs/routes/experimental/camera-control-h264.tsx index bd4ccfd..34c8789 100644 --- a/src/docs/routes/experimental/camera-control-h264.tsx +++ b/src/docs/routes/experimental/camera-control-h264.tsx @@ -18,8 +18,8 @@ function CameraControlWebsocketH264Demo() { const [mousePos, setMousePos] = useState<{ x: number, y: number, intensity: number } | null>(null); const [clickPos, setClickPos] = useState<{ x: number, y: number, intensity: number } | null>(null); return ( -
- +
+ void; onClick?: (pos: { x: number; y: number; intensity: number }) => void; showIntensity?: boolean; + onZoom?: (box: { startX: number; startY: number, width: number, height: number }) => void; } export const CameraControl: React.FC = ({ @@ -14,46 +15,74 @@ export const CameraControl: React.FC = ({ onMousePositionChange, onClick, showIntensity = false, + onZoom }) => { - const source = useContext(ImageContext); + const { image, reportSize, reportZoom, reportDrag, clearZoom } = useContext(ImageContext); const canvasRef = useRef(null); const [popupPos, setPopupPos] = useState<{ x: number; y: number } | null>(null); const [pixelValue, setPixelValue] = useState(null); const [cursorPosition, setCursorPosition] = useState<{ x: number; y: number } | null>(null); + const [zoomBox, setZoomBox] = useState<{ startX: number; startY: number; width: number; height: number } | null>(null); + const [dragStart, setDragStart] = useState<{ startX: number; startY: number; lastX: number, lastY: number } | null>(null); const [frameCount, setFrameCount] = useState(0); const [startTime, setStartTime] = useState(performance.now()); // Helper to get dimensions safely const getDimensions = () => { - if (!source) return { w: 0, h: 0 }; - if (typeof source === "object" && "video" in source && source.video) { - return { w: source.video.videoWidth, h: source.video.videoHeight }; + if (!image) return { w: 0, h: 0 }; + if (typeof image === "object" && "video" in image && image.video) { + return { w: image.video.videoWidth, h: image.video.videoHeight }; } - if (source instanceof ImageBitmap) { - return { w: source.width, h: source.height }; + if (image instanceof ImageBitmap) { + return { w: image.width, h: image.height }; } return { w: 0, h: 0 }; }; + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !reportSize) return; + + // TODO: This should be debounced + const observer = new ResizeObserver((entries) => { + const { width, height } = entries[0].contentRect; + // Set canvas to size of canvas? + canvas.width = width; + canvas.height = height; + + // Report the size back to the provider + reportSize(Math.floor(width), Math.floor(height)); + }); + + observer.observe(canvas); + return () => observer.disconnect(); + }, [reportSize]); + // Draw source when updated useEffect(() => { - if (!source || !canvasRef.current) return; + if (!image || !canvasRef.current) return; const canvas = canvasRef.current; const ctx = canvas.getContext("2d"); if (!ctx) return; - const { w, h } = getDimensions(); - if (w === 0 || h === 0) return; + // const { w, h } = getDimensions(); + // if (w === 0 || h === 0) return; - canvas.width = w; - canvas.height = h; + + // canvas.width = w; + // canvas.height = h; try { - if (typeof source === "object" && "video" in source && source.video) { - ctx.drawImage(source.video, 0, 0); - } else if (source instanceof ImageBitmap) { - ctx.drawImage(source, 0, 0); + if (typeof image === "object" && "video" in image && image.video) { + ctx.drawImage(image.video, 0, 0); + } else if (image instanceof ImageBitmap) { + ctx.drawImage(image, 0, 0); + } + if (zoomBox) { + ctx.strokeStyle = 'yellow'; // Set bounding box color + ctx.lineWidth = 2; // Set line width + ctx.strokeRect(zoomBox.startX, zoomBox.startY, zoomBox.width, zoomBox.height); } } catch (err) { console.warn("Draw failed:", err); @@ -64,19 +93,32 @@ export const CameraControl: React.FC = ({ setFrameCount((prev) => prev + 1); const now = performance.now(); if (now - startTime >= 1000) { - console.log(`FPS: ${frameCount}`); + // console.log(`FPS: ${frameCount}`); setFrameCount(0); setStartTime(now); } - }, [source instanceof ImageBitmap ? source : (source as any)?.frameId]); + }, [image instanceof ImageBitmap ? image : (image as any)?.frameId]); // Throttle mouse move let lastMove = 0; const handleMouseMove = (e: React.MouseEvent) => { + if (dragStart) { + + console.log(e.clientX); + + let deltaX = e.clientX - dragStart.lastX; + let deltaY = e.clientY - dragStart.lastY; + setDragStart({ ...dragStart, lastX: e.clientX, lastY: e.clientY }); + reportDrag(e.clientX - dragStart.startX, e.clientY - dragStart.startY, deltaX, deltaY, true); + return + } + const now = performance.now(); if (now - lastMove < 50) return; lastMove = now; + + const info = getMousePixelInfo(e); setPopupPos({ x: e.clientX, y: e.clientY }); setPixelValue(showIntensity ? `intensity: ${info.intensity}` : `rgba(${info.pixel[0]}, ${info.pixel[1]}, ${info.pixel[2]}, ${info.pixel[3] / 255})`); @@ -92,6 +134,11 @@ export const CameraControl: React.FC = ({ const y = Math.floor(e.clientY - rect.top); const ctx = canvas.getContext("2d"); + + if (zoomBox) { + setZoomBox({ startX: zoomBox.startX, startY: zoomBox.startY, width: x - zoomBox.startX, height: y - zoomBox.startY }) + } + if (!ctx) return { x, y, intensity: 0, pixel: [0, 0, 0, 0] }; try { @@ -119,21 +166,51 @@ export const CameraControl: React.FC = ({ } }; - const { w, h } = getDimensions(); + const handleMouseDown = (e: React.MouseEvent) => { + if (e.shiftKey) { + setDragStart({ startX: e.clientX, startY: e.clientY, lastX: e.clientX, lastY: e.clientY }); + } else { + const info = getMousePixelInfo(e); + setZoomBox({ startX: info.x, startY: info.y, width: 0, height: 0 }); + } + } + + const handleMouseUp = (e: React.MouseEvent) => { + if (dragStart) { + console.log(dragStart); + reportDrag(e.clientX - dragStart.startX, e.clientY - dragStart.startY, 0, 0, false) + setDragStart(null); + } + if (zoomBox) { + onZoom?.(zoomBox); + reportZoom(zoomBox.startX, zoomBox.startY, zoomBox.width, zoomBox.height); + } + setZoomBox(null); + } + + const handleDoubleClick = (e: React.MouseEvent) => { + clearZoom() + } + // const { w, h } = getDimensions(); return ( -
+
{/* Crosshair */} diff --git a/src/ui/experimental/image-context.tsx b/src/ui/experimental/image-context.tsx index 6f3bf9a..d002849 100644 --- a/src/ui/experimental/image-context.tsx +++ b/src/ui/experimental/image-context.tsx @@ -1,4 +1,16 @@ import { createContext } from "react"; -export type ImageSource = { video: HTMLVideoElement | null; frameId: number } | ImageBitmap | null; -export const ImageContext = createContext(null); \ No newline at end of file +export type ImageSource = { + image: { video: HTMLVideoElement | null; frameId: number } | ImageBitmap | null; + reportSize: (width: number, height: number) => void; + reportZoom: (startX: number, startY: number, width: number, height: number) => void; + reportDrag: (totalX: number, totalY: number, deltaX: number, deltaY: number, active: boolean) => void; + clearZoom: () => void; +} +export const ImageContext = createContext({ + image: null, + reportSize: () => { }, + reportZoom: () => { }, + reportDrag: () => { }, + clearZoom: () => { } +}); \ No newline at end of file diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index d63dcf8..a8c16a8 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useMemo, useCallback } from 'react'; import { ImageContext } from './image-context'; @@ -6,15 +6,19 @@ import { ImageContext } from './image-context'; export interface WebsocketH264ProviderProps { children: React.ReactNode, - wsUrl: string; + url: string; } -export const WebsocketH264Provider: React.FC = ({ children, wsUrl }) => { +export const WebsocketH264Provider: React.FC = ({ children, url }) => { + const [sessionID, setSessionID] = useState("019c5500-67da-79b0-b7bf-ebe373778106"); const [imageBitmap, setImageBitmap] = useState(null); + const [sourceWidth, setSourceWidth] = useState(1024); + const [sourceHeight, setSourceHeight] = useState(1024); const [currentWidth, setCurrentWidth] = useState(1024); const [currentHeight, setCurrentHeight] = useState(1024); + useEffect(() => { let ws: WebSocket; let decoder: VideoDecoder | null = null; @@ -146,7 +150,7 @@ export const WebsocketH264Provider: React.FC = ({ ch }; const connect = () => { - ws = new WebSocket(wsUrl); + ws = new WebSocket("ws://" + url + "/ws?session_id=" + sessionID); ws.binaryType = 'arraybuffer'; ws.onopen = () => { @@ -159,19 +163,17 @@ export const WebsocketH264Provider: React.FC = ({ ch try { const meta = JSON.parse(ev.data); if (meta.type === 'config') { - console.log("meta.width"); - console.log(meta.width); - console.log("meta.height"); - console.log(meta.height); + console.log(meta); setCurrentWidth(meta.width); setCurrentHeight(meta.height); + setSourceWidth(meta.source_width); + setSourceHeight(meta.source_height); configured = false; // force reconfigure } } catch (e) { console.warn("Failed to parse metadata:", e); } } else { - onAccessUnit(ev.data); } }; @@ -196,10 +198,106 @@ export const WebsocketH264Provider: React.FC = ({ ch ws?.close(); decoder?.close(); }; - }, [wsUrl]); + }, [url]); + + + const reportSize = useCallback(async (width: number, height: number) => { + + const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/resolution", { + method: "POST", + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ width, height }), + }); + }, []); + + const reportZoom = useCallback(async (startX: number, startY: number, width: number, height: number) => { + + const crop_response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { + method: "GET", + }); + + let { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json() ?? { x: 0, y: 0, width: sourceWidth, height: sourceHeight }; + + let xScale = currentCropWidth / currentWidth; + let yScale = currentCropHeight / currentHeight; + + if (width < 0){ + startX = startX + width; + width = -1*width; + } + if (height < 0){ + startY = startY + height; + height = -1*height; + } + + let x = currentCropX + Math.floor(startX * xScale); + let y = currentCropY + Math.floor(startY * yScale); + + let cropWidth = Math.floor(width * xScale); + let cropHeight = Math.floor(height * yScale); + + if (cropWidth == 0 || cropHeight == 0) { + return + } + + console.log(x, y, cropWidth, cropHeight); + + + const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { + method: "POST", + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ x, y, width: cropWidth, height: cropHeight }), + }); + }, [sourceWidth, currentWidth, sourceHeight, currentHeight]); + + const reportDrag = useCallback(async (totalX: number, totalY: number, deltaX: number, deltaY: number, active: boolean) => { + + if (!active) { + const crop_response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { + method: "GET", + }); + let { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json(); + + let xScale = currentCropWidth / currentWidth; + let yScale = currentCropHeight / currentHeight; + + + let x = currentCropX - Math.floor(totalX * xScale); + let y = currentCropY - Math.floor(totalY * yScale); + + console.log({currentCropX, currentCropWidth, currentWidth, xScale, totalX, shiftX: Math.floor(totalX * xScale), x}); + + + const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { + method: "POST", + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ x, y, width: currentCropWidth, height: currentCropHeight }), + }); + } + }, [currentWidth, currentHeight]); + + const clearZoom = useCallback(async () => { + const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { + method: "DELETE", + }); + }, []); + + const contextValue = useMemo(() => ({ + image: imageBitmap, + reportSize, + reportZoom, + reportDrag, + clearZoom + }), [imageBitmap, reportSize, reportZoom, clearZoom]); return ( - + {children} ) From e1e261df6f5fec2d3f629b22ff55b02d842f6d81 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Tue, 17 Feb 2026 14:31:15 +1100 Subject: [PATCH 02/15] Getting zoom and cropping working correctly. --- src/ui/experimental/camera-control.tsx | 194 ++++++++++++++---- .../experimental/websocket-h264-provider.tsx | 29 +-- 2 files changed, 164 insertions(+), 59 deletions(-) diff --git a/src/ui/experimental/camera-control.tsx b/src/ui/experimental/camera-control.tsx index a219724..25cc79c 100644 --- a/src/ui/experimental/camera-control.tsx +++ b/src/ui/experimental/camera-control.tsx @@ -2,12 +2,39 @@ import React, { useEffect, useRef, useState, useContext } from "react"; import { cn } from "../../lib/utils"; import { ImageContext } from "./image-context"; +function debounceResize( + fn: (entry: ResizeObserverEntry) => void, + delay: number, +) { + let timer: ReturnType | undefined; + + const debounced = (entry: ResizeObserverEntry) => { + if (timer) clearTimeout(timer); + timer = setTimeout(() => fn(entry), delay); + }; + + debounced.cancel = () => { + if (timer) clearTimeout(timer); + timer = undefined; + }; + + return debounced; +} + export interface CameraControlProps { className?: string; - onMousePositionChange?: (pos: { x: number; y: number; intensity: number } | null) => void; + onMousePositionChange?: ( + pos: { x: number; y: number; intensity: number } | null, + ) => void; onClick?: (pos: { x: number; y: number; intensity: number }) => void; showIntensity?: boolean; - onZoom?: (box: { startX: number; startY: number, width: number, height: number }) => void; + onZoom?: (box: { + startX: number; + startY: number; + width: number; + height: number; + }) => void; + sizeFollowsImage?: boolean; } export const CameraControl: React.FC = ({ @@ -15,15 +42,32 @@ export const CameraControl: React.FC = ({ onMousePositionChange, onClick, showIntensity = false, - onZoom + onZoom, + sizeFollowsImage = false, }) => { - const { image, reportSize, reportZoom, reportDrag, clearZoom } = useContext(ImageContext); + const { image, reportSize, reportZoom, reportDrag, clearZoom } = + useContext(ImageContext); const canvasRef = useRef(null); - const [popupPos, setPopupPos] = useState<{ x: number; y: number } | null>(null); + const [popupPos, setPopupPos] = useState<{ x: number; y: number } | null>( + null, + ); const [pixelValue, setPixelValue] = useState(null); - const [cursorPosition, setCursorPosition] = useState<{ x: number; y: number } | null>(null); - const [zoomBox, setZoomBox] = useState<{ startX: number; startY: number; width: number; height: number } | null>(null); - const [dragStart, setDragStart] = useState<{ startX: number; startY: number; lastX: number, lastY: number } | null>(null); + const [cursorPosition, setCursorPosition] = useState<{ + x: number; + y: number; + } | null>(null); + const [zoomBox, setZoomBox] = useState<{ + startX: number; + startY: number; + width: number; + height: number; + } | null>(null); + const [dragStart, setDragStart] = useState<{ + startX: number; + startY: number; + lastX: number; + lastY: number; + } | null>(null); const [frameCount, setFrameCount] = useState(0); const [startTime, setStartTime] = useState(performance.now()); @@ -43,16 +87,17 @@ export const CameraControl: React.FC = ({ const canvas = canvasRef.current; if (!canvas || !reportSize) return; - // TODO: This should be debounced - const observer = new ResizeObserver((entries) => { - const { width, height } = entries[0].contentRect; + const handler = debounceResize((entry) => { + const { width, height } = entry.contentRect; // Set canvas to size of canvas? canvas.width = width; canvas.height = height; // Report the size back to the provider reportSize(Math.floor(width), Math.floor(height)); - }); + }, 100); + + const observer = new ResizeObserver((entries) => handler(entries[0])); observer.observe(canvas); return () => observer.disconnect(); @@ -66,12 +111,12 @@ export const CameraControl: React.FC = ({ const ctx = canvas.getContext("2d"); if (!ctx) return; - // const { w, h } = getDimensions(); - // if (w === 0 || h === 0) return; - - - // canvas.width = w; - // canvas.height = h; + if (sizeFollowsImage) { + const { w, h } = getDimensions(); + if (w === 0 || h === 0) return; + canvas.width = w; + canvas.height = h; + } try { if (typeof image === "object" && "video" in image && image.video) { @@ -80,9 +125,14 @@ export const CameraControl: React.FC = ({ ctx.drawImage(image, 0, 0); } if (zoomBox) { - ctx.strokeStyle = 'yellow'; // Set bounding box color + ctx.strokeStyle = "yellow"; // Set bounding box color ctx.lineWidth = 2; // Set line width - ctx.strokeRect(zoomBox.startX, zoomBox.startY, zoomBox.width, zoomBox.height); + ctx.strokeRect( + zoomBox.startX, + zoomBox.startY, + zoomBox.width, + zoomBox.height, + ); } } catch (err) { console.warn("Draw failed:", err); @@ -103,31 +153,52 @@ export const CameraControl: React.FC = ({ let lastMove = 0; const handleMouseMove = (e: React.MouseEvent) => { if (dragStart) { - console.log(e.clientX); - let deltaX = e.clientX - dragStart.lastX; - let deltaY = e.clientY - dragStart.lastY; + const deltaX = e.clientX - dragStart.lastX; + const deltaY = e.clientY - dragStart.lastY; setDragStart({ ...dragStart, lastX: e.clientX, lastY: e.clientY }); - reportDrag(e.clientX - dragStart.startX, e.clientY - dragStart.startY, deltaX, deltaY, true); - return + reportDrag( + e.clientX - dragStart.startX, + e.clientY - dragStart.startY, + deltaX, + deltaY, + true, + ); + return; } const now = performance.now(); if (now - lastMove < 50) return; lastMove = now; + const info = getMousePixelInfo(e); + if (zoomBox) { + setZoomBox({ + ...zoomBox, + width: info.x - zoomBox.startX, + height: info.y - zoomBox.startY, + }); + } - const info = getMousePixelInfo(e); setPopupPos({ x: e.clientX, y: e.clientY }); - setPixelValue(showIntensity ? `intensity: ${info.intensity}` : `rgba(${info.pixel[0]}, ${info.pixel[1]}, ${info.pixel[2]}, ${info.pixel[3] / 255})`); - onMousePositionChange?.({ x: info.x, y: info.y, intensity: info.intensity }); + setPixelValue( + showIntensity + ? `intensity: ${info.intensity}` + : `rgba(${info.pixel[0]}, ${info.pixel[1]}, ${info.pixel[2]}, ${info.pixel[3] / 255})`, + ); + onMousePositionChange?.({ + x: info.x, + y: info.y, + intensity: info.intensity, + }); }; const getMousePixelInfo = (e: React.MouseEvent) => { const canvas = canvasRef.current; - if (!canvas || canvas.width === 0 || canvas.height === 0) return { x: 0, y: 0, intensity: 0, pixel: [0, 0, 0, 0] }; + if (!canvas || canvas.width === 0 || canvas.height === 0) + return { x: 0, y: 0, intensity: 0, pixel: [0, 0, 0, 0] }; const rect = canvas.getBoundingClientRect(); const x = Math.floor(e.clientX - rect.left); @@ -135,10 +206,6 @@ export const CameraControl: React.FC = ({ const ctx = canvas.getContext("2d"); - if (zoomBox) { - setZoomBox({ startX: zoomBox.startX, startY: zoomBox.startY, width: x - zoomBox.startX, height: y - zoomBox.startY }) - } - if (!ctx) return { x, y, intensity: 0, pixel: [0, 0, 0, 0] }; try { @@ -158,6 +225,7 @@ export const CameraControl: React.FC = ({ const handleMouseClick = (e: React.MouseEvent) => { const info = getMousePixelInfo(e); + onClick?.({ x: info.x, y: info.y, intensity: info.intensity }); if (e.ctrlKey) { setCursorPosition({ x: info.x, y: info.y }); @@ -168,17 +236,28 @@ export const CameraControl: React.FC = ({ const handleMouseDown = (e: React.MouseEvent) => { if (e.shiftKey) { - setDragStart({ startX: e.clientX, startY: e.clientY, lastX: e.clientX, lastY: e.clientY }); + setDragStart({ + startX: e.clientX, + startY: e.clientY, + lastX: e.clientX, + lastY: e.clientY, + }); } else { const info = getMousePixelInfo(e); setZoomBox({ startX: info.x, startY: info.y, width: 0, height: 0 }); } - } + }; const handleMouseUp = (e: React.MouseEvent) => { if (dragStart) { console.log(dragStart); - reportDrag(e.clientX - dragStart.startX, e.clientY - dragStart.startY, 0, 0, false) + reportDrag( + e.clientX - dragStart.startX, + e.clientY - dragStart.startY, + 0, + 0, + false, + ); setDragStart(null); } if (zoomBox) { @@ -186,15 +265,22 @@ export const CameraControl: React.FC = ({ reportZoom(zoomBox.startX, zoomBox.startY, zoomBox.width, zoomBox.height); } setZoomBox(null); - } + }; const handleDoubleClick = (e: React.MouseEvent) => { - clearZoom() - } - // const { w, h } = getDimensions(); + clearZoom(); + }; return ( -
+
= ({ display: "block", border: "1px solid red", width: "100%", - height: "100%" + height: "100%", // width: `${w}px`, // height: `${h}px`, }} @@ -226,8 +312,26 @@ export const CameraControl: React.FC = ({ zIndex: 10, }} > -
-
+
+
)} {/* Popup */} @@ -251,4 +355,4 @@ export const CameraControl: React.FC = ({ )}
); -}; \ No newline at end of file +}; diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index a8c16a8..3b320c8 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -11,7 +11,7 @@ export interface WebsocketH264ProviderProps { export const WebsocketH264Provider: React.FC = ({ children, url }) => { - const [sessionID, setSessionID] = useState("019c5500-67da-79b0-b7bf-ebe373778106"); + const [sessionID, setSessionID] = useState("019c6950-555b-7a13-afce-81c0e3188de4"); const [imageBitmap, setImageBitmap] = useState(null); const [sourceWidth, setSourceWidth] = useState(1024); const [sourceHeight, setSourceHeight] = useState(1024); @@ -218,10 +218,10 @@ export const WebsocketH264Provider: React.FC = ({ ch method: "GET", }); - let { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json() ?? { x: 0, y: 0, width: sourceWidth, height: sourceHeight }; + const { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json() ?? { x: 0, y: 0, width: sourceWidth, height: sourceHeight }; - let xScale = currentCropWidth / currentWidth; - let yScale = currentCropHeight / currentHeight; + const xScale = currentCropWidth / currentWidth; + const yScale = currentCropHeight / currentHeight; if (width < 0){ startX = startX + width; @@ -232,11 +232,11 @@ export const WebsocketH264Provider: React.FC = ({ ch height = -1*height; } - let x = currentCropX + Math.floor(startX * xScale); - let y = currentCropY + Math.floor(startY * yScale); + const x = currentCropX + Math.floor(startX * xScale); + const y = currentCropY + Math.floor(startY * yScale); - let cropWidth = Math.floor(width * xScale); - let cropHeight = Math.floor(height * yScale); + const cropWidth = Math.floor(width * xScale); + const cropHeight = Math.floor(height * yScale); if (cropWidth == 0 || cropHeight == 0) { return @@ -260,14 +260,15 @@ export const WebsocketH264Provider: React.FC = ({ ch const crop_response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { method: "GET", }); - let { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json(); + const { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json(); - let xScale = currentCropWidth / currentWidth; - let yScale = currentCropHeight / currentHeight; + const xScale = currentCropWidth / currentWidth; + const yScale = currentCropHeight / currentHeight; + const scale = Math.max(xScale, yScale); - let x = currentCropX - Math.floor(totalX * xScale); - let y = currentCropY - Math.floor(totalY * yScale); + const x = (currentCropX - Math.floor(totalX * scale)); + const y = (currentCropY - Math.floor(totalY * scale)); console.log({currentCropX, currentCropWidth, currentWidth, xScale, totalX, shiftX: Math.floor(totalX * xScale), x}); @@ -280,7 +281,7 @@ export const WebsocketH264Provider: React.FC = ({ ch body: JSON.stringify({ x, y, width: currentCropWidth, height: currentCropHeight }), }); } - }, [currentWidth, currentHeight]); + }, [sourceWidth, currentWidth, sourceHeight, currentHeight]); const clearZoom = useCallback(async () => { const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { From 7713cdec5ab7a3841492f880e59be7077368a306 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Wed, 18 Feb 2026 13:04:03 +1100 Subject: [PATCH 03/15] Cleaning up using refs. --- .../experimental/websocket-h264-provider.tsx | 65 +++++++++++-------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index 3b320c8..d695c76 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useMemo, useCallback } from 'react'; +import React, { useState, useEffect, useMemo, useCallback, useRef } from 'react'; import { ImageContext } from './image-context'; @@ -11,7 +11,7 @@ export interface WebsocketH264ProviderProps { export const WebsocketH264Provider: React.FC = ({ children, url }) => { - const [sessionID, setSessionID] = useState("019c6950-555b-7a13-afce-81c0e3188de4"); + const [sessionID, setSessionID] = useState(""); const [imageBitmap, setImageBitmap] = useState(null); const [sourceWidth, setSourceWidth] = useState(1024); const [sourceHeight, setSourceHeight] = useState(1024); @@ -19,13 +19,19 @@ export const WebsocketH264Provider: React.FC = ({ ch const [currentHeight, setCurrentHeight] = useState(1024); + const wsRef = useRef(null); + const abortedRef = useRef(false); + const configuredRef = useRef(false); + const decoderRef = useRef(null); + const reconnectTimerRef = useRef(null); + + useEffect(() => { let ws: WebSocket; - let decoder: VideoDecoder | null = null; - let configured = false; let sps: Uint8Array | null = null; let pps: Uint8Array | null = null; let nextTs = 0; + const frameDurationUs = Math.round(1_000_000 / 50); const splitAnnexB = (buf: ArrayBuffer): Uint8Array[] => { @@ -75,9 +81,9 @@ export const WebsocketH264Provider: React.FC = ({ ch }; const ensureDecoder = () => { - if (decoder) return; + if (decoderRef.current) return; - decoder = new VideoDecoder({ + decoderRef.current = new VideoDecoder({ output: async frame => { const bitmap = await createImageBitmap(frame); setImageBitmap(bitmap); @@ -88,7 +94,7 @@ export const WebsocketH264Provider: React.FC = ({ ch }; const tryConfigure = async (): Promise => { - if (configured || !sps || !pps) return false; + if (configuredRef.current || !sps || !pps) return false; ensureDecoder(); const description = buildAvcC(sps, pps); @@ -106,9 +112,9 @@ export const WebsocketH264Provider: React.FC = ({ ch return false; } - if (decoder!.state === 'configured') decoder!.reset(); - decoder!.configure(config); - configured = true; + if (decoderRef.current!.state === 'configured') decoderRef.current!.reset(); + decoderRef.current!.configure(config); + configuredRef.current = true; return true; }; @@ -132,7 +138,7 @@ export const WebsocketH264Provider: React.FC = ({ ch data: payload }); nextTs += frameDurationUs; - decoder!.decode(chunk); + decoderRef.current!.decode(chunk); }; const onAccessUnit = (buf: ArrayBuffer) => { @@ -145,17 +151,18 @@ export const WebsocketH264Provider: React.FC = ({ ch else if (t === 8) pps = n.slice(); } - if (!configured && sps && pps) tryConfigure().then(ok => { if (ok) feedChunk(nals); }); - else if (configured) feedChunk(nals); + if (!configuredRef.current && sps && pps) tryConfigure().then(ok => { if (ok) feedChunk(nals); }); + else if (configuredRef.current) feedChunk(nals); }; const connect = () => { ws = new WebSocket("ws://" + url + "/ws?session_id=" + sessionID); + wsRef.current = ws; ws.binaryType = 'arraybuffer'; ws.onopen = () => { console.log("Connected"); - configured = false; + configuredRef.current = false; }; ws.onmessage = ev => { @@ -168,7 +175,7 @@ export const WebsocketH264Provider: React.FC = ({ ch setCurrentHeight(meta.height); setSourceWidth(meta.source_width); setSourceHeight(meta.source_height); - configured = false; // force reconfigure + configuredRef.current = false; // force reconfigure } } catch (e) { console.warn("Failed to parse metadata:", e); @@ -179,11 +186,11 @@ export const WebsocketH264Provider: React.FC = ({ ch }; ws.onclose = () => { - configured = false; - try { - decoder?.flush().catch(() => { }); - } catch { } - setTimeout(connect, 3000); + configuredRef.current = false; + try { decoderRef.current?.flush().catch(() => {}); } catch {} + if (!abortedRef.current) { + reconnectTimerRef.current = window.setTimeout(connect, 3000); + } }; ws.onerror = e => { @@ -195,10 +202,16 @@ export const WebsocketH264Provider: React.FC = ({ ch connect(); return () => { - ws?.close(); - decoder?.close(); + abortedRef.current = true; + if (reconnectTimerRef.current !== null) { + clearTimeout(reconnectTimerRef.current); + reconnectTimerRef.current = null; + } + try { wsRef.current?.close(); } catch {} + try { decoderRef.current?.close(); } catch {} + }; - }, [url]); + }, [currentHeight, currentWidth, sessionID, url]); const reportSize = useCallback(async (width: number, height: number) => { @@ -252,7 +265,7 @@ export const WebsocketH264Provider: React.FC = ({ ch }, body: JSON.stringify({ x, y, width: cropWidth, height: cropHeight }), }); - }, [sourceWidth, currentWidth, sourceHeight, currentHeight]); + }, [url, sessionID, sourceWidth, sourceHeight, currentWidth, currentHeight]); const reportDrag = useCallback(async (totalX: number, totalY: number, deltaX: number, deltaY: number, active: boolean) => { @@ -281,7 +294,7 @@ export const WebsocketH264Provider: React.FC = ({ ch body: JSON.stringify({ x, y, width: currentCropWidth, height: currentCropHeight }), }); } - }, [sourceWidth, currentWidth, sourceHeight, currentHeight]); + }, [url, sessionID, currentWidth, currentHeight]); const clearZoom = useCallback(async () => { const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { @@ -295,7 +308,7 @@ export const WebsocketH264Provider: React.FC = ({ ch reportZoom, reportDrag, clearZoom - }), [imageBitmap, reportSize, reportZoom, clearZoom]); + }), [imageBitmap, reportSize, reportZoom, reportDrag, clearZoom]); return ( From 9bf86bb9bf7462adb5dcb248f9949b6c153159ce Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Wed, 18 Feb 2026 15:56:48 +1100 Subject: [PATCH 04/15] Formatted. Fixed references/state so that ws isn't reconnected on image resize. --- .../experimental/websocket-h264-provider.tsx | 568 +++++++++++------- 1 file changed, 344 insertions(+), 224 deletions(-) diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index d695c76..d51f7f9 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -1,181 +1,236 @@ -import React, { useState, useEffect, useMemo, useCallback, useRef } from 'react'; - - -import { ImageContext } from './image-context'; +import React, { + useState, + useEffect, + useMemo, + useCallback, + useRef, +} from "react"; +import { ImageContext } from "./image-context"; export interface WebsocketH264ProviderProps { - children: React.ReactNode, + children: React.ReactNode; url: string; } -export const WebsocketH264Provider: React.FC = ({ children, url }) => { - - const [sessionID, setSessionID] = useState(""); +export const WebsocketH264Provider: React.FC = ({ + children, + url, +}) => { + // ================== + // State + // ================== + const [sessionID, setSessionID] = useState( + "019c6ef1-821e-7b7b-a28b-085f73d0a248", + ); const [imageBitmap, setImageBitmap] = useState(null); const [sourceWidth, setSourceWidth] = useState(1024); const [sourceHeight, setSourceHeight] = useState(1024); const [currentWidth, setCurrentWidth] = useState(1024); const [currentHeight, setCurrentHeight] = useState(1024); - + // ================== + // Refs + // ================== const wsRef = useRef(null); + const dimsRef = useRef<{ width: number; height: number }>({ + width: 1024, + height: 1024, + }); const abortedRef = useRef(false); const configuredRef = useRef(false); const decoderRef = useRef(null); const reconnectTimerRef = useRef(null); + const spsRef = useRef(null); + const ppsRef = useRef(null); + const nextTsRef = useRef(0); + + // ================== + // Helper Functions + // ================== + + const frameDurationUs = Math.round(1_000_000 / 50); + + const splitAnnexB = (buf: ArrayBuffer): Uint8Array[] => { + const b = new Uint8Array(buf), + out: Uint8Array[] = []; + let i = 0; + const isStart = (i: number) => + (i + 3 < b.length && b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1) || + (i + 4 < b.length && + b[i] === 0 && + b[i + 1] === 0 && + b[i + 2] === 0 && + b[i + 3] === 1); + const consumeStart = (i: number) => (b[i + 2] === 1 ? i + 3 : i + 4); + + while (i < b.length - 3 && !isStart(i)) i++; + if (i >= b.length - 3) return out; + i = consumeStart(i); + let start = i; + while (i < b.length) { + if (isStart(i)) { + out.push(b.subarray(start, i)); + i = consumeStart(i); + start = i; + } else i++; + } + if (start < b.length) out.push(b.subarray(start)); + return out; + }; + + const nalType = (nal: Uint8Array) => nal[0] & 0x1f; + const isKeyframe = (nals: Uint8Array[]) => nals.some((n) => nalType(n) === 5); + + const buildAvcC = (spsNal: Uint8Array, ppsNal: Uint8Array): Uint8Array => { + const spsLen = spsNal.length, + ppsLen = ppsNal.length; + const avcc = new Uint8Array(7 + 2 + spsLen + 1 + 2 + ppsLen); + let o = 0; + avcc[o++] = 1; + avcc[o++] = spsNal[1]; + avcc[o++] = spsNal[2]; + avcc[o++] = spsNal[3]; + avcc[o++] = 0xff; + avcc[o++] = 0xe1; + avcc[o++] = (spsLen >>> 8) & 0xff; + avcc[o++] = spsLen & 0xff; + avcc.set(spsNal, o); + o += spsLen; + avcc[o++] = 1; + avcc[o++] = (ppsLen >>> 8) & 0xff; + avcc[o++] = ppsLen & 0xff; + avcc.set(ppsNal, o); + o += ppsLen; + return avcc; + }; + + const codecFromSps = (spsNal: Uint8Array): string => { + const hex = (n: number) => n.toString(16).toUpperCase().padStart(2, "0"); + return `avc1.${hex(spsNal[1])}${hex(spsNal[2])}${hex(spsNal[3])}`; + }; + + const ensureDecoder = () => { + if (decoderRef.current) return; + + decoderRef.current = new VideoDecoder({ + output: async (frame) => { + const bitmap = await createImageBitmap(frame); + setImageBitmap(bitmap); + frame.close(); + }, + error: (e) => console.error("Decoder error:", e), + }); + }; + const tryConfigure = async (): Promise => { + if (configuredRef.current || !spsRef.current || !ppsRef.current) + return false; + ensureDecoder(); - useEffect(() => { - let ws: WebSocket; - let sps: Uint8Array | null = null; - let pps: Uint8Array | null = null; - let nextTs = 0; - - const frameDurationUs = Math.round(1_000_000 / 50); - - const splitAnnexB = (buf: ArrayBuffer): Uint8Array[] => { - const b = new Uint8Array(buf), out: Uint8Array[] = []; - let i = 0; - const isStart = (i: number) => - (i + 3 < b.length && b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 1) || - (i + 4 < b.length && b[i] === 0 && b[i + 1] === 0 && b[i + 2] === 0 && b[i + 3] === 1); - const consumeStart = (i: number) => (b[i + 2] === 1 ? i + 3 : i + 4); - - while (i < b.length - 3 && !isStart(i)) i++; - if (i >= b.length - 3) return out; - i = consumeStart(i); let start = i; - while (i < b.length) { - if (isStart(i)) { - out.push(b.subarray(start, i)); - i = consumeStart(i); - start = i; - } else i++; - } - if (start < b.length) out.push(b.subarray(start)); - return out; - }; - - const nalType = (nal: Uint8Array) => nal[0] & 0x1f; - const isKeyframe = (nals: Uint8Array[]) => nals.some(n => nalType(n) === 5); - - const buildAvcC = (spsNal: Uint8Array, ppsNal: Uint8Array): Uint8Array => { - const spsLen = spsNal.length, ppsLen = ppsNal.length; - const avcc = new Uint8Array(7 + 2 + spsLen + 1 + 2 + ppsLen); - let o = 0; - avcc[o++] = 1; - avcc[o++] = spsNal[1]; - avcc[o++] = spsNal[2]; - avcc[o++] = spsNal[3]; - avcc[o++] = 0xFF; - avcc[o++] = 0xE1; - avcc[o++] = (spsLen >>> 8) & 0xff; avcc[o++] = spsLen & 0xff; avcc.set(spsNal, o); o += spsLen; - avcc[o++] = 1; - avcc[o++] = (ppsLen >>> 8) & 0xff; avcc[o++] = ppsLen & 0xff; avcc.set(ppsNal, o); o += ppsLen; - return avcc; - }; - - const codecFromSps = (spsNal: Uint8Array): string => { - const hex = (n: number) => n.toString(16).toUpperCase().padStart(2, '0'); - return `avc1.${hex(spsNal[1])}${hex(spsNal[2])}${hex(spsNal[3])}`; - }; + const description = buildAvcC(spsRef.current, ppsRef.current); + const codec = codecFromSps(spsRef.current); - const ensureDecoder = () => { - if (decoderRef.current) return; + const { width: codedWidth, height: codedHeight } = dimsRef.current; - decoderRef.current = new VideoDecoder({ - output: async frame => { - const bitmap = await createImageBitmap(frame); - setImageBitmap(bitmap); - frame.close(); - }, - error: e => console.error("Decoder error:", e) - }); + const config: VideoDecoderConfig = { + codec, + codedWidth: codedWidth, + codedHeight: codedHeight, + description: description.buffer, }; - const tryConfigure = async (): Promise => { - if (configuredRef.current || !sps || !pps) return false; - ensureDecoder(); - - const description = buildAvcC(sps, pps); - const codec = codecFromSps(sps); - const config: VideoDecoderConfig = { - codec, - codedWidth: currentWidth, - codedHeight: currentHeight, - description: description.buffer - }; - - const support = await VideoDecoder.isConfigSupported(config).catch(() => null); - if (!support?.supported) { - console.warn("Unsupported config:", config); - return false; - } + const support = await VideoDecoder.isConfigSupported(config).catch( + () => null, + ); + if (!support?.supported) { + console.warn("Unsupported config:", config); + return false; + } - if (decoderRef.current!.state === 'configured') decoderRef.current!.reset(); - decoderRef.current!.configure(config); - configuredRef.current = true; - return true; - }; + if (decoderRef.current!.state === "configured") decoderRef.current!.reset(); + decoderRef.current!.configure(config); + configuredRef.current = true; + return true; + }; + + const feedChunk = (nals: Uint8Array[]) => { + let total = 0; + for (const n of nals) total += 4 + n.length; + const payload = new Uint8Array(total); + let o = 0; + for (const n of nals) { + const L = n.length; + payload[o++] = (L >>> 24) & 0xff; + payload[o++] = (L >>> 16) & 0xff; + payload[o++] = (L >>> 8) & 0xff; + payload[o++] = L & 0xff; + payload.set(n, o); + o += L; + } - const feedChunk = (nals: Uint8Array[]) => { - let total = 0; - for (const n of nals) total += 4 + n.length; - const payload = new Uint8Array(total); - let o = 0; - for (const n of nals) { - const L = n.length; - payload[o++] = (L >>> 24) & 0xff; - payload[o++] = (L >>> 16) & 0xff; - payload[o++] = (L >>> 8) & 0xff; - payload[o++] = L & 0xff; - payload.set(n, o); o += L; - } + const chunk = new EncodedVideoChunk({ + type: isKeyframe(nals) ? "key" : "delta", + timestamp: nextTsRef.current, + data: payload, + }); + nextTsRef.current += frameDurationUs; + decoderRef.current!.decode(chunk); + }; + + const onAccessUnit = (buf: ArrayBuffer) => { + const nals = splitAnnexB(buf); + if (!nals.length) return; + + for (const n of nals) { + const t = nalType(n); + if (t === 7) spsRef.current = n.slice(); + else if (t === 8) ppsRef.current = n.slice(); + } - const chunk = new EncodedVideoChunk({ - type: isKeyframe(nals) ? 'key' : 'delta', - timestamp: nextTs, - data: payload + if (!configuredRef.current && spsRef.current && ppsRef.current) { + console.log("Reconfigure"); + void tryConfigure().then((ok) => { + if (ok) feedChunk(nals); }); - nextTs += frameDurationUs; - decoderRef.current!.decode(chunk); - }; + } else if (configuredRef.current) feedChunk(nals); + }; - const onAccessUnit = (buf: ArrayBuffer) => { - const nals = splitAnnexB(buf); - if (!nals.length) return; - - for (const n of nals) { - const t = nalType(n); - if (t === 7) sps = n.slice(); - else if (t === 8) pps = n.slice(); - } + useEffect(() => { + if (!("VideoDecoder" in window)) { + console.error("WebCodecs VideoDecoder is not supported in this browser."); + return; + } - if (!configuredRef.current && sps && pps) tryConfigure().then(ok => { if (ok) feedChunk(nals); }); - else if (configuredRef.current) feedChunk(nals); - }; + let ws: WebSocket; + spsRef.current = null; + ppsRef.current = null; + nextTsRef.current = 0; const connect = () => { ws = new WebSocket("ws://" + url + "/ws?session_id=" + sessionID); wsRef.current = ws; - ws.binaryType = 'arraybuffer'; + ws.binaryType = "arraybuffer"; ws.onopen = () => { console.log("Connected"); configuredRef.current = false; }; - ws.onmessage = ev => { - if (typeof ev.data === 'string') { + ws.onmessage = (ev) => { + if (typeof ev.data === "string") { try { const meta = JSON.parse(ev.data); - if (meta.type === 'config') { + if (meta.type === "config") { console.log(meta); setCurrentWidth(meta.width); setCurrentHeight(meta.height); setSourceWidth(meta.source_width); setSourceHeight(meta.source_height); + + dimsRef.current = { width: meta.width, height: meta.height }; configuredRef.current = false; // force reconfigure + void tryConfigure(); } } catch (e) { console.warn("Failed to parse metadata:", e); @@ -187,15 +242,19 @@ export const WebsocketH264Provider: React.FC = ({ ch ws.onclose = () => { configuredRef.current = false; - try { decoderRef.current?.flush().catch(() => {}); } catch {} + try { + decoderRef.current?.flush().catch(() => {}); + } catch {/**/} if (!abortedRef.current) { reconnectTimerRef.current = window.setTimeout(connect, 3000); } }; - ws.onerror = e => { - console.error('WebSocket error:', e); - try { ws.close(); } catch { } + ws.onerror = (e) => { + console.error("WebSocket error:", e); + try { + ws.close(); + } catch {/**/} }; }; @@ -207,112 +266,173 @@ export const WebsocketH264Provider: React.FC = ({ ch clearTimeout(reconnectTimerRef.current); reconnectTimerRef.current = null; } - try { wsRef.current?.close(); } catch {} - try { decoderRef.current?.close(); } catch {} - + try { + wsRef.current?.close(); + } catch {/**/} + try { + decoderRef.current?.close(); + } catch {/**/} }; - }, [currentHeight, currentWidth, sessionID, url]); - + + // `tryConfigure` and `onAccessUnit` read only from refs and are intentionally stable. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [sessionID, url]); const reportSize = useCallback(async (width: number, height: number) => { - - const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/resolution", { - method: "POST", - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ width, height }), - }); - }, []); - - const reportZoom = useCallback(async (startX: number, startY: number, width: number, height: number) => { - - const crop_response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { - method: "GET", - }); - - const { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json() ?? { x: 0, y: 0, width: sourceWidth, height: sourceHeight }; - - const xScale = currentCropWidth / currentWidth; - const yScale = currentCropHeight / currentHeight; - - if (width < 0){ - startX = startX + width; - width = -1*width; - } - if (height < 0){ - startY = startY + height; - height = -1*height; - } - - const x = currentCropX + Math.floor(startX * xScale); - const y = currentCropY + Math.floor(startY * yScale); - - const cropWidth = Math.floor(width * xScale); - const cropHeight = Math.floor(height * yScale); - - if (cropWidth == 0 || cropHeight == 0) { - return - } - - console.log(x, y, cropWidth, cropHeight); - - - const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { - method: "POST", - headers: { - 'Content-Type': 'application/json', + const response = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/resolution", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ width, height }), }, - body: JSON.stringify({ x, y, width: cropWidth, height: cropHeight }), - }); - }, [url, sessionID, sourceWidth, sourceHeight, currentWidth, currentHeight]); - - const reportDrag = useCallback(async (totalX: number, totalY: number, deltaX: number, deltaY: number, active: boolean) => { - - if (!active) { - const crop_response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { - method: "GET", - }); - const { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight } = await crop_response.json(); + ); + }, [url, sessionID]); + + const reportZoom = useCallback( + async (startX: number, startY: number, width: number, height: number) => { + const crop_response = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/crop", + { + method: "GET", + }, + ); + + const { + x: currentCropX, + y: currentCropY, + width: currentCropWidth, + height: currentCropHeight, + } = (await crop_response.json()) ?? { + x: 0, + y: 0, + width: sourceWidth, + height: sourceHeight, + }; const xScale = currentCropWidth / currentWidth; const yScale = currentCropHeight / currentHeight; - const scale = Math.max(xScale, yScale); + if (width < 0) { + startX = startX + width; + width = -1 * width; + } + if (height < 0) { + startY = startY + height; + height = -1 * height; + } + + const x = currentCropX + Math.floor(startX * xScale); + const y = currentCropY + Math.floor(startY * yScale); - const x = (currentCropX - Math.floor(totalX * scale)); - const y = (currentCropY - Math.floor(totalY * scale)); + const cropWidth = Math.floor(width * xScale); + const cropHeight = Math.floor(height * yScale); - console.log({currentCropX, currentCropWidth, currentWidth, xScale, totalX, shiftX: Math.floor(totalX * xScale), x}); + if (cropWidth == 0 || cropHeight == 0) { + return; + } + console.log(x, y, cropWidth, cropHeight); - const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { - method: "POST", - headers: { - 'Content-Type': 'application/json', + const response = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/crop", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ x, y, width: cropWidth, height: cropHeight }), }, - body: JSON.stringify({ x, y, width: currentCropWidth, height: currentCropHeight }), - }); - } - }, [url, sessionID, currentWidth, currentHeight]); + ); + }, + [url, sessionID, sourceWidth, sourceHeight, currentWidth, currentHeight], + ); + + const reportDrag = useCallback( + async ( + totalX: number, + totalY: number, + deltaX: number, + deltaY: number, + active: boolean, + ) => { + if (!active) { + const crop_response = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/crop", + { + method: "GET", + }, + ); + const { + x: currentCropX, + y: currentCropY, + width: currentCropWidth, + height: currentCropHeight, + } = await crop_response.json(); + + const xScale = currentCropWidth / currentWidth; + const yScale = currentCropHeight / currentHeight; + + const scale = Math.max(xScale, yScale); + + const x = currentCropX - Math.floor(totalX * scale); + const y = currentCropY - Math.floor(totalY * scale); + + console.log({ + currentCropX, + currentCropWidth, + currentWidth, + xScale, + totalX, + shiftX: Math.floor(totalX * xScale), + x, + }); + + const response = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/crop", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + x, + y, + width: currentCropWidth, + height: currentCropHeight, + }), + }, + ); + } + }, + [url, sessionID, currentWidth, currentHeight], + ); const clearZoom = useCallback(async () => { - const response = await fetch("http://" + url + "/api/sessions/" + sessionID + "/crop", { - method: "DELETE", - }); - }, []); - - const contextValue = useMemo(() => ({ - image: imageBitmap, - reportSize, - reportZoom, - reportDrag, - clearZoom - }), [imageBitmap, reportSize, reportZoom, reportDrag, clearZoom]); + const response = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/crop", + { + method: "DELETE", + }, + ); + }, [url, sessionID]); + + const contextValue = useMemo( + () => ({ + image: imageBitmap, + reportSize, + reportZoom, + reportDrag, + clearZoom, + }), + [imageBitmap, reportSize, reportZoom, reportDrag, clearZoom], + ); return ( {children} - ) -} \ No newline at end of file + ); +}; From 832ed5b31181ca3cd82ce08f0621db8feef61729 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Fri, 20 Feb 2026 12:57:32 +1100 Subject: [PATCH 05/15] sessionId injetion and cleanup with refs --- src/ui/experimental/camera-control.tsx | 2 - .../experimental/websocket-h264-provider.tsx | 220 +++++++++++++----- 2 files changed, 162 insertions(+), 60 deletions(-) diff --git a/src/ui/experimental/camera-control.tsx b/src/ui/experimental/camera-control.tsx index 25cc79c..988ee31 100644 --- a/src/ui/experimental/camera-control.tsx +++ b/src/ui/experimental/camera-control.tsx @@ -153,7 +153,6 @@ export const CameraControl: React.FC = ({ let lastMove = 0; const handleMouseMove = (e: React.MouseEvent) => { if (dragStart) { - console.log(e.clientX); const deltaX = e.clientX - dragStart.lastX; const deltaY = e.clientY - dragStart.lastY; @@ -250,7 +249,6 @@ export const CameraControl: React.FC = ({ const handleMouseUp = (e: React.MouseEvent) => { if (dragStart) { - console.log(dragStart); reportDrag( e.clientX - dragStart.startX, e.clientY - dragStart.startY, diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index d51f7f9..21f490a 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -20,9 +20,6 @@ export const WebsocketH264Provider: React.FC = ({ // ================== // State // ================== - const [sessionID, setSessionID] = useState( - "019c6ef1-821e-7b7b-a28b-085f73d0a248", - ); const [imageBitmap, setImageBitmap] = useState(null); const [sourceWidth, setSourceWidth] = useState(1024); const [sourceHeight, setSourceHeight] = useState(1024); @@ -38,17 +35,27 @@ export const WebsocketH264Provider: React.FC = ({ height: 1024, }); const abortedRef = useRef(false); + const configuringRef = useRef(false); const configuredRef = useRef(false); const decoderRef = useRef(null); const reconnectTimerRef = useRef(null); const spsRef = useRef(null); const ppsRef = useRef(null); const nextTsRef = useRef(0); + const sidRef = useRef(sessionID ?? null); + + // Internal resolved session ID. We mirror the prop; if null, we create and then set it here. + const [resolvedSessionId, setResolvedSessionId] = useState( + sessionID, + ); + + useEffect(() => { + setResolvedSessionId(sessionID ?? null); + }, [sessionID]); // ================== // Helper Functions // ================== - const frameDurationUs = Math.round(1_000_000 / 50); const splitAnnexB = (buf: ArrayBuffer): Uint8Array[] => { @@ -111,7 +118,7 @@ export const WebsocketH264Provider: React.FC = ({ }; const ensureDecoder = () => { - if (decoderRef.current) return; + if (decoderRef.current && decoderRef.current.state !== "closed") return; decoderRef.current = new VideoDecoder({ output: async (frame) => { @@ -124,34 +131,49 @@ export const WebsocketH264Provider: React.FC = ({ }; const tryConfigure = async (): Promise => { + if (abortedRef.current) return false; + if (configuringRef.current) return false; if (configuredRef.current || !spsRef.current || !ppsRef.current) return false; - ensureDecoder(); - const description = buildAvcC(spsRef.current, ppsRef.current); - const codec = codecFromSps(spsRef.current); + configuringRef.current = true; + try { + ensureDecoder(); - const { width: codedWidth, height: codedHeight } = dimsRef.current; + const description = buildAvcC(spsRef.current, ppsRef.current); + const codec = codecFromSps(spsRef.current); + const { width: codedWidth, height: codedHeight } = dimsRef.current; - const config: VideoDecoderConfig = { - codec, - codedWidth: codedWidth, - codedHeight: codedHeight, - description: description.buffer, - }; + const config: VideoDecoderConfig = { + codec, + codedWidth: codedWidth, + codedHeight: codedHeight, + description: description.buffer, + }; - const support = await VideoDecoder.isConfigSupported(config).catch( - () => null, - ); - if (!support?.supported) { - console.warn("Unsupported config:", config); + const support = await VideoDecoder.isConfigSupported(config).catch( + () => null, + ); + if (!support?.supported) { + console.warn("Unsupported config:", config); + return false; + } + + if (abortedRef.current) return false; + + const dec = decoderRef.current; + if (!dec || dec.state === "closed") return false; + + if (dec.state === "configured") dec.reset(); + dec.configure(config); + configuredRef.current = true; + return true; + } catch (e) { + console.warn("tryConfigure failed:", e); return false; + } finally { + configuringRef.current = false; } - - if (decoderRef.current!.state === "configured") decoderRef.current!.reset(); - decoderRef.current!.configure(config); - configuredRef.current = true; - return true; }; const feedChunk = (nals: Uint8Array[]) => { @@ -175,7 +197,14 @@ export const WebsocketH264Provider: React.FC = ({ data: payload, }); nextTsRef.current += frameDurationUs; - decoderRef.current!.decode(chunk); + + const dec = decoderRef.current; + if (!dec || dec.state === "closed") return; + try { + dec.decode(chunk); + } catch { + /**/ + } }; const onAccessUnit = (buf: ArrayBuffer) => { @@ -202,13 +231,16 @@ export const WebsocketH264Provider: React.FC = ({ return; } + const aborter = new AbortController(); + abortedRef.current = false; + let ws: WebSocket; spsRef.current = null; ppsRef.current = null; nextTsRef.current = 0; - const connect = () => { - ws = new WebSocket("ws://" + url + "/ws?session_id=" + sessionID); + const connect = (sid: string) => { + ws = new WebSocket("ws://" + url + "/ws?session_id=" + sid); wsRef.current = ws; ws.binaryType = "arraybuffer"; @@ -244,9 +276,14 @@ export const WebsocketH264Provider: React.FC = ({ configuredRef.current = false; try { decoderRef.current?.flush().catch(() => {}); - } catch {/**/} - if (!abortedRef.current) { - reconnectTimerRef.current = window.setTimeout(connect, 3000); + } catch { + /**/ + } + if (!abortedRef.current && sidRef.current) { + reconnectTimerRef.current = window.setTimeout( + () => connect(sidRef.current!), + 3000, + ); } }; @@ -254,47 +291,112 @@ export const WebsocketH264Provider: React.FC = ({ console.error("WebSocket error:", e); try { ws.close(); - } catch {/**/} + } catch { + /**/ + } }; }; - connect(); + const ensureSessionId = async (): Promise => { + if (sidRef.current) return sidRef.current; + if (resolvedSessionId) { + sidRef.current = resolvedSessionId; + return resolvedSessionId; + } + + const res = await fetch("http://" + url + "/api/sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + signal: aborter.signal, + body: JSON.stringify({ + colour_mapping: "none", + crop: null, + resolution: { + width: 1024, + height: 1024, + }, + }), + }); + if (!res.ok) + throw new Error( + `Failed to create session: ${res.status} ${res.statusText}`, + ); + + const body = await res.json(); + const sid = body.id as string; + if (!sid) throw new Error("Server did not return session_id"); + + // Inform parent so it can persist/store as it sees fit + onSessionCreated?.(sid); + + // Keep our internal view so we can connect immediately + setResolvedSessionId(sid); + sidRef.current = sid; + return sid; + }; + + (async () => { + try { + const sid = await ensureSessionId(); + if (!abortedRef.current) connect(sid); + } catch (e) { + if ( + !abortedRef.current && + !(e instanceof DOMException && e.name === "AbortError") + ) { + console.error(e); + } + } + })(); return () => { abortedRef.current = true; + aborter.abort(); + if (reconnectTimerRef.current !== null) { clearTimeout(reconnectTimerRef.current); reconnectTimerRef.current = null; } try { wsRef.current?.close(); - } catch {/**/} + } catch { + /**/ + } try { decoderRef.current?.close(); - } catch {/**/} + } catch { + /**/ + } }; - + // `tryConfigure` and `onAccessUnit` read only from refs and are intentionally stable. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [sessionID, url]); + }, [url]); - const reportSize = useCallback(async (width: number, height: number) => { - const response = await fetch( - "http://" + url + "/api/sessions/" + sessionID + "/resolution", - { - method: "POST", - headers: { - "Content-Type": "application/json", + const reportSize = useCallback( + async (width: number, height: number) => { + const sid = sidRef.current; + if (!sid) return; + const response = await fetch( + "http://" + url + "/api/sessions/" + sid + "/resolution", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ width, height }), }, - body: JSON.stringify({ width, height }), - }, - ); - }, [url, sessionID]); + ); + }, + [url], + ); const reportZoom = useCallback( async (startX: number, startY: number, width: number, height: number) => { + const sid = sidRef.current; + if (!sid) return; const crop_response = await fetch( - "http://" + url + "/api/sessions/" + sessionID + "/crop", + "http://" + url + "/api/sessions/" + sid + "/crop", { method: "GET", }, @@ -334,10 +436,8 @@ export const WebsocketH264Provider: React.FC = ({ return; } - console.log(x, y, cropWidth, cropHeight); - const response = await fetch( - "http://" + url + "/api/sessions/" + sessionID + "/crop", + "http://" + url + "/api/sessions/" + sid + "/crop", { method: "POST", headers: { @@ -347,7 +447,7 @@ export const WebsocketH264Provider: React.FC = ({ }, ); }, - [url, sessionID, sourceWidth, sourceHeight, currentWidth, currentHeight], + [url, sourceWidth, sourceHeight, currentWidth, currentHeight], ); const reportDrag = useCallback( @@ -359,8 +459,10 @@ export const WebsocketH264Provider: React.FC = ({ active: boolean, ) => { if (!active) { + const sid = sidRef.current; + if (!sid) return; const crop_response = await fetch( - "http://" + url + "/api/sessions/" + sessionID + "/crop", + "http://" + url + "/api/sessions/" + sid + "/crop", { method: "GET", }, @@ -391,7 +493,7 @@ export const WebsocketH264Provider: React.FC = ({ }); const response = await fetch( - "http://" + url + "/api/sessions/" + sessionID + "/crop", + "http://" + url + "/api/sessions/" + sid + "/crop", { method: "POST", headers: { @@ -407,17 +509,19 @@ export const WebsocketH264Provider: React.FC = ({ ); } }, - [url, sessionID, currentWidth, currentHeight], + [url, currentWidth, currentHeight], ); const clearZoom = useCallback(async () => { + const sid = sidRef.current; + if (!sid) return; const response = await fetch( - "http://" + url + "/api/sessions/" + sessionID + "/crop", + "http://" + url + "/api/sessions/" + sid + "/crop", { method: "DELETE", }, ); - }, [url, sessionID]); + }, [url]); const contextValue = useMemo( () => ({ From 7c13ece4912bb14ed5edbb29c952af1cb9896b61 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Tue, 3 Mar 2026 09:16:37 +1100 Subject: [PATCH 06/15] Working version before changing to using padding passed via API. --- .../experimental/camera-control-h264.tsx | 6 +- src/ui/experimental/h264-api.tsx | 38 ++++ src/ui/experimental/h264-fetch.tsx | 115 ++++++++++ .../experimental/websocket-h264-provider.tsx | 213 ++++++++++-------- 4 files changed, 271 insertions(+), 101 deletions(-) create mode 100644 src/ui/experimental/h264-api.tsx create mode 100644 src/ui/experimental/h264-fetch.tsx diff --git a/src/docs/routes/experimental/camera-control-h264.tsx b/src/docs/routes/experimental/camera-control-h264.tsx index 34c8789..92c6365 100644 --- a/src/docs/routes/experimental/camera-control-h264.tsx +++ b/src/docs/routes/experimental/camera-control-h264.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useState, useMemo } from 'react'; import { createFileRoute } from '@tanstack/react-router' import { PageHeader } from "../../components/page-header" import { CameraControl } from '../../../ui/experimental/camera-control' @@ -7,6 +7,7 @@ import { TypographyH1 } from '../../../ui/elements/typography' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../../../ui/layout/card" import { DemoContainer } from "@/docs/components/demo-container" +import { h264FetchApi } from '@/ui/experimental/h264-fetch'; export const Route = createFileRoute('/experimental/camera-control-h264')({ component: CameraControlWebsocketH264Page, @@ -17,9 +18,10 @@ export const Route = createFileRoute('/experimental/camera-control-h264')({ function CameraControlWebsocketH264Demo() { const [mousePos, setMousePos] = useState<{ x: number, y: number, intensity: number } | null>(null); const [clickPos, setClickPos] = useState<{ x: number, y: number, intensity: number } | null>(null); + const api = useMemo(() => h264FetchApi("localhost:9999/test"), []); return (
- + Promise; + + /** Get source resolution - this only works if there is an active encoder */ + getSourceResolution: () => Promise; + + /** Resolution of the view (stream == display by design). */ + setResolution: ( + sessionId: string, + width: number, + height: number, + signal?: AbortSignal, + ) => Promise; + + /** Get current crop box. */ + getCrop: (sessionId: string, signal?: AbortSignal) => Promise; + + /** Set crop box. */ + setCrop: ( + sessionId: string, + crop: Crop, + signal?: AbortSignal, + ) => Promise; + + /** Clear crop. */ + clearCrop: (sessionId: string, signal?: AbortSignal) => Promise; + + /** Optional: customize WebSocket construction (auth headers, subprotocols, polyfills). */ + wsFactory: (sessionId: string) => WebSocket; + + /** Optional: build absolute HTTP/WS URLs if you don’t want the component to concatenate strings. */ + buildHttpUrl?: (path: string) => string; // e.g., p => `${base}${p}` + buildWsUrl?: (path: string) => string; // e.g., p => `${wssBase}${p}` +} diff --git a/src/ui/experimental/h264-fetch.tsx b/src/ui/experimental/h264-fetch.tsx new file mode 100644 index 0000000..ba389e3 --- /dev/null +++ b/src/ui/experimental/h264-fetch.tsx @@ -0,0 +1,115 @@ +import { type Crop, type H264Api } from "./h264-api"; + +export function h264FetchApi(url: string): H264Api { + return { + async createSession(signal) { + const res = await fetch("http://" + url + "/api/sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + signal: signal, + body: JSON.stringify({ + colour_mapping: "none", + crop: null, + resolution: { + width: 1024, + height: 1024, + }, + }), + }); + if (!res.ok) + throw new Error( + `Failed to create session: ${res.status} ${res.statusText}`, + ); + + const body = await res.json(); + const sid = body.id as string; + if (!sid) throw new Error("Server did not return session_id"); + return String(sid); + }, + async getSourceResolution(signal?: AbortSignal) { + const res = await fetch("http://" + url + "/api/resolution", { + method: "GET", + signal: signal, + }); + if (!res.ok) + throw new Error( + `Failed to get resolution. Is there an encoder running?`, + ); + const data = await res.json(); + return { + width: data.source_width, + height: data.source_height + } + }, + async setResolution( + sessionId: string, + width: number, + height: number, + signal?: AbortSignal, + ) { + const res = await fetch( + "http://" + url + "/api/sessions/" + sessionId + "/resolution", + { + method: "POST", + signal: signal, + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ width, height }), + }, + ); + if (!res.ok) + throw new Error( + `Failed to set resolution: ${res.status} ${res.statusText}`, + ); + return; + }, + async getCrop(sessionId: string, signal?: AbortSignal) { + const crop_response = await fetch( + "http://" + url + "/api/sessions/" + sessionId + "/crop", + { + method: "GET", + signal: signal, + }, + ); + if (!crop_response.ok) + throw new Error( + `Failed to get current crop: ${crop_response.status} ${crop_response.statusText}`, + ); + const crop: Crop = await crop_response.json(); + return crop; + }, + async setCrop(sessionID: string, crop: Crop, signal?: AbortSignal) { + const res = await fetch( + "http://" + url + "/api/sessions/" + sessionID + "/crop", + { + method: "POST", + signal: signal, + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(crop), + }, + ); + if (!res.ok) + throw new Error(`Failed to set crop: ${res.status} ${res.statusText}`); + return; + }, + async clearCrop(sessionId: string, signal?: AbortSignal) { + const res = await fetch( + "http://" + url + "/api/sessions/" + sessionId + "/crop", + { + method: "DELETE", + signal: signal, + }, + ); + if (!res.ok) + throw new Error( + `Failed to clear crop: ${res.status} ${res.statusText}`, + ); + }, + wsFactory(sessionId) { + return new WebSocket("ws://" + url + "/ws?session_id=" + sessionId); + }, + }; +} diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index 21f490a..37807f8 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -8,14 +8,20 @@ import React, { import { ImageContext } from "./image-context"; +import { type H264Api } from "./h264-api"; + export interface WebsocketH264ProviderProps { children: React.ReactNode; - url: string; + sessionId?: string; + onSessionCreated?: (sessionId: string) => void; + api: H264Api; } export const WebsocketH264Provider: React.FC = ({ children, - url, + sessionId = null, + onSessionCreated = null, + api, }) => { // ================== // State @@ -25,6 +31,8 @@ export const WebsocketH264Provider: React.FC = ({ const [sourceHeight, setSourceHeight] = useState(1024); const [currentWidth, setCurrentWidth] = useState(1024); const [currentHeight, setCurrentHeight] = useState(1024); + const [paddingWidth, setPaddingWidth] = useState(0); + const [paddingHeight, setPaddingHeight] = useState(0); // ================== // Refs @@ -42,16 +50,17 @@ export const WebsocketH264Provider: React.FC = ({ const spsRef = useRef(null); const ppsRef = useRef(null); const nextTsRef = useRef(0); - const sidRef = useRef(sessionID ?? null); + const sidRef = useRef(sessionId ?? null); + const lastConfigRef = useRef<{ width: number; height: number } | null>(null); // Internal resolved session ID. We mirror the prop; if null, we create and then set it here. const [resolvedSessionId, setResolvedSessionId] = useState( - sessionID, + sessionId, ); useEffect(() => { - setResolvedSessionId(sessionID ?? null); - }, [sessionID]); + setResolvedSessionId(sessionId ?? null); + }, [sessionId]); // ================== // Helper Functions @@ -148,7 +157,7 @@ export const WebsocketH264Provider: React.FC = ({ codec, codedWidth: codedWidth, codedHeight: codedHeight, - description: description.buffer, + description: description, }; const support = await VideoDecoder.isConfigSupported(config).catch( @@ -240,7 +249,8 @@ export const WebsocketH264Provider: React.FC = ({ nextTsRef.current = 0; const connect = (sid: string) => { - ws = new WebSocket("ws://" + url + "/ws?session_id=" + sid); + ws = api.wsFactory(sid); + wsRef.current = ws; ws.binaryType = "arraybuffer"; @@ -259,10 +269,26 @@ export const WebsocketH264Provider: React.FC = ({ setCurrentHeight(meta.height); setSourceWidth(meta.source_width); setSourceHeight(meta.source_height); - - dimsRef.current = { width: meta.width, height: meta.height }; - configuredRef.current = false; // force reconfigure - void tryConfigure(); + setPaddingWidth(meta.padding_width); + setPaddingHeight(meta.padding_width); + + const last = lastConfigRef.current; + const changed = + !last || + last.width !== meta.width || + last.height !== meta.height; + + if (changed) { + lastConfigRef.current = { + width: meta.width, + height: meta.height, + }; + dimsRef.current = { width: meta.width, height: meta.height }; + configuredRef.current = false; // force reconfigure + void tryConfigure(); + } else { + /**/ + } } } catch (e) { console.warn("Failed to parse metadata:", e); @@ -304,26 +330,7 @@ export const WebsocketH264Provider: React.FC = ({ return resolvedSessionId; } - const res = await fetch("http://" + url + "/api/sessions", { - method: "POST", - headers: { "Content-Type": "application/json" }, - signal: aborter.signal, - body: JSON.stringify({ - colour_mapping: "none", - crop: null, - resolution: { - width: 1024, - height: 1024, - }, - }), - }); - if (!res.ok) - throw new Error( - `Failed to create session: ${res.status} ${res.statusText}`, - ); - - const body = await res.json(); - const sid = body.id as string; + const sid = await api.createSession(aborter.signal); if (!sid) throw new Error("Server did not return session_id"); // Inform parent so it can persist/store as it sees fit @@ -371,51 +378,58 @@ export const WebsocketH264Provider: React.FC = ({ // `tryConfigure` and `onAccessUnit` read only from refs and are intentionally stable. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [url]); + }, [api]); const reportSize = useCallback( async (width: number, height: number) => { const sid = sidRef.current; if (!sid) return; - const response = await fetch( - "http://" + url + "/api/sessions/" + sid + "/resolution", - { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ width, height }), - }, - ); + api.setResolution(sid, width, height); }, - [url], + [api], ); const reportZoom = useCallback( async (startX: number, startY: number, width: number, height: number) => { const sid = sidRef.current; if (!sid) return; - const crop_response = await fetch( - "http://" + url + "/api/sessions/" + sid + "/crop", - { - method: "GET", - }, - ); + const crop_response = await api.getCrop(sid); const { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight, - } = (await crop_response.json()) ?? { + } = crop_response ?? { x: 0, y: 0, width: sourceWidth, height: sourceHeight, }; - const xScale = currentCropWidth / currentWidth; - const yScale = currentCropHeight / currentHeight; + const native_aspect = sourceWidth / sourceHeight; + const aspect = currentWidth / currentHeight; + + console.log(aspect, currentCropWidth/currentCropHeight); + + let x_pad = currentCropX; + let y_pad = currentCropY; + let scale = 0, + xScale = 0, + yScale = 0; + + if (y_pad == 0 && x_pad == 0) { + if (native_aspect > aspect) { + scale = currentWidth / sourceWidth; + y_pad = -Math.floor(currentHeight / (2 * scale) - sourceHeight / 2); + } else if (native_aspect < aspect) { + scale = currentHeight / sourceHeight; + x_pad = -Math.floor(currentWidth / (2 * scale) - sourceWidth / 2); + } + } else { + xScale = currentCropWidth / currentWidth; + yScale = currentCropHeight / currentHeight; + } if (width < 0) { startX = startX + width; @@ -426,28 +440,41 @@ export const WebsocketH264Provider: React.FC = ({ height = -1 * height; } - const x = currentCropX + Math.floor(startX * xScale); - const y = currentCropY + Math.floor(startY * yScale); + let x = 0, + y = 0; + + let cropWidth = 0, + cropHeight = 0; + + if (scale == 0) { + x = currentCropX + Math.floor(startX * xScale); + y = currentCropY + Math.floor(startY * yScale); + cropWidth = Math.floor(width * xScale); + cropHeight = Math.floor(height * yScale); + } else { + x = x_pad + Math.floor(startX / scale); + y = y_pad + Math.floor(startY / scale); + x = Math.max(x, 0); + y = Math.max(y, 0); + + cropWidth = Math.floor(width / scale); + cropHeight = Math.floor(height / scale); + } - const cropWidth = Math.floor(width * xScale); - const cropHeight = Math.floor(height * yScale); + console.log(x, y); + console.log(cropWidth, cropHeight); if (cropWidth == 0 || cropHeight == 0) { return; } - - const response = await fetch( - "http://" + url + "/api/sessions/" + sid + "/crop", - { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ x, y, width: cropWidth, height: cropHeight }), - }, - ); + await api.setCrop(sid, { + x, + y, + width: cropWidth, + height: cropHeight, + }); }, - [url, sourceWidth, sourceHeight, currentWidth, currentHeight], + [api, sourceWidth, sourceHeight, currentWidth, currentHeight], ); const reportDrag = useCallback( @@ -461,18 +488,20 @@ export const WebsocketH264Provider: React.FC = ({ if (!active) { const sid = sidRef.current; if (!sid) return; - const crop_response = await fetch( - "http://" + url + "/api/sessions/" + sid + "/crop", - { - method: "GET", - }, - ); + + const crop_response = await api.getCrop(sid); + const { x: currentCropX, y: currentCropY, width: currentCropWidth, height: currentCropHeight, - } = await crop_response.json(); + } = crop_response ?? { + x: 0, + y: 0, + width: sourceWidth, + height: sourceHeight, + }; const xScale = currentCropWidth / currentWidth; const yScale = currentCropHeight / currentHeight; @@ -492,36 +521,22 @@ export const WebsocketH264Provider: React.FC = ({ x, }); - const response = await fetch( - "http://" + url + "/api/sessions/" + sid + "/crop", - { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - x, - y, - width: currentCropWidth, - height: currentCropHeight, - }), - }, - ); + await api.setCrop(sid, { + x, + y, + width: currentCropWidth, + height: currentCropHeight, + }); } }, - [url, currentWidth, currentHeight], + [api, sourceWidth, sourceHeight, currentWidth, currentHeight], ); const clearZoom = useCallback(async () => { const sid = sidRef.current; if (!sid) return; - const response = await fetch( - "http://" + url + "/api/sessions/" + sid + "/crop", - { - method: "DELETE", - }, - ); - }, [url]); + api.clearCrop(sid); + }, [api]); const contextValue = useMemo( () => ({ From baf0a77b99a10bbd05493fc2b57367dc91392c3a Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Tue, 3 Mar 2026 16:27:15 +1100 Subject: [PATCH 07/15] Using fixed crop resonse --- src/ui/experimental/h264-api.tsx | 10 +++ src/ui/experimental/h264-fetch.tsx | 21 +++++- .../experimental/websocket-h264-provider.tsx | 70 +++++++++---------- 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/src/ui/experimental/h264-api.tsx b/src/ui/experimental/h264-api.tsx index 4733f69..e7adda8 100644 --- a/src/ui/experimental/h264-api.tsx +++ b/src/ui/experimental/h264-api.tsx @@ -1,5 +1,9 @@ export type Crop = { x: number; y: number; width: number; height: number }; export type Resolution = { width: number; height: number }; +export type SessionResolution = Resolution & { + paddingWidth: number; + paddingHeight: number; +}; export interface H264Api { /** Create a session if needed. Return the session ID. */ @@ -8,6 +12,12 @@ export interface H264Api { /** Get source resolution - this only works if there is an active encoder */ getSourceResolution: () => Promise; + /** Get session resolution */ + getSessionResolution: ( + sessionId: string, + signal?: AbortSignal, + ) => Promise; + /** Resolution of the view (stream == display by design). */ setResolution: ( sessionId: string, diff --git a/src/ui/experimental/h264-fetch.tsx b/src/ui/experimental/h264-fetch.tsx index ba389e3..d705910 100644 --- a/src/ui/experimental/h264-fetch.tsx +++ b/src/ui/experimental/h264-fetch.tsx @@ -38,8 +38,25 @@ export function h264FetchApi(url: string): H264Api { const data = await res.json(); return { width: data.source_width, - height: data.source_height - } + height: data.source_height, + }; + }, + async getSessionResolution(sessionId: string, signal?: AbortSignal) { + const res = await fetch("http://" + url + "/api/sessions/" + sessionId + "/resolution", { + method: "GET", + signal: signal, + }); + if (!res.ok) + throw new Error( + `Failed to get resolution.`, + ); + const data = await res.json(); + return { + width: data.source_width, + height: data.source_height, + paddingWidth: data.padding_width, + paddingHeight: data.padding_height, + }; }, async setResolution( sessionId: string, diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index 37807f8..4c2cfc1 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -31,6 +31,10 @@ export const WebsocketH264Provider: React.FC = ({ const [sourceHeight, setSourceHeight] = useState(1024); const [currentWidth, setCurrentWidth] = useState(1024); const [currentHeight, setCurrentHeight] = useState(1024); + const [currentCropHeight, setCurrentCropHeight] = useState(1024); + const [currentCropWidth, setCurrentCropWidth] = useState(1024); + const [currentCropStartX, setCurrentCropStartX] = useState(0); + const [currentCropStartY, setCurrentCropStartY] = useState(0); const [paddingWidth, setPaddingWidth] = useState(0); const [paddingHeight, setPaddingHeight] = useState(0); @@ -270,7 +274,11 @@ export const WebsocketH264Provider: React.FC = ({ setSourceWidth(meta.source_width); setSourceHeight(meta.source_height); setPaddingWidth(meta.padding_width); - setPaddingHeight(meta.padding_width); + setPaddingHeight(meta.padding_height); + setCurrentCropWidth(meta.crop_width); + setCurrentCropHeight(meta.crop_height); + setCurrentCropStartX(meta.crop_x); + setCurrentCropStartY(meta.crop_y); const last = lastConfigRef.current; const changed = @@ -393,6 +401,7 @@ export const WebsocketH264Provider: React.FC = ({ async (startX: number, startY: number, width: number, height: number) => { const sid = sidRef.current; if (!sid) return; + // TODO: Should I get this from meta in websocket instead of rest. const crop_response = await api.getCrop(sid); const { @@ -410,25 +419,18 @@ export const WebsocketH264Provider: React.FC = ({ const native_aspect = sourceWidth / sourceHeight; const aspect = currentWidth / currentHeight; - console.log(aspect, currentCropWidth/currentCropHeight); - let x_pad = currentCropX; let y_pad = currentCropY; - let scale = 0, - xScale = 0, - yScale = 0; - - if (y_pad == 0 && x_pad == 0) { - if (native_aspect > aspect) { - scale = currentWidth / sourceWidth; - y_pad = -Math.floor(currentHeight / (2 * scale) - sourceHeight / 2); - } else if (native_aspect < aspect) { - scale = currentHeight / sourceHeight; - x_pad = -Math.floor(currentWidth / (2 * scale) - sourceWidth / 2); - } - } else { - xScale = currentCropWidth / currentWidth; - yScale = currentCropHeight / currentHeight; + let scale = 0; + + if (native_aspect >= aspect) { + scale = currentWidth / currentCropWidth; + x_pad = currentCropX; + y_pad = currentCropY - Math.floor(paddingHeight / (2 * scale)); + } else if (native_aspect < aspect) { + scale = currentHeight / currentCropHeight; + x_pad = currentCropX - Math.floor(paddingWidth / (2 * scale)); + y_pad = currentCropY; } if (width < 0) { @@ -446,23 +448,13 @@ export const WebsocketH264Provider: React.FC = ({ let cropWidth = 0, cropHeight = 0; - if (scale == 0) { - x = currentCropX + Math.floor(startX * xScale); - y = currentCropY + Math.floor(startY * yScale); - cropWidth = Math.floor(width * xScale); - cropHeight = Math.floor(height * yScale); - } else { - x = x_pad + Math.floor(startX / scale); - y = y_pad + Math.floor(startY / scale); - x = Math.max(x, 0); - y = Math.max(y, 0); - - cropWidth = Math.floor(width / scale); - cropHeight = Math.floor(height / scale); - } + x = x_pad + Math.floor(startX / scale); + y = y_pad + Math.floor(startY / scale); + x = Math.max(x, 0); + y = Math.max(y, 0); - console.log(x, y); - console.log(cropWidth, cropHeight); + cropWidth = Math.floor(width / scale); + cropHeight = Math.floor(height / scale); if (cropWidth == 0 || cropHeight == 0) { return; @@ -474,7 +466,15 @@ export const WebsocketH264Provider: React.FC = ({ height: cropHeight, }); }, - [api, sourceWidth, sourceHeight, currentWidth, currentHeight], + [ + api, + sourceWidth, + sourceHeight, + currentWidth, + currentHeight, + paddingWidth, + paddingHeight, + ], ); const reportDrag = useCallback( From 02ed30bf33d7ac27e1d33f730040f5c522663e01 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Thu, 5 Mar 2026 18:03:54 +1100 Subject: [PATCH 08/15] Space button for panning. --- .../experimental/camera-control-h264.tsx | 2 +- src/ui/experimental/camera-control.tsx | 121 +++++++++++++----- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/src/docs/routes/experimental/camera-control-h264.tsx b/src/docs/routes/experimental/camera-control-h264.tsx index 92c6365..0681e0e 100644 --- a/src/docs/routes/experimental/camera-control-h264.tsx +++ b/src/docs/routes/experimental/camera-control-h264.tsx @@ -18,7 +18,7 @@ export const Route = createFileRoute('/experimental/camera-control-h264')({ function CameraControlWebsocketH264Demo() { const [mousePos, setMousePos] = useState<{ x: number, y: number, intensity: number } | null>(null); const [clickPos, setClickPos] = useState<{ x: number, y: number, intensity: number } | null>(null); - const api = useMemo(() => h264FetchApi("localhost:9999/test"), []); + const api = useMemo(() => h264FetchApi("localhost:9999"), []); return (
diff --git a/src/ui/experimental/camera-control.tsx b/src/ui/experimental/camera-control.tsx index 988ee31..826dffe 100644 --- a/src/ui/experimental/camera-control.tsx +++ b/src/ui/experimental/camera-control.tsx @@ -1,4 +1,10 @@ -import React, { useEffect, useRef, useState, useContext } from "react"; +import React, { + useEffect, + useRef, + useState, + useContext, + useCallback, +} from "react"; import { cn } from "../../lib/utils"; import { ImageContext } from "./image-context"; @@ -43,7 +49,7 @@ export const CameraControl: React.FC = ({ onClick, showIntensity = false, onZoom, - sizeFollowsImage = false, + sizeFollowsImage = false }) => { const { image, reportSize, reportZoom, reportDrag, clearZoom } = useContext(ImageContext); @@ -68,6 +74,8 @@ export const CameraControl: React.FC = ({ lastX: number; lastY: number; } | null>(null); + const [spaceHeld, setSpaceHeld] = useState(false); + const [cursorDisplay, setCursorDisplay] = useState("crosshair"); const [frameCount, setFrameCount] = useState(0); const [startTime, setStartTime] = useState(performance.now()); @@ -153,7 +161,6 @@ export const CameraControl: React.FC = ({ let lastMove = 0; const handleMouseMove = (e: React.MouseEvent) => { if (dragStart) { - const deltaX = e.clientX - dragStart.lastX; const deltaY = e.clientY - dragStart.lastY; setDragStart({ ...dragStart, lastX: e.clientX, lastY: e.clientY }); @@ -233,40 +240,84 @@ export const CameraControl: React.FC = ({ } }; - const handleMouseDown = (e: React.MouseEvent) => { - if (e.shiftKey) { - setDragStart({ - startX: e.clientX, - startY: e.clientY, - lastX: e.clientX, - lastY: e.clientY, - }); - } else { - const info = getMousePixelInfo(e); - setZoomBox({ startX: info.x, startY: info.y, width: 0, height: 0 }); - } + const handleMouseDown = useCallback( + (e: React.MouseEvent) => { + if (spaceHeld) { + setCursorDisplay("grabbing"); + setDragStart({ + startX: e.clientX, + startY: e.clientY, + lastX: e.clientX, + lastY: e.clientY, + }); + } else { + const info = getMousePixelInfo(e); + setZoomBox({ startX: info.x, startY: info.y, width: 0, height: 0 }); + } + }, + [spaceHeld], + ); + + const handleMouseUp = useCallback( + (e: React.MouseEvent) => { + if (dragStart) { + if (spaceHeld) { + setCursorDisplay("grab"); + } else { + setCursorDisplay("crosshair"); + } + reportDrag( + e.clientX - dragStart.startX, + e.clientY - dragStart.startY, + 0, + 0, + false, + ); + setDragStart(null); + } + if (zoomBox) { + onZoom?.(zoomBox); + reportZoom( + zoomBox.startX, + zoomBox.startY, + zoomBox.width, + zoomBox.height, + ); + } + setZoomBox(null); + }, + [dragStart, spaceHeld, zoomBox, onZoom, reportDrag, reportZoom], + ); + + const handleDoubleClick = (e: React.MouseEvent) => { + clearZoom(); }; - const handleMouseUp = (e: React.MouseEvent) => { - if (dragStart) { - reportDrag( - e.clientX - dragStart.startX, - e.clientY - dragStart.startY, - 0, - 0, - false, - ); - setDragStart(null); - } - if (zoomBox) { - onZoom?.(zoomBox); - reportZoom(zoomBox.startX, zoomBox.startY, zoomBox.width, zoomBox.height); + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.code === "Space") { + if (!spaceHeld) { + setCursorDisplay("grab"); + setSpaceHeld(true); + } + e.preventDefault(); + } + }, + [spaceHeld], + ); + + const handleKeyUp = (e: React.KeyboardEvent) => { + if (e.code === "Space") { + setSpaceHeld(false); + if (!dragStart) { + setCursorDisplay("crosshair"); + }; + e.preventDefault(); } - setZoomBox(null); }; - const handleDoubleClick = (e: React.MouseEvent) => { - clearZoom(); + const handleMouseEnter = (e: React.MouseEvent) => { + e.currentTarget.focus(); }; return ( @@ -282,13 +333,17 @@ export const CameraControl: React.FC = ({ Date: Fri, 6 Mar 2026 10:02:12 +1100 Subject: [PATCH 09/15] Use metadata from stream instead of rest to get current crop for dragging. --- .../experimental/websocket-h264-provider.tsx | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/websocket-h264-provider.tsx index 4c2cfc1..e39e2ce 100644 --- a/src/ui/experimental/websocket-h264-provider.tsx +++ b/src/ui/experimental/websocket-h264-provider.tsx @@ -399,38 +399,23 @@ export const WebsocketH264Provider: React.FC = ({ const reportZoom = useCallback( async (startX: number, startY: number, width: number, height: number) => { - const sid = sidRef.current; - if (!sid) return; - // TODO: Should I get this from meta in websocket instead of rest. - const crop_response = await api.getCrop(sid); - - const { - x: currentCropX, - y: currentCropY, - width: currentCropWidth, - height: currentCropHeight, - } = crop_response ?? { - x: 0, - y: 0, - width: sourceWidth, - height: sourceHeight, - }; + const native_aspect = sourceWidth / sourceHeight; const aspect = currentWidth / currentHeight; - let x_pad = currentCropX; - let y_pad = currentCropY; + let x_pad = currentCropStartX; + let y_pad = currentCropStartY; let scale = 0; if (native_aspect >= aspect) { scale = currentWidth / currentCropWidth; - x_pad = currentCropX; - y_pad = currentCropY - Math.floor(paddingHeight / (2 * scale)); + x_pad = currentCropStartX; + y_pad = currentCropStartY - Math.floor(paddingHeight / (2 * scale)); } else if (native_aspect < aspect) { scale = currentHeight / currentCropHeight; - x_pad = currentCropX - Math.floor(paddingWidth / (2 * scale)); - y_pad = currentCropY; + x_pad = currentCropStartX - Math.floor(paddingWidth / (2 * scale)); + y_pad = currentCropStartY; } if (width < 0) { @@ -459,6 +444,9 @@ export const WebsocketH264Provider: React.FC = ({ if (cropWidth == 0 || cropHeight == 0) { return; } + + const sid = sidRef.current; + if (!sid) return; await api.setCrop(sid, { x, y, @@ -474,6 +462,10 @@ export const WebsocketH264Provider: React.FC = ({ currentHeight, paddingWidth, paddingHeight, + currentCropHeight, + currentCropWidth, + currentCropStartX, + currentCropStartY, ], ); From a124d5c00f77d44c74cf5cce6996e0e8c2c42957 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Tue, 31 Mar 2026 12:40:50 +1100 Subject: [PATCH 10/15] Move camera control and related components/providers to a sub directory. --- src/ui/experimental/{ => camera-control}/camera-control.tsx | 2 +- src/ui/experimental/{ => camera-control}/h264-api.tsx | 0 src/ui/experimental/{ => camera-control}/h264-fetch.tsx | 0 src/ui/experimental/{ => camera-control}/image-context.tsx | 0 src/ui/experimental/{ => camera-control}/video-provider.tsx | 0 .../{ => camera-control}/websocket-h264-provider.tsx | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename src/ui/experimental/{ => camera-control}/camera-control.tsx (99%) rename src/ui/experimental/{ => camera-control}/h264-api.tsx (100%) rename src/ui/experimental/{ => camera-control}/h264-fetch.tsx (100%) rename src/ui/experimental/{ => camera-control}/image-context.tsx (100%) rename src/ui/experimental/{ => camera-control}/video-provider.tsx (100%) rename src/ui/experimental/{ => camera-control}/websocket-h264-provider.tsx (100%) diff --git a/src/ui/experimental/camera-control.tsx b/src/ui/experimental/camera-control/camera-control.tsx similarity index 99% rename from src/ui/experimental/camera-control.tsx rename to src/ui/experimental/camera-control/camera-control.tsx index 826dffe..97762a9 100644 --- a/src/ui/experimental/camera-control.tsx +++ b/src/ui/experimental/camera-control/camera-control.tsx @@ -5,7 +5,7 @@ import React, { useContext, useCallback, } from "react"; -import { cn } from "../../lib/utils"; +import { cn } from "../../../lib/utils"; import { ImageContext } from "./image-context"; function debounceResize( diff --git a/src/ui/experimental/h264-api.tsx b/src/ui/experimental/camera-control/h264-api.tsx similarity index 100% rename from src/ui/experimental/h264-api.tsx rename to src/ui/experimental/camera-control/h264-api.tsx diff --git a/src/ui/experimental/h264-fetch.tsx b/src/ui/experimental/camera-control/h264-fetch.tsx similarity index 100% rename from src/ui/experimental/h264-fetch.tsx rename to src/ui/experimental/camera-control/h264-fetch.tsx diff --git a/src/ui/experimental/image-context.tsx b/src/ui/experimental/camera-control/image-context.tsx similarity index 100% rename from src/ui/experimental/image-context.tsx rename to src/ui/experimental/camera-control/image-context.tsx diff --git a/src/ui/experimental/video-provider.tsx b/src/ui/experimental/camera-control/video-provider.tsx similarity index 100% rename from src/ui/experimental/video-provider.tsx rename to src/ui/experimental/camera-control/video-provider.tsx diff --git a/src/ui/experimental/websocket-h264-provider.tsx b/src/ui/experimental/camera-control/websocket-h264-provider.tsx similarity index 100% rename from src/ui/experimental/websocket-h264-provider.tsx rename to src/ui/experimental/camera-control/websocket-h264-provider.tsx From d997fa4a3ab69e4f8fd24db47c814e8619c664e9 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Tue, 31 Mar 2026 12:58:26 +1100 Subject: [PATCH 11/15] Add type definition for reused bounding box. --- .../camera-control/camera-control.tsx | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/ui/experimental/camera-control/camera-control.tsx b/src/ui/experimental/camera-control/camera-control.tsx index 97762a9..b9178b5 100644 --- a/src/ui/experimental/camera-control/camera-control.tsx +++ b/src/ui/experimental/camera-control/camera-control.tsx @@ -27,6 +27,13 @@ function debounceResize( return debounced; } +type BoundingBox = { + startX: number; + startY: number; + width: number; + height: number; +}; + export interface CameraControlProps { className?: string; onMousePositionChange?: ( @@ -34,12 +41,7 @@ export interface CameraControlProps { ) => void; onClick?: (pos: { x: number; y: number; intensity: number }) => void; showIntensity?: boolean; - onZoom?: (box: { - startX: number; - startY: number; - width: number; - height: number; - }) => void; + onZoom?: (box: BoundingBox) => void; sizeFollowsImage?: boolean; } @@ -49,7 +51,7 @@ export const CameraControl: React.FC = ({ onClick, showIntensity = false, onZoom, - sizeFollowsImage = false + sizeFollowsImage = false, }) => { const { image, reportSize, reportZoom, reportDrag, clearZoom } = useContext(ImageContext); @@ -62,12 +64,7 @@ export const CameraControl: React.FC = ({ x: number; y: number; } | null>(null); - const [zoomBox, setZoomBox] = useState<{ - startX: number; - startY: number; - width: number; - height: number; - } | null>(null); + const [zoomBox, setZoomBox] = useState(null); const [dragStart, setDragStart] = useState<{ startX: number; startY: number; @@ -309,9 +306,9 @@ export const CameraControl: React.FC = ({ const handleKeyUp = (e: React.KeyboardEvent) => { if (e.code === "Space") { setSpaceHeld(false); - if (!dragStart) { + if (!dragStart) { setCursorDisplay("crosshair"); - }; + } e.preventDefault(); } }; From aab712de3045866f7b1978410742102a914c5a51 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Tue, 31 Mar 2026 14:32:47 +1100 Subject: [PATCH 12/15] Create default resolution constant --- .../experimental/camera-control/h264-api.tsx | 10 ++++++++++ .../experimental/camera-control/h264-fetch.tsx | 6 +++--- .../camera-control/websocket-h264-provider.tsx | 18 +++++++++--------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/ui/experimental/camera-control/h264-api.tsx b/src/ui/experimental/camera-control/h264-api.tsx index e7adda8..ab95d29 100644 --- a/src/ui/experimental/camera-control/h264-api.tsx +++ b/src/ui/experimental/camera-control/h264-api.tsx @@ -5,6 +5,16 @@ export type SessionResolution = Resolution & { paddingHeight: number; }; +type DefaultResolutionType = { + width: number, + height: number +} + +export const DefaultResolution: DefaultResolutionType = { + width: 1024, + height: 1024 +}; + export interface H264Api { /** Create a session if needed. Return the session ID. */ createSession: (signal?: AbortSignal) => Promise; diff --git a/src/ui/experimental/camera-control/h264-fetch.tsx b/src/ui/experimental/camera-control/h264-fetch.tsx index d705910..58e3c68 100644 --- a/src/ui/experimental/camera-control/h264-fetch.tsx +++ b/src/ui/experimental/camera-control/h264-fetch.tsx @@ -1,4 +1,4 @@ -import { type Crop, type H264Api } from "./h264-api"; +import { type Crop, type H264Api, DefaultResolution } from "./h264-api"; export function h264FetchApi(url: string): H264Api { return { @@ -11,8 +11,8 @@ export function h264FetchApi(url: string): H264Api { colour_mapping: "none", crop: null, resolution: { - width: 1024, - height: 1024, + width: DefaultResolution.width, + height: DefaultResolution.height, }, }), }); diff --git a/src/ui/experimental/camera-control/websocket-h264-provider.tsx b/src/ui/experimental/camera-control/websocket-h264-provider.tsx index e39e2ce..ec68f95 100644 --- a/src/ui/experimental/camera-control/websocket-h264-provider.tsx +++ b/src/ui/experimental/camera-control/websocket-h264-provider.tsx @@ -8,7 +8,7 @@ import React, { import { ImageContext } from "./image-context"; -import { type H264Api } from "./h264-api"; +import { type H264Api, DefaultResolution } from "./h264-api"; export interface WebsocketH264ProviderProps { children: React.ReactNode; @@ -27,12 +27,12 @@ export const WebsocketH264Provider: React.FC = ({ // State // ================== const [imageBitmap, setImageBitmap] = useState(null); - const [sourceWidth, setSourceWidth] = useState(1024); - const [sourceHeight, setSourceHeight] = useState(1024); - const [currentWidth, setCurrentWidth] = useState(1024); - const [currentHeight, setCurrentHeight] = useState(1024); - const [currentCropHeight, setCurrentCropHeight] = useState(1024); - const [currentCropWidth, setCurrentCropWidth] = useState(1024); + const [sourceWidth, setSourceWidth] = useState(DefaultResolution.width); + const [sourceHeight, setSourceHeight] = useState(DefaultResolution.height); + const [currentWidth, setCurrentWidth] = useState(DefaultResolution.width); + const [currentHeight, setCurrentHeight] = useState(DefaultResolution.height); + const [currentCropHeight, setCurrentCropHeight] = useState(DefaultResolution.height); + const [currentCropWidth, setCurrentCropWidth] = useState(DefaultResolution.width); const [currentCropStartX, setCurrentCropStartX] = useState(0); const [currentCropStartY, setCurrentCropStartY] = useState(0); const [paddingWidth, setPaddingWidth] = useState(0); @@ -43,8 +43,8 @@ export const WebsocketH264Provider: React.FC = ({ // ================== const wsRef = useRef(null); const dimsRef = useRef<{ width: number; height: number }>({ - width: 1024, - height: 1024, + width: DefaultResolution.width, + height: DefaultResolution.height, }); const abortedRef = useRef(false); const configuringRef = useRef(false); From e413dc83a600885573d15c2cf189c7aed17bdd84 Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Wed, 1 Apr 2026 13:14:14 +1100 Subject: [PATCH 13/15] Added docstrings and small cleanup --- .../experimental/camera-control/camera-control.tsx | 12 ++++++++++-- .../camera-control/websocket-h264-provider.tsx | 10 ++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ui/experimental/camera-control/camera-control.tsx b/src/ui/experimental/camera-control/camera-control.tsx index b9178b5..e88320c 100644 --- a/src/ui/experimental/camera-control/camera-control.tsx +++ b/src/ui/experimental/camera-control/camera-control.tsx @@ -45,6 +45,16 @@ export interface CameraControlProps { sizeFollowsImage?: boolean; } +/** + * Returns a webcomponent that provides a container (canvas) for various video or image + * producers, and implements various controls and annotations. + * @param onMousePositionChange - Callback called on mouse movement inside canvas. + * @param onClick - Callback called on a mouse click within canvas. + * @param showIntensity - Whether to show the pixel intensity at mouse position as a tooltip. + * @param onZoom - Callback called after a bounding box is drawn. + * @param sizeFollowsImage - Resize the canvas if the image size changes + * @returns + */ export const CameraControl: React.FC = ({ className, onMousePositionChange, @@ -345,8 +355,6 @@ export const CameraControl: React.FC = ({ border: "1px solid red", width: "100%", height: "100%", - // width: `${w}px`, - // height: `${h}px`, }} /> {/* Crosshair */} diff --git a/src/ui/experimental/camera-control/websocket-h264-provider.tsx b/src/ui/experimental/camera-control/websocket-h264-provider.tsx index ec68f95..6167aa0 100644 --- a/src/ui/experimental/camera-control/websocket-h264-provider.tsx +++ b/src/ui/experimental/camera-control/websocket-h264-provider.tsx @@ -17,6 +17,14 @@ export interface WebsocketH264ProviderProps { api: H264Api; } +/** + * A provider to be used with the CameraControl Webcomponent. This provider + * receives streams from the h264-websocket-stream server and maps camera control + * controls to approriate endpoints to control the stream. + * @param api - An API the satisfies the H264Api Interface for + * communicating with h264 websocket stream server. + * @returns + */ export const WebsocketH264Provider: React.FC = ({ children, sessionId = null, @@ -294,8 +302,6 @@ export const WebsocketH264Provider: React.FC = ({ dimsRef.current = { width: meta.width, height: meta.height }; configuredRef.current = false; // force reconfigure void tryConfigure(); - } else { - /**/ } } } catch (e) { From 7d2c944e3070123f2561eb4a3dd4970a95e649af Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Wed, 1 Apr 2026 13:14:29 +1100 Subject: [PATCH 14/15] Change guarding for video frames by create isVideo --- src/ui/experimental/camera-control/camera-control.tsx | 7 +++---- src/ui/experimental/camera-control/image-context.tsx | 9 ++++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/ui/experimental/camera-control/camera-control.tsx b/src/ui/experimental/camera-control/camera-control.tsx index e88320c..c36ebf8 100644 --- a/src/ui/experimental/camera-control/camera-control.tsx +++ b/src/ui/experimental/camera-control/camera-control.tsx @@ -6,7 +6,7 @@ import React, { useCallback, } from "react"; import { cn } from "../../../lib/utils"; -import { ImageContext } from "./image-context"; +import { ImageContext, isVideo } from "./image-context"; function debounceResize( fn: (entry: ResizeObserverEntry) => void, @@ -88,8 +88,7 @@ export const CameraControl: React.FC = ({ // Helper to get dimensions safely const getDimensions = () => { - if (!image) return { w: 0, h: 0 }; - if (typeof image === "object" && "video" in image && image.video) { + if (isVideo(image)) { return { w: image.video.videoWidth, h: image.video.videoHeight }; } if (image instanceof ImageBitmap) { @@ -134,7 +133,7 @@ export const CameraControl: React.FC = ({ } try { - if (typeof image === "object" && "video" in image && image.video) { + if (isVideo(image)) { ctx.drawImage(image.video, 0, 0); } else if (image instanceof ImageBitmap) { ctx.drawImage(image, 0, 0); diff --git a/src/ui/experimental/camera-control/image-context.tsx b/src/ui/experimental/camera-control/image-context.tsx index d002849..1020ab0 100644 --- a/src/ui/experimental/camera-control/image-context.tsx +++ b/src/ui/experimental/camera-control/image-context.tsx @@ -13,4 +13,11 @@ export const ImageContext = createContext({ reportZoom: () => { }, reportDrag: () => { }, clearZoom: () => { } -}); \ No newline at end of file +}); + +export type VideoFrame = { video: HTMLVideoElement; frameId: number }; +export const isVideo = (image: unknown): image is VideoFrame => + typeof image === "object" && + image !== null && + "video" in image && + !!(image as Record).video; \ No newline at end of file From aa337840b00ea9b03a91d7a876685b65433dec1a Mon Sep 17 00:00:00 2001 From: Stephen Mudie Date: Wed, 1 Apr 2026 13:21:12 +1100 Subject: [PATCH 15/15] Add default delay to debounce function and use it. --- src/ui/experimental/camera-control/camera-control.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ui/experimental/camera-control/camera-control.tsx b/src/ui/experimental/camera-control/camera-control.tsx index c36ebf8..8400f3a 100644 --- a/src/ui/experimental/camera-control/camera-control.tsx +++ b/src/ui/experimental/camera-control/camera-control.tsx @@ -10,7 +10,7 @@ import { ImageContext, isVideo } from "./image-context"; function debounceResize( fn: (entry: ResizeObserverEntry) => void, - delay: number, + delay: number = 100 ) { let timer: ReturnType | undefined; @@ -109,7 +109,7 @@ export const CameraControl: React.FC = ({ // Report the size back to the provider reportSize(Math.floor(width), Math.floor(height)); - }, 100); + }); const observer = new ResizeObserver((entries) => handler(entries[0]));