feat(screen-mirror): implement cursor-follow interaction using absolute pointer mapping

src/components/Trackpad/ScreenMirror.tsx

@@ -4,35 +4,125 @@ import type React from "react"
 import { useRef } from "react"
 import { useConnection } from "../../contexts/ConnectionProvider"
 import { useMirrorStream } from "../../hooks/useMirrorStream"
+import { useRemoteConnection } from "../../hooks/useRemoteConnection"
 
 interface ScreenMirrorProps {
 	scrollMode: boolean
 	isTracking: boolean
-	handlers: React.HTMLAttributes<HTMLDivElement>
 }
 
 const TEXTS = {
 	WAITING: "Waiting for screen...",
 	AUTOMATIC: "Mirroring will start automatically",
 }
 
-export const ScreenMirror = ({
-	scrollMode,
-	isTracking,
-	handlers,
-}: ScreenMirrorProps) => {
+export const ScreenMirror = ({ scrollMode, isTracking }: ScreenMirrorProps) => {
 	const { wsRef, status } = useConnection()
 	const canvasRef = useRef<HTMLCanvasElement>(null)
 	const { hasFrame } = useMirrorStream(wsRef, canvasRef, status)
+	const { send } = useRemoteConnection()
+	const lastPos = useRef<{ x: number; y: number } | null>(null)
+	const lastMoveTime = useRef(0)
+
+	const getAbsoluteCoords = (clientX: number, clientY: number) => {
+		if (!canvasRef.current || !hasFrame) return null
+		const canvas = canvasRef.current
+		const rect = canvas.getBoundingClientRect()
+
+		const cw = rect.width
+		const ch = rect.height
+		const iw = canvas.width
+		const ih = canvas.height
+
+		if (iw === 0 || ih === 0) return null
+
+		const scale = Math.min(cw / iw, ch / ih)
+		const dw = iw * scale
+		const dh = ih * scale
+		const offsetX = (cw - dw) / 2
+		const offsetY = (ch - dh) / 2
+
+		const cx = clientX - rect.left
+		const cy = clientY - rect.top
+
+		const x = (cx - offsetX) / scale
+		const y = (cy - offsetY) / scale
+
+		if (x < 0 || x > iw || y < 0 || y > ih) {
+			return {
+				x: Math.max(0, Math.min(iw, x)),
+				y: Math.max(0, Math.min(ih, y)),
+			}
+		}
+
+		return { x, y }
+	}
+
+	const handlePointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
+		const coords = getAbsoluteCoords(e.clientX, e.clientY)
+		if (!coords) return
+
+		e.currentTarget.setPointerCapture(e.pointerId)
+		if (e.pointerType === "mouse" || !scrollMode) {
+			send({ type: "absolute", x: coords.x, y: coords.y })
+		}
+		if (e.pointerType === "mouse") {
+			const button =
+				e.button === 0 ? "left" : e.button === 2 ? "right" : "middle"
+			send({ type: "click", button, press: true })
+		} else {
+			if (!scrollMode) {
+				send({ type: "click", button: "left", press: true })
+			}
+		}
+		lastPos.current = { x: e.clientX, y: e.clientY }
+	}
+
+	const handlePointerMove = (e: React.PointerEvent<HTMLDivElement>) => {
+		const now = performance.now()
+		if (now - lastMoveTime.current < 16) return
+		lastMoveTime.current = now
+
+		const coords = getAbsoluteCoords(e.clientX, e.clientY)
+		if (!coords) return
+
+		if (e.pointerType === "mouse") {
+			send({ type: "absolute", x: coords.x, y: coords.y })
+		} else {
+			if (scrollMode && lastPos.current) {
+				const dx = e.clientX - lastPos.current.x
+				const dy = e.clientY - lastPos.current.y
+				send({ type: "scroll", dx: -dx * 1.5, dy: -dy * 1.5 })
+				lastPos.current = { x: e.clientX, y: e.clientY }
+			} else if (lastPos.current && !scrollMode) {
+				send({ type: "absolute", x: coords.x, y: coords.y })
+			}
+		}
+	}
+
+	const handlePointerUp = (e: React.PointerEvent<HTMLDivElement>) => {
+		if (e.currentTarget.hasPointerCapture(e.pointerId)) {
+			e.currentTarget.releasePointerCapture(e.pointerId)
+		}
+		if (e.pointerType === "mouse") {
+			const button =
+				e.button === 0 ? "left" : e.button === 2 ? "right" : "middle"
+			send({ type: "click", button, press: false })
+		} else {
+			if (!scrollMode) {
+				send({ type: "click", button: "left", press: false })
+			}
+		}
+		lastPos.current = null
+	}
 
 	return (
 		<div className="absolute inset-0 flex items-center justify-center bg-black overflow-hidden select-none touch-none">
 			{/* Mirror Canvas */}
 			<canvas
 				ref={canvasRef}
-				className={`w-full h-full object-contain transition-opacity duration-500 ${
-					hasFrame ? "opacity-100" : "opacity-0"
-				}`}
+				className={`w-full h-full object-contain transition-opacity duration-500 ${hasFrame ? "opacity-100" : "opacity-0"
+					}`}
 			/>
 
 			{/* Standby UI */}
@@ -49,9 +139,13 @@ export const ScreenMirror = ({
 			{/* Transparent Gesture Overlay */}
 			<div
 				className="absolute inset-0 z-10"
-				{...handlers}
+				onPointerDown={handlePointerDown}
+				onPointerMove={handlePointerMove}
+				onPointerUp={handlePointerUp}
+				onPointerCancel={handlePointerUp}
+				onContextMenu={(e) => e.preventDefault()}
 				style={{
-					cursor: scrollMode ? "ns-resize" : isTracking ? "none" : "default",
+					cursor: scrollMode ? "ns-resize" : "crosshair",
 				}}
 			/>
 		</div>

src/routes/trackpad.tsx

@@ -224,11 +224,7 @@ function TrackpadPage() {
 					scrollMode={scrollMode}
 					handlers={handlers}
 				/>
-				<ScreenMirror
-					isTracking={isTracking}
-					scrollMode={scrollMode}
-					handlers={handlers}
-				/>
+				<ScreenMirror isTracking={isTracking} scrollMode={scrollMode} />
 				{bufferText !== "" && <BufferBar bufferText={bufferText} />}
 			</div>
 

src/server/InputHandler.ts

@@ -14,8 +14,11 @@ export interface InputMessage {
 		| "text"
 		| "zoom"
 		| "combo"
+		| "absolute"
 	dx?: number
 	dy?: number
+	x?: number
+	y?: number
 	button?: "left" | "right" | "middle"
 	press?: boolean
 	key?: string
@@ -58,6 +61,7 @@ export class InputHandler {
 		}
 
 		const MAX_COORD = 2000
+		const MAX_ABS_COORD = 10000
 		if (this.isFiniteNumber(msg.dx)) {
 			msg.dx = this.clamp(msg.dx, -MAX_COORD, MAX_COORD)
 		} else {
@@ -68,6 +72,16 @@ export class InputHandler {
 		} else {
 			msg.dy = 0
 		}
+		if (this.isFiniteNumber(msg.x)) {
+			msg.x = this.clamp(msg.x, -MAX_ABS_COORD, MAX_ABS_COORD)
+		} else {
+			msg.x = undefined
+		}
+		if (this.isFiniteNumber(msg.y)) {
+			msg.y = this.clamp(msg.y, -MAX_ABS_COORD, MAX_ABS_COORD)
+		} else {
+			msg.y = undefined
+		}
 		if (this.isFiniteNumber(msg.delta)) {
 			msg.delta = this.clamp(msg.delta, -MAX_COORD, MAX_COORD)
 		} else {
@@ -144,6 +158,29 @@ export class InputHandler {
 				}
 				break
 
+			case "absolute":
+				if (
+					typeof msg.x === "number" &&
+					typeof msg.y === "number" &&
+					Number.isFinite(msg.x) &&
+					Number.isFinite(msg.y)
+				) {
+					try {
+						// Attempt ydotool absolute movement first
+						const success = await moveRelative(msg.x, msg.y, true)
+
+						// Fallback to absolute positioning if ydotool is unavailable or fails
+						if (!success) {
+							await mouse.setPosition(
+								new Point(Math.round(msg.x), Math.round(msg.y)),
+							)
+						}
+					} catch (err) {
+						console.error("Absolute move event failed:", err)
+					}
+				}
+				break
+
 			case "click": {
 				const VALID_BUTTONS = ["left", "right", "middle"]
 				if (msg.button && VALID_BUTTONS.includes(msg.button)) {

src/server/websocket.ts

@@ -345,6 +345,7 @@ export async function createWsServer(
 						"combo",
 						"copy",
 						"paste",
+						"absolute",
 					]
 					if (!msg.type || !VALID_INPUT_TYPES.includes(msg.type)) {
 						logger.warn(`Unknown message type: ${msg.type}`)

src/server/ydotool.ts

@@ -50,20 +50,23 @@ export async function checkYdotool(): Promise<boolean> {
  * @param dy Y offset
  * @returns true if successful, false otherwise
  */
-export async function moveRelative(dx: number, dy: number): Promise<boolean> {
+export async function moveRelative(
+	dx: number,
+	dy: number,
+	absolute = false,
+): Promise<boolean> {
 	if (!(await checkYdotool()) || !ydotoolPath) {
 		return false
 	}
 
 	try {
-		// ydotool mousemove -x <dx> -y <dy>
-		await execFileAsync(ydotoolPath, [
-			"mousemove",
-			"-x",
-			String(dx),
-			"-y",
-			String(dy),
-		])
+		const args = ["mousemove"]
+		if (absolute) {
+			args.push("--absolute")
+		}
+		args.push("-x", String(dx), "-y", String(dy))
+
+		await execFileAsync(ydotoolPath, args)
 		return true
 	} catch (err) {
 		console.error("[ydotool] Error executing mousemove:", err)