-
-
Notifications
You must be signed in to change notification settings - Fork 26
Expand file tree
/
Copy pathirl.js
More file actions
8300 lines (7759 loc) · 377 KB
/
Copy pathirl.js
File metadata and controls
8300 lines (7759 loc) · 377 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// src/irl.js — IRL AR playground
//
// Full-screen walking avatar + camera AR passthrough + tap-to-place 3D objects.
// Visual style mirrors /avatars/:id/ar (bottom panel, gradient bg, hero CTA).
// Walking + joystick from walk-embed. Camera mode makes the real floor the stage.
import {
AmbientLight,
Box3,
BoxGeometry,
CircleGeometry,
CylinderGeometry,
DirectionalLight,
Group,
HemisphereLight,
Mesh,
MeshBasicMaterial,
MeshPhysicalMaterial,
MeshStandardMaterial,
OctahedronGeometry,
OrthographicCamera,
PerspectiveCamera,
PlaneGeometry,
Quaternion,
Raycaster,
Scene,
ShadowMaterial,
SphereGeometry,
Sprite,
SpriteMaterial,
SRGBColorSpace,
Timer,
TorusGeometry,
Vector2,
Vector3,
WebGLRenderer,
WebGLRenderTarget,
} from 'three';
import { applyCinematicDefaults, loadEnvironment, detectQualityTier } from './shared/cinematic-render.js';
import nipplejs from 'nipplejs';
import { AnimationManager } from './animation-manager.js';
import { WebXRSession } from './ar/webxr.js';
import {
clampPinScale, createPinchState, pinchEnd, pinchMove, pinchStart, touchDist,
} from './ar/pinch-scale.js';
import { resolvePlacementCapability } from './ar/placement-capability.js';
import { openQuickLook } from './ar/quick-look.js';
import { withQuickLookBanner } from './ar/quicklook-banner.js';
import { createPersistGate, placementHint } from './ar/anchor-lifecycle.js';
import { IrlNet } from './irl-net.js';
import { wireShareButton } from './irl/share-frame.js';
import { log } from './shared/log.js';
import { createAvatarPicker } from './avatar-picker.js';
import { errorStateEl, skeletonHTML, emptyStateHTML, errorStateHTML, ensureStateKitStyles, attachRetry } from './shared/state-kit.js';
import { startOnboarding, ensurePermission, needsMotionGesture, setPermissionState } from './irl/onboarding.js';
import { reserveWebGLContext, releaseWebGLContext } from './webgl-budget.js';
import { detectTier, BUDGETS, TIER_ORDER, shiftTier } from './irl/perf-budget.js';
import { sharedGLTFLoader, createLoadQueue } from './irl/load-queue.js';
import { mountPinIdle, getIdleClipJson } from './irl/pin-idle.js';
import { celebrationRingFrame, CELEBRATION_DURATION } from './irl/pin-celebration.js';
import { createWealthAura3D } from './shared/wealth-aura-3d.js';
import { fetchWealthState } from './shared/agent-wealth-state.js';
import { trackLevelUp, installLevelUpCelebrations } from './shared/wealth-levelup.js';
import { roomOriginWorld, agentWorldPosition, localToGeo, calibrateRoomOrigin } from './irl/room-anchor.js';
import { anchorPoseToPin, yawDegFromQuat, roomPlacementFromHit, roomRelFromGeo } from './irl/floor-anchor.js';
import { initRoomMode } from './irl/room-mode.js';
import { initMarkerMode } from './irl/marker-mode.js';
import { isMarkerRoomId, markerWorldPos, markerRelFromWorld, markerYawFromEdge, markerRoomBlock } from './irl/marker-anchor.js';
import { markerPoseCamera } from './irl/marker-pose.js';
import { cornerCenter, cornerRightMid } from './irl/qr-detect.js';
import { createRoomGhost } from './irl/room-ghost.js';
import { isFiniteReading, isCompassFresh, shouldUseAbsoluteYaw, resolveLockYaw, clampPitch, screenPitchDeg } from './irl/sensor-fusion.js';
import { deriveVerticalFovDeg, DEFAULT_DIAG_FOV_DEG } from './irl/camera-fov.js';
import { pinBandAction } from './irl/proximity-band.js';
import { gpsAccuracyBucket, easeGpsTransition, GPS_TRANSITION_MS } from './irl/gps-lifecycle.js';
import { pickLabelHit } from './irl/tap-pick.js';
import {
shouldCueArrival,
relativeBearing,
isFacingAgent,
edgeNudgePlacement,
nearestAgent,
} from './irl/proximity-cue.js';
import { loadInto } from './shared/async-state.js';
import { openMapPlacePicker } from './irl/map-place.js';
import { openPrivacyCenter, maybeShowFirstRunDisclosure, getDiscoveryPrecision } from './irl/privacy-center.js';
import { GlassesBridge } from './irl/glasses/bridge.js';
import { openGlassesConnect } from './irl/glasses/connect-ui.js';
import { loadLeaflet } from './shared/leaflet-loader.js';
import { initDiscovery } from './irl/discovery.js';
import { initIrlDrops } from './shared/irl-drops.js';
import { walletChipEl, hasWallet } from './shared/agent-wallet-chip.js';
import { buildSolanaPayUri } from './shared/solana-pay.js';
import { renderQRToSVG } from './erc8004/qr.js';
const AVATAR_URL_DEFAULT = '/avatars/default.glb';
const ANIMATIONS_MANIFEST_URL = '/animations/manifest.json';
const CLIP_IDLE = 'idle';
const CLIP_WALK = 'av-walk-feminine';
const CLIP_RUN = 'av-walk-feminine';
const WALK_SPEED = 1.6;
const RUN_SPEED = 4.0;
const NATURAL_WALK_SPEED = 1.5;
const NATURAL_RUN_SPEED = 3.4;
const TURN_LERP = 0.18;
const CAM_LERP = 0.12;
const LEAN_WALK_RAD = 0.05;
const LEAN_RUN_RAD = 0.13;
const LEAN_LERP = 0.12;
const GROUND_RADIUS = 16;
const CAM_OFFSET = new Vector3(0, 1.85, 3.6);
const CAM_LOOK_OFFSET = new Vector3(0, 1.1, 0);
const SPAWN_DURATION = 0.38; // seconds for placed object scale-up
const TAP_THRESHOLD = 10; // px — beyond this it's a drag, not a tap
// ── Camera-aware agents ──────────────────────────────────────────────────
// Loaded nearby agents notice the viewer: they yaw their body toward the phone
// camera, lead with a head turn inside a natural neck cone, idle-drift when
// nobody's close, and perk up once when you first walk into range.
const AWARE_RADIUS_M = 12; // viewer must be this close (m) to engage tracking
const NOTICE_RADIUS_M = 4; // crossing this (m) fires the one-shot "notice"
const AWARE_MAX_AGENTS = 5; // only the nearest N get full per-frame head tracking
const HEAD_CLAMP = 0.7; // max neck yaw/pitch from rest (rad) — natural cone
const BODY_SLERP = 0.05; // eased body yaw toward the camera
const BODY_RETURN_SLERP = 0.03; // eased body yaw back to the placed heading
const HEAD_SLERP = 0.12; // eased head toward its gaze target
const HEAD_SLERP_NOTICE = 0.40; // boosted head slerp during a notice reaction
const NOTICE_BOOST_SEC = 0.40; // seconds the head-slerp boost lasts after a notice
const POP_DURATION = 0.45; // seconds for the perk-up scale pop
const POP_AMOUNT = 0.04; // peak +Y scale during the pop (1.0 → 1.04 → 1.0)
// ── URL params ────────────────────────────────────────────────────────────
const params = new URLSearchParams(location.search);
const avatarIdParam = params.get('avatar') || '';
const highlightPinId = params.get('highlight') || '';
// Deep-focus a specific agent's pin (from an agent profile's "View in IRL" link):
// once that agent's pin loads nearby, flash it and open its inspect card. Fires
// once so it doesn't re-open the sheet on every nearby refresh.
const agentFocusId = params.get('agent') || '';
let _agentFocusDone = false;
function resolveAvatarUrl(id) {
if (!id) return AVATAR_URL_DEFAULT;
if (/^https?:\/\//i.test(id) || id.startsWith('/')) return id;
return `/api/avatars/${encodeURIComponent(id)}/glb`;
}
// ── DOM refs ──────────────────────────────────────────────────────────────
const $ = (id) => document.getElementById(id);
// Pre-allocated vectors used in tick() camera-awareness detection (avoids GC churn)
const _camWorldDir = new Vector3();
const _camDirH = new Vector3();
const _toAgentH = new Vector3();
const canvas = $('irl-canvas');
const videoEl = $('irl-camera');
const joystickEl = $('irl-joystick');
const nameEl = $('irl-avatar-name');
const subtitleEl = $('irl-subtitle');
// First-run guidance: the subtitle is the always-visible signpost to the next
// concrete step in the core loop (turn on Camera AR → aim → Pin here). It tracks
// three states so a brand-new user is never left guessing what to tap next.
const SUBTITLE = {
cameraOff: 'Turn on Camera AR, then tap Pin here to anchor your agent in real space.',
aiming: 'Tap Move here to set your agent on the floor, then Pin here to anchor it.',
placing: 'Aim the ring at the floor and tap to set your agent down, then Pin here.',
pinned: 'Your agent is anchored here. Tap Pin here again to release it.',
orbit: 'Drag to orbit your agent. Turn on Camera AR to place it in your real room.',
};
function setSubtitle(text) { if (subtitleEl) subtitleEl.textContent = text; }
const statusEl = $('irl-status');
const statusTxt = $('irl-status-text');
const spinner = $('irl-spinner');
const cameraBtn = $('irl-camera-btn');
const cameraLabel = $('irl-camera-label');
const placeBtn = $('irl-place-btn');
const clearBtn = $('irl-clear-btn');
const pickerEl = $('irl-picker');
const avatarBtn = $('irl-avatar-btn');
const lockBtn = $('irl-lock-btn');
const carryBtn = $('irl-carry-btn'); // "Move here" — carry the agent and set it on the floor
const anchorBtn = $('irl-anchor-btn'); // WebXR "Place on floor" — revealed only when supported
const xrOverlay = $('irl-xr-overlay'); // WebXR dom-overlay root (in-session hint + exit + error)
const xrBarEl = $('irl-xr-bar'); // hint+exit pill; .is-anchored reveals the ✓ confirm
const xrHintEl = $('irl-xr-hint');
const xrExitBtn = $('irl-xr-exit');
// ── Status helpers ────────────────────────────────────────────────────────
function setStatus(msg, { error = false, warn = false, loading = false, sticky = false } = {}) {
clearTimeout(setStatus._t);
if (!msg) { statusEl.classList.add('is-hidden'); return; }
statusTxt.textContent = msg;
statusEl.classList.remove('is-hidden');
statusEl.classList.toggle('is-error', error);
statusEl.classList.toggle('is-warn', warn && !error);
spinner.classList.toggle('hidden', !loading);
if (!sticky) setStatus._t = setTimeout(() => statusEl.classList.add('is-hidden'), 3000);
}
// ── Renderer / scene ──────────────────────────────────────────────────────
const renderer = new WebGLRenderer({ canvas, antialias: true, alpha: true, preserveDrawingBuffer: true });
renderer.setSize(window.innerWidth, window.innerHeight, false);
renderer.setClearColor(0x000000, 0);
// Filmic tone mapping + VSM soft shadows + IBL from a real HDRI: the shared
// bar every viewer on the platform now matches (src/shared/cinematic-render.js).
const qualityTier = detectQualityTier();
applyCinematicDefaults(renderer, { exposure: 1.15, tier: qualityTier });
const scene = new Scene();
loadEnvironment(renderer, scene, qualityTier === 'mobile' ? null : 'studio');
// Keyed like a studio portrait, not a flood: the IBL carries the specular/rim
// detail, the sun carries shape + shadow. The old 0.55 ambient + 0.6 hemi fill
// flattened every model into a cardboard cutout.
const ambientLight = new AmbientLight(0xffffff, 0.3);
scene.add(ambientLight);
const hemi = new HemisphereLight(0xbcd6ff, 0x202830, 0.45);
hemi.position.set(0, 5, 0);
scene.add(hemi);
const sun = new DirectionalLight(0xffffff, 1.55);
sun.position.set(4, 8, 6);
sun.castShadow = true;
sun.shadow.mapSize.set(1024, 1024);
sun.shadow.camera.near = 0.5;
sun.shadow.camera.far = 32;
sun.shadow.camera.left = -12;
sun.shadow.camera.right = 12;
sun.shadow.camera.top = 12;
sun.shadow.camera.bottom = -12;
sun.shadow.bias = -0.0005;
scene.add(sun);
// ── Performance tier + render budget (E2) ───────────────────────────────────
// IRL owns ONE long-lived WebGL context shared by every pin (all pins live in
// this single scene). Reserve it against the page-wide budget so the homepage's
// other renderers stay under the browser's ~16-context cap; never spawn a
// per-pin context. Released on pagehide.
reserveWebGLContext();
window.addEventListener('pagehide', releaseWebGLContext, { once: true });
// Pick a device class from REAL signals (cores, memory, DPR, mobile UA, GPU
// capabilities) and hold its budget. `baseTier` is the hardware ceiling: the
// runtime watchdog may degrade below it under load and recover back up to it,
// but never above what the device can actually carry.
let activeTier = detectTier(renderer);
const baseTier = activeTier;
let budget = BUDGETS[activeTier];
// One shared GLTFLoader (Draco + meshopt) behind a nearest-first, concurrency
// capped queue — replaces the old `new GLTFLoader()` per pin + `loadedCount < 5`
// guard. Priority is the pin's live camera distance (set in enforceLOD).
const glbQueue = createLoadQueue({
run: (pin) => sharedGLTFLoader().loadAsync(pin.avatar_url),
maxActive: budget.maxGLB,
priorityOf: (pin) => (pin._lodDist != null ? pin._lodDist : (pin.distance_m != null ? pin.distance_m : 1e9)),
});
// Warm the shared idle clip while the first GLBs are still downloading, so a
// freshly mounted pin can start breathing on its very first frame.
getIdleClipJson();
// Anisotropic filtering on every model texture — cloth/skin detail stays sharp
// at the oblique angles an avatar standing on the real floor is actually seen
// from. Capped at 8: past that the visual gain is nil and the bandwidth isn't.
const _maxAniso = Math.min(8, renderer.capabilities.getMaxAnisotropy() || 1);
const _ANISO_SLOTS = ['map', 'normalMap', 'roughnessMap', 'metalnessMap', 'aoMap', 'emissiveMap'];
function sharpenModelTextures(root) {
root.traverse((n) => {
if (!n.isMesh) return;
const mats = Array.isArray(n.material) ? n.material : [n.material];
for (const m of mats) {
if (!m) continue;
for (const slot of _ANISO_SLOTS) {
const tex = m[slot];
if (tex && tex.anisotropy < _maxAniso) {
tex.anisotropy = _maxAniso;
tex.needsUpdate = true;
}
}
}
});
}
// Apply the active tier to the renderer + load queue. Called at boot and each
// time the watchdog steps the tier. Idempotent.
function applyTierToRenderer() {
renderer.setPixelRatio(Math.min(window.devicePixelRatio, budget.pixelRatio));
renderer.setSize(window.innerWidth, window.innerHeight, false);
const wantShadow = budget.shadow > 0;
renderer.shadowMap.enabled = wantShadow;
sun.castShadow = wantShadow;
if (wantShadow && sun.shadow.mapSize.x !== budget.shadow) {
sun.shadow.mapSize.set(budget.shadow, budget.shadow);
// Force the shadow map to regenerate at the new resolution.
if (sun.shadow.map) { sun.shadow.map.dispose(); sun.shadow.map = null; }
}
glbQueue.setMaxActive(budget.maxGLB);
}
applyTierToRenderer();
// ── Ground ────────────────────────────────────────────────────────────────
const groundOpaque = new Mesh(
new CircleGeometry(GROUND_RADIUS, 64),
new MeshStandardMaterial({ color: 0x121820, roughness: 0.95, metalness: 0.0 }),
);
groundOpaque.rotation.x = -Math.PI / 2;
groundOpaque.receiveShadow = true;
scene.add(groundOpaque);
const groundShadow = new Mesh(
new CircleGeometry(GROUND_RADIUS, 64),
new ShadowMaterial({ opacity: 0.5 }),
);
groundShadow.rotation.x = -Math.PI / 2;
groundShadow.receiveShadow = true;
groundShadow.visible = false;
scene.add(groundShadow);
// Invisible plane at y=0 for raycasting tap-to-place targets
const rayPlane = new Mesh(
new PlaneGeometry(60, 60),
new MeshStandardMaterial({ visible: false, side: 2 }),
);
rayPlane.rotation.x = -Math.PI / 2;
rayPlane.position.y = 0.005;
scene.add(rayPlane);
// Carry-and-place reticle — a soft teal ground ring that tracks where the screen
// centre meets the floor while "Move here" is armed, so you can aim a real spot in
// the room and set your agent down there (the gesture the fixed camera-offset never
// allowed). Hidden until carry mode reveals it; pulses on a successful drop.
const carryReticle = new Group();
{
const ring = new Mesh(
new TorusGeometry(0.28, 0.022, 10, 44),
new MeshBasicMaterial({ color: 0x34d399, transparent: true, opacity: 0.9, depthTest: false }),
);
ring.rotation.x = -Math.PI / 2;
ring.renderOrder = 10;
const dot = new Mesh(
new CircleGeometry(0.05, 24),
new MeshBasicMaterial({ color: 0x34d399, transparent: true, opacity: 0.55, depthTest: false }),
);
dot.rotation.x = -Math.PI / 2;
dot.renderOrder = 10;
carryReticle.add(ring, dot);
}
carryReticle.position.y = 0.012;
carryReticle.visible = false;
scene.add(carryReticle);
// ── Camera ────────────────────────────────────────────────────────────────
// near = 0.02 m: held one-handed, a virtual agent brought close to inspect can put
// its near face inside the clip cone and vanish (and so dodge the inspect tap). 2 cm
// keeps it visible without the depth-fighting a 1 cm near would risk against the
// coplanar ground/shadow planes at the 0.02→200 ratio. Raycasting itself is
// near-independent (Raycaster.near defaults to 0), so this is purely a render fix.
const camera = new PerspectiveCamera(50, window.innerWidth / window.innerHeight, 0.02, 200);
const avatarRig = new Group();
scene.add(avatarRig);
const camDesired = new Vector3();
const camLookTarget = new Vector3();
const camLookCurrent = new Vector3();
const lockForward = new Vector3(); // scratch — derives lock yaw/pitch at pin time
let cameraYaw = 0;
let cameraPitch = 0.05;
const PITCH_MIN = -0.5;
const PITCH_MAX = 0.65;
function applyCameraImmediate() {
const offset = CAM_OFFSET.clone();
offset.applyAxisAngle(new Vector3(1, 0, 0), -cameraPitch);
offset.applyAxisAngle(new Vector3(0, 1, 0), cameraYaw);
camDesired.copy(avatarRig.position).add(offset);
camera.position.copy(camDesired);
camLookTarget.copy(avatarRig.position).add(CAM_LOOK_OFFSET);
camLookCurrent.copy(camLookTarget);
camera.lookAt(camLookCurrent);
}
applyCameraImmediate();
// ── AR passthrough ────────────────────────────────────────────────────────
let arActive = false;
let mediaStream = null;
// Single mutex over camera + render-loop ownership. getUserMedia camera-AR and the
// WebXR immersive session both want the rear camera and the RAF; a fast tap during
// the async handoff (disableAR → session start, or session end → enableAR) could
// double-acquire the camera (black screen / dangling track). This blocks every
// entry point — enableAR(), enterFloorAnchor(), the Camera button — until the prior
// acquire/release fully settles. Always cleared in a finally so a failure can't
// wedge the toggle permanently.
let _arTransitioning = false;
let arFrozenCamPos = null;
let arFrozenCamLook = null;
// Pivot for the local (no-GPS) gyro world-lock: once set, the camera holds this
// fixed point and only rotates with the phone, so a pinned avatar stays anchored
// in the room and is only visible when the camera points at it. null = inactive.
let gyroLockCamPos = null;
// Last rear-camera sensor dimensions (from the video track) — captured when AR
// starts and refreshed on viewport change so the FOV can be re-derived against the
// current viewport aspect rather than frozen at the orientation AR opened in.
let arTrackW = 0;
let arTrackH = 0;
// Re-derive and apply the camera vertical FOV from the live video track + current
// viewport. Safe to call any time: a no-op (just the projection-matrix refresh)
// when AR is off / no track, so the resize path can call it unconditionally.
function applyCameraFov() {
const track = mediaStream?.getVideoTracks?.()[0];
if (track) {
const s = track.getSettings?.() ?? {};
if (Number.isFinite(s.width) && s.width > 0) arTrackW = s.width;
if (Number.isFinite(s.height) && s.height > 0) arTrackH = s.height;
}
if (!arActive || !(arTrackW > 0) || !(arTrackH > 0)) return;
camera.fov = deriveVerticalFovDeg({
trackWidth: arTrackW,
trackHeight: arTrackH,
viewWidth: window.innerWidth,
viewHeight: window.innerHeight,
diagFovDeg: DEFAULT_DIAG_FOV_DEG,
});
camera.updateProjectionMatrix();
}
async function enableAR() {
// Refuse while a camera/XR handoff is still settling, while AR is already on, and
// while the immersive XR session owns the camera — acquiring getUserMedia in any
// of those states collides on the rear camera. The XR dom-overlay is
// pointer-events:none, so a tap can fall through to the Camera button mid-session;
// xrSession is a real guard case, not a theoretical one. The guard is always
// released in a finally so a permission denial or a getUserMedia error can't wedge
// the toggle.
if (_arTransitioning || arActive || xrSession) return;
_arTransitioning = true;
try {
await _enableARBody();
} finally {
_arTransitioning = false;
}
}
async function _enableARBody() {
if (!navigator.mediaDevices?.getUserMedia) {
// Designed state via the shared guidance sheet, not a toast — the 3D scene
// still works without the camera, so say so instead of just "unavailable".
setPermissionState('camera', 'unsupported');
showErrorState({
title: 'Camera not available here',
body: 'This browser can’t open the camera, so AR passthrough is off. The 3D scene still works — or reopen IRL in Chrome or Safari.',
});
return;
}
// Permission routes through the onboarding module (E1): a denial lands on the
// designed card + topbar re-request chip. The granted fast-path resolves
// immediately so AR proceeds within the same user gesture.
const camPerm = await ensurePermission('camera');
if (camPerm !== 'granted') return;
setStatus('Requesting camera…', { loading: true, sticky: true });
try {
mediaStream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: { ideal: 'environment' } },
audio: false,
});
} catch (err) {
// Clear the "Requesting camera…" progress toast, then surface a recoverable
// card (the full onboarding copy is E1's; this is the reactive recovery).
setStatus(null);
if (err?.name === 'NotAllowedError') {
setPermissionState('camera', 'denied');
showErrorState({
title: 'Camera access needed for AR',
body: 'Allow camera access in your browser settings, then tap Try again to drop your agent into the real world.',
actionLabel: 'Try again',
onAction: enableAR,
});
} else {
showErrorState({
title: 'Couldn’t start the camera',
body: `The camera didn’t start (${err?.message ?? err}). Close any other app using it, then tap Try again.`,
actionLabel: 'Try again',
onAction: enableAR,
});
}
return;
}
videoEl.srcObject = mediaStream;
// Reveal the AR state BEFORE play(). A display:none <video> can leave
// play()'s promise unsettled in some browsers (notably iOS Safari), which
// would otherwise hang camera activation forever. Make it visible first,
// then start playback without blocking on the promise.
arActive = true;
document.body.classList.add('is-ar');
cameraBtn.classList.add('is-active');
cameraBtn.setAttribute('aria-pressed', 'true');
cameraLabel.textContent = 'Camera On';
setSubtitle(SUBTITLE.aiming);
groundOpaque.visible = false;
groundShadow.visible = true;
renderer.setClearColor(0x000000, 0);
scene.background = null;
videoEl.play().catch(() => {/* autoplay policies — frames still arrive via the stream */});
// Match Three.js FOV to device rear camera so the avatar's scale agrees with
// real-world objects. The sensor dimensions are captured so the resize /
// orientation path can re-derive the FOV when the viewport rotates — the
// derivation is no longer a one-shot frozen at portrait (task-02).
applyCameraFov();
// Freeze camera so the avatar walks through the real room instead of the
// follow-camera chasing it (which would shift the real-world background).
arFrozenCamPos = camera.position.clone();
arFrozenCamLook = camLookCurrent.clone();
setStatus('Camera on. Aim at a spot, then tap Pin here');
}
function disableAR() {
if (mediaStream) {
mediaStream.getTracks().forEach(t => { try { t.stop(); } catch {} });
mediaStream = null;
}
videoEl.srcObject = null;
arActive = false;
document.body.classList.remove('is-ar');
cameraBtn.classList.remove('is-active');
cameraBtn.setAttribute('aria-pressed', 'false');
cameraLabel.textContent = 'Camera AR';
setSubtitle(SUBTITLE.cameraOff);
groundOpaque.visible = true;
groundShadow.visible = false;
camera.fov = 50;
camera.updateProjectionMatrix();
ambientLight.intensity = 0.55;
sun.intensity = 1.4;
arFrozenCamPos = null;
arFrozenCamLook = null;
gyroLockCamPos = null;
if (carryModeActive) setCarryMode(false); // carry only makes sense with the camera on
_carryActive = false;
arTrackW = 0;
arTrackH = 0;
setStatus('Camera off');
}
cameraBtn.addEventListener('click', () => {
// Unlock the arrival-cue chime inside this real gesture (same tap that grants
// camera + motion), so it can play later without an autoplay-policy warning.
unlockArrivalAudio();
// Ignore taps mid-handoff: enableAR/disableAR own the rear camera and the RAF, so
// a tap landing between release and re-acquire would collide; and the XR session
// owns the camera while it runs. enableAR() re-checks the guard too — this just
// keeps the toggle from flipping state under a double-tap.
if (_arTransitioning || xrSession) return;
if (arActive) disableAR();
else enableAR();
});
// Disable hero button if camera API is absent
if (!navigator.mediaDevices?.getUserMedia) {
cameraBtn.disabled = true;
cameraBtn.setAttribute('aria-disabled', 'true');
cameraLabel.textContent = 'Camera unavailable';
}
// ── Placed objects ────────────────────────────────────────────────────────
let placeModeActive = false;
let selectedType = 'orb';
// Carry-and-place: while armed, a floor tap sets the agent down at that world
// point and it eases there (no teleport). _carryTarget/_carryActive drive the glide
// in tick(); the gyro world-lock then holds the agent at the chosen spot.
let carryModeActive = false;
let _carryActive = false;
const _carryTarget = new Vector3();
const placedObjects = []; // { mesh, spawnT, type }
// ── Session persistence (localStorage) ───────────────────────────────────
const SESSION_KEY = 'irl_session_v1';
// Snapshot the persisted session ONCE at module load. loadAvatar() runs
// _saveSession() against the empty boot scene before _restoreSession() gets to
// read it, so reading localStorage live inside restore would see that wiped
// state. Capture the real saved session here instead.
const _savedSession = (() => {
try {
const raw = localStorage.getItem(SESSION_KEY);
return raw ? JSON.parse(raw) : null;
} catch { return null; }
})();
function _saveSession() {
try {
localStorage.setItem(SESSION_KEY, JSON.stringify({
avatarId: _currentAvatarId,
locked: avatarLocked,
placedObjects: placedObjects.map(o => ({
type: o.type,
x: o.mesh.position.x,
z: o.mesh.position.z,
})),
}));
} catch {}
}
function _restoreSession() {
const s = _savedSession;
if (!s) return;
try {
// Objects first, then lock — so setLocked()'s own save (and the explicit
// one below) capture the fully restored scene rather than an empty one.
for (const o of (s.placedObjects ?? [])) {
const def = OBJ_DEFS[o.type];
if (!def) continue;
const mesh = def.create();
mesh.position.x = Number(o.x) || 0;
mesh.position.z = Number(o.z) || 0;
mesh.scale.setScalar(1); // skip spawn animation on restore
scene.add(mesh);
placedObjects.push({ mesh, spawnT: SPAWN_DURATION, type: o.type });
}
if (placedObjects.length) clearBtn.hidden = false;
if (s.locked) setLocked(true);
// loadAvatar() persisted the empty boot scene before we got here; write
// the restored state back so it survives a refresh with no further edits.
_saveSession();
} catch {}
}
const OBJ_DEFS = {
orb: {
create() {
const m = new Mesh(
new SphereGeometry(0.22, 32, 24),
new MeshPhysicalMaterial({
color: 0x88bbff,
emissive: 0x1a44aa,
emissiveIntensity: 0.55,
metalness: 0.15,
roughness: 0.04,
transmission: 0.45,
thickness: 0.3,
}),
);
m.castShadow = true;
m.position.y = 0.22;
return m;
},
},
crate: {
create() {
const m = new Mesh(
new BoxGeometry(0.4, 0.4, 0.4),
new MeshStandardMaterial({ color: 0xb07830, roughness: 0.85, metalness: 0.0 }),
);
m.castShadow = true;
m.position.y = 0.2;
return m;
},
},
crystal: {
create() {
const m = new Mesh(
new OctahedronGeometry(0.28, 0),
new MeshPhysicalMaterial({
color: 0xcc77ff,
emissive: 0x440066,
emissiveIntensity: 0.5,
metalness: 0.0,
roughness: 0.0,
transmission: 0.65,
thickness: 0.4,
}),
);
m.castShadow = true;
m.position.y = 0.28;
return m;
},
},
ring: {
create() {
const m = new Mesh(
new TorusGeometry(0.22, 0.065, 16, 48),
new MeshStandardMaterial({ color: 0xffd700, metalness: 0.95, roughness: 0.08 }),
);
m.castShadow = true;
m.rotation.x = Math.PI / 2;
m.position.y = 0.065;
return m;
},
},
pillar: {
create() {
const m = new Mesh(
new CylinderGeometry(0.1, 0.12, 0.9, 12),
new MeshStandardMaterial({ color: 0xd8d0c4, roughness: 0.72, metalness: 0.0 }),
);
m.castShadow = true;
m.position.y = 0.45;
return m;
},
},
};
// Object picker wiring
document.querySelectorAll('.irl-obj-btn').forEach(btn => {
btn.addEventListener('click', () => {
selectedType = btn.dataset.type;
document.querySelectorAll('.irl-obj-btn').forEach(b => b.classList.remove('active'));
btn.classList.add('active');
});
});
document.querySelector('.irl-obj-btn[data-type="orb"]')?.classList.add('active');
placeBtn.addEventListener('click', () => {
placeModeActive = !placeModeActive;
if (placeModeActive) setCarryMode(false); // the two floor-tap modes are exclusive
placeBtn.setAttribute('aria-pressed', String(placeModeActive));
placeBtn.classList.toggle('is-active', placeModeActive);
pickerEl.hidden = !placeModeActive;
if (placeModeActive) {
canvas.style.cursor = 'crosshair';
setStatus('Tap the floor to drop an object');
} else {
canvas.style.cursor = '';
setStatus(null);
}
});
// ── Carry-and-place ("Move here") ─────────────────────────────────────────────
// Arm a floor-aim reticle; a tap sets the agent down at that real-world spot. This
// is what makes "carry my agent across the room and put it on the couch" possible —
// before this the agent was frozen 3.6 m in front of the camera (CAM_OFFSET).
function setCarryMode(on) {
carryModeActive = on && arActive;
carryBtn?.setAttribute('aria-pressed', String(carryModeActive));
carryBtn?.classList.toggle('is-active', carryModeActive);
canvas.style.cursor = carryModeActive ? 'crosshair' : '';
if (carryModeActive) {
if (placeModeActive) placeBtn.click(); // exclusive with the object-drop mode
setSubtitle(SUBTITLE.placing);
setStatus('Aim the ring at the floor, then tap to set your agent down');
} else {
carryReticle.visible = false;
setSubtitle(avatarLocked ? SUBTITLE.pinned : SUBTITLE.aiming);
if (!placeModeActive) setStatus(null);
}
}
// Set the agent down at a chosen floor point and turn it to face the viewer, so a
// freshly placed agent looks at you rather than away. The glide + final snap run in
// tick() via _carryActive; the gyro world-lock holds the spot once you Pin here.
function placeAvatarAt(point) {
_carryTarget.set(point.x, 0, point.z);
_carryActive = true;
const dx = camera.position.x - point.x, dz = camera.position.z - point.z;
if (dx * dx + dz * dz > 1e-4) avatarYaw = Math.atan2(dx, dz);
carryReticle.scale.setScalar(1.4); // confirm pulse — eased back to rest in tick()
setStatus('Agent set down. Tap Pin here to anchor it in the room');
_saveSession();
}
if (carryBtn) {
carryBtn.addEventListener('click', async () => {
if (!arActive) {
setStatus('Turn on Camera AR first, then move your agent into the room.', { warn: true });
return;
}
// Arming carry requires the world-lock: only the locked/gyro camera tracks the
// room as you pan, so the floor reticle lands on the real floor and the agent
// stays put where you set it. Engage it first if you haven't pinned yet.
if (!carryModeActive && !avatarLocked) await setLocked(true);
setCarryMode(!carryModeActive);
});
}
clearBtn.addEventListener('click', () => {
for (const obj of placedObjects) scene.remove(obj.mesh);
placedObjects.length = 0;
clearBtn.hidden = true;
setStatus('Objects cleared');
_saveSession();
});
// ── Tap-to-place raycasting ───────────────────────────────────────────────
const raycaster = new Raycaster();
const pointerNDC = new Vector2();
let tapDownX = 0, tapDownY = 0;
canvas.addEventListener('pointerdown', e => {
// The WebXR floor-anchor session (and its failed-start error card) layers a modal
// overlay over the canvas; don't arm a long-press behind it.
if (xrOverlay && !xrOverlay.hidden) return;
// A camera-mode pinch owns both fingers — the second finger's pointerdown is
// never the start of a long-press.
if (camPinchBlocking()) return;
tapDownX = e.clientX; tapDownY = e.clientY; armLongPress(e);
});
canvas.addEventListener('pointerup', e => {
cancelLongPress();
// While the WebXR floor-anchor overlay is up, the immersive session owns taps —
// its 'select' places the anchor — and on a failed start the error card is modal.
// Either way the 2D pin/place raycast behind it must not also fire (dead taps,
// stray sheets). The overlay is display:none whenever AR/gyro mode is active, so
// this only gates during an actual XR session or its error state.
if (xrOverlay && !xrOverlay.hidden) return;
// A pinch (or a finger lifting right after one) is never a tap — mirrors the
// WebXR session's own _handleBeforeSelect debounce.
if (camPinchBlocking()) return;
// While calibrating, taps belong to the nudge gesture — never re-open a sheet.
if (calibrateActive) return;
if (Math.hypot(e.clientX - tapDownX, e.clientY - tapDownY) > TAP_THRESHOLD) return;
pointerNDC.set(
(e.clientX / window.innerWidth) * 2 - 1,
-(e.clientY / window.innerHeight) * 2 + 1,
);
raycaster.setFromCamera(pointerNDC, camera);
if (carryModeActive) {
// ── Carry mode: set the agent down on the floor where you tapped ──
const hits = raycaster.intersectObject(rayPlane);
if (!hits.length) return;
placeAvatarAt(hits[0].point);
return;
}
if (placeModeActive) {
// ── Place mode: drop an object on the ground ─────────────────────
const hits = raycaster.intersectObject(rayPlane);
if (!hits.length) return;
const def = OBJ_DEFS[selectedType];
if (!def) return;
const mesh = def.create();
mesh.position.x = hits[0].point.x;
mesh.position.z = hits[0].point.z;
mesh.scale.setScalar(0.001);
scene.add(mesh);
placedObjects.push({ mesh, spawnT: 0, type: selectedType });
clearBtn.hidden = false;
_saveSession();
} else {
// ── Tap on a nearby agent: mesh body first, 2D label net as fallback ──
// Mesh ray: the nearest intersection is closest to the camera, so two
// agents projecting to the same pixel resolve to the FRONT body for free.
let pin = null;
const agentGroups = nearbyPins.map(p => p.group).filter(Boolean);
if (agentGroups.length) {
const hits = raycaster.intersectObjects(agentGroups, true);
if (hits.length) pin = _pinForObject(hits[0].object);
}
// Label net: a finger-sized slop around each on-screen name label catches
// taps that miss the (often small or distant) body — the floor for tap
// reliability on a phone. Front agent wins a cluster tie (see helper).
if (!pin) pin = _nearestLabelWithinSlop(e.clientX, e.clientY);
if (pin) openPinSheet(pin);
// A clean tap on empty space hits no agent — the canonical immersive
// gesture: toggle the chrome so you can clear the view (or bring it back)
// without hunting for the eye button.
else toggleImmersive();
}
});
// Nearest on-screen name label within a finger-sized radius of the tap (B4).
// Uses the screen positions cached by updateLabels() each frame, so it costs a
// short loop with no projection. The ranking (clear pixel winner takes it; cluster
// ties resolve to the nearer/front agent) lives in the pure, unit-tested
// pickLabelHit helper — here we only adapt each pin into a candidate it understands.
function _nearestLabelWithinSlop(px, py) {
const candidates = [];
for (const pin of nearbyPins) {
if (!pin._labelOnScreen) continue;
candidates.push({ sx: pin._labelSx, sy: pin._labelSy, distance: pin.distance_m, pin });
}
return pickLabelHit(candidates, px, py)?.pin ?? null;
}
function _pinForObject(obj) {
// Walk up to the agent group and read the pin cached on it (set in
// spawnNearbyPin). O(depth) with no per-node array scan.
let node = obj;
while (node) {
if (node.userData && node.userData.pin) return node.userData.pin;
node = node.parent;
}
return null;
}
// ── Joystick ──────────────────────────────────────────────────────────────
const input = {
joy: { x: 0, y: 0, active: false },
keys: { forward: 0, back: 0, left: 0, right: 0, run: false },
};
// nipplejs's .on() returns undefined (not chainable), so attach each listener
// on the stored manager — chaining would throw and abort the module's boot.
const joystick = nipplejs.create({
zone: joystickEl, mode: 'static', position: { left: '50%', top: '50%' },
size: 110, color: 'rgba(255,255,255,0.85)', restOpacity: 0.6,
});
joystick.on('move', evt => {
const v = evt?.data?.vector;
if (v) { input.joy.x = v.x; input.joy.y = v.y; input.joy.active = Math.hypot(v.x, v.y) > 0.05; }
});
joystick.on('end', () => { input.joy.x = 0; input.joy.y = 0; input.joy.active = false; });
// Don't drive the avatar while the keyboard belongs to something else: a text
// field (typing a caption / message) or an open modal sheet (its own focus trap
// owns Tab/Escape). Otherwise WASD/arrows would both type and walk.
function movementKeysCaptured() {
const a = document.activeElement;
if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA' || a.isContentEditable)) return true;
return !!document.querySelector('.is-open[aria-modal="true"]');
}
window.addEventListener('keydown', e => {
if (movementKeysCaptured()) return;
switch (e.code) {
case 'KeyW': case 'ArrowUp': input.keys.forward = 1; break;
case 'KeyS': case 'ArrowDown': input.keys.back = 1; break;
case 'KeyA': case 'ArrowLeft': input.keys.left = 1; break;
case 'KeyD': case 'ArrowRight': input.keys.right = 1; break;
case 'ShiftLeft': case 'ShiftRight': input.keys.run = true; break;
}
});
window.addEventListener('keyup', e => {
switch (e.code) {
case 'KeyW': case 'ArrowUp': input.keys.forward = 0; break;
case 'KeyS': case 'ArrowDown': input.keys.back = 0; break;
case 'KeyA': case 'ArrowLeft': input.keys.left = 0; break;
case 'KeyD': case 'ArrowRight': input.keys.right = 0; break;
case 'ShiftLeft': case 'ShiftRight': input.keys.run = false; break;
}
});
// ── Pinch-to-resize (camera-mode AR, no WebXR) ────────────────────────────
// The WebXR floor-anchor lane has had two-finger pinch-resize since its own
// session controller (src/ar/webxr.js) shipped; plain camera-mode AR (no
// device support for immersive-ar, or the user never opened the floor-anchor
// flow) never got the gesture. Same pure state machine, but there's no XR
// content group to scale here — it drives avatarRig directly, since the
// owner's own AR avatar renders through avatarRig, not as one of nearbyPins
// (savePin's local pin literal never spawns a group for the owner's own pin).
// A short post-pinch debounce (mirroring WebXRSession's _handleBeforeSelect)
// keeps the second finger's lift from misfiring drag-to-orbit or a tap.
const _camPinch = createPinchState();
let _camScale = 1;
let _camPinchEndedAt = -Infinity;
function camPinchBlocking() {
return _camPinch.active || performance.now() - _camPinchEndedAt < 350;
}
canvas.addEventListener('touchstart', e => {
if (!arActive || xrSession || calibrateActive || placeModeActive) return;
if (e.touches.length !== 2) return;
pinchStart(_camPinch, touchDist(e.touches), avatarRig.scale.x);
}, { passive: true });
canvas.addEventListener('touchmove', e => {
if (!_camPinch.active || e.touches.length !== 2) return;
const s = pinchMove(_camPinch, touchDist(e.touches));
if (s == null) return;
_camScale = s;
avatarRig.scale.setScalar(s);
setStatus(`Size ${Math.round(s * 100)}%. Pinch to resize`);
}, { passive: true });
canvas.addEventListener('touchend', () => {
const s = pinchEnd(_camPinch);
if (s == null) return;
_camPinchEndedAt = performance.now();
_camScale = s;
if (!gpsPin?.id) {
setStatus(`Size ${Math.round(s * 100)}%. Tap Pin here to save your agent at this size`);
return;
}
fetch('/api/irl/pins', {
method: 'PATCH', credentials: 'include',
headers: deviceHeaders({ 'Content-Type': 'application/json' }),
body: JSON.stringify({ id: gpsPin.id, deviceToken: _deviceToken, scale: s }),
}).then(r => {
if (!r.ok) { setStatus('Size couldn’t be saved, pinch again to retry', { error: true }); return; }
setStatus(`Saved at ${Math.round(s * 100)}% size. Nearby viewers see it this big too`);
}).catch(() => setStatus('Size couldn’t be saved (offline?), pinch again to retry', { error: true }));
}, { passive: true });
// ── Drag-to-orbit (disabled in place mode so taps don't orbit) ───────────
{
let dragging = false, lastX = 0, lastY = 0, downId = -1;
canvas.addEventListener('pointerdown', e => {
if (placeModeActive || calibrateActive || camPinchBlocking()) return;
// Don't orbit the (paused, hidden) IRL camera behind the WebXR overlay.
if (xrOverlay && !xrOverlay.hidden) return;
const r = joystickEl.getBoundingClientRect();
if (e.clientX >= r.left && e.clientX <= r.right && e.clientY >= r.top && e.clientY <= r.bottom) return;
dragging = true; downId = e.pointerId; lastX = e.clientX; lastY = e.clientY;
canvas.setPointerCapture?.(e.pointerId);
});
const onMove = e => {
// Calibration may engage mid-drag (long-press fires while still held) — yield