From e1f998e5cf6256e69bb09a53b8264b87e4db5377 Mon Sep 17 00:00:00 2001 From: Saad Abdali Date: Wed, 20 May 2026 20:52:51 -0400 Subject: [PATCH 1/3] Feature: ability to store cross-streets, and use voice for cross-street recognition --- client/src/Api.js | 17 +- client/src/components/AudioRecorder.jsx | 12 +- .../components/BetweenIntersectionsHint.jsx | 84 +++ .../src/components/LocationAutocomplete.jsx | 162 ++++++ client/src/components/LocationVoiceButton.jsx | 182 ++++++ .../facilityAddressLink/mapLinkUtils.js | 13 + client/src/lesc/components/Holds.jsx | 4 + client/src/lesc/components/Incident.jsx | 8 +- client/src/lesc/components/IncidentForm.jsx | 213 +++++-- .../custody/CustodyDetailContent.jsx | 8 +- .../custody/CustodyDetailContent.test.js | 7 + client/src/utils/format.js | 30 + client/src/utils/validators.js | 36 +- docs/cross-streets-implementation-plan.md | 416 +++++++++++++ docs/cross-streets-research.md | 492 ++++++++++++++++ docs/cross-streets-spot-fixes.md | 178 ++++++ docs/cross-streets.md | 548 ++++++++++++++++++ package-lock.json | 410 +++++++++++++ server/lib/forms/849b/generate.js | 6 +- server/lib/forms/shared/formUtils.js | 30 + server/lib/hospitalCancellation647f.js | 6 +- server/lib/incidentPermissions.js | 17 +- server/lib/intersections.js | 259 +++++++++ server/lib/phoneticMatch.js | 99 ++++ server/lib/streetNormalization.js | 176 ++++++ server/models/incident.js | 4 + server/package.json | 1 + server/prisma/ERD.md | 12 + server/prisma/schema.prisma | 9 + server/routes/api/ai/transcribe.js | 54 ++ server/routes/api/arrests.js | 7 +- server/routes/api/geocode/intersections.js | 73 +++ server/scripts/bench-datasf-intersections.js | 156 +++++ server/scripts/bench-intersections.js | 246 ++++++++ server/scripts/build-intersection-data.js | 159 +++++ server/test/lib/intersections.test.js | 121 ++++ server/test/lib/phoneticMatch.test.js | 55 ++ server/test/lib/streetNormalization.test.js | 157 +++++ .../routes/api/geocode-intersections.test.js | 103 ++++ 39 files changed, 4483 insertions(+), 87 deletions(-) create mode 100644 client/src/components/BetweenIntersectionsHint.jsx create mode 100644 client/src/components/LocationAutocomplete.jsx create mode 100644 client/src/components/LocationVoiceButton.jsx create mode 100644 docs/cross-streets-implementation-plan.md create mode 100644 docs/cross-streets-research.md create mode 100644 docs/cross-streets-spot-fixes.md create mode 100644 docs/cross-streets.md create mode 100644 server/lib/intersections.js create mode 100644 server/lib/phoneticMatch.js create mode 100644 server/lib/streetNormalization.js create mode 100644 server/routes/api/geocode/intersections.js create mode 100644 server/scripts/bench-datasf-intersections.js create mode 100644 server/scripts/bench-intersections.js create mode 100644 server/scripts/build-intersection-data.js create mode 100644 server/test/lib/intersections.test.js create mode 100644 server/test/lib/phoneticMatch.test.js create mode 100644 server/test/lib/streetNormalization.test.js create mode 100644 server/test/routes/api/geocode-intersections.test.js diff --git a/client/src/Api.js b/client/src/Api.js index 217e5e19..ac023ff4 100644 --- a/client/src/Api.js +++ b/client/src/Api.js @@ -442,6 +442,17 @@ const Api = { params: { latitude, longitude } }).catch(handleError); }, + intersections (text, { signal, limit } = {}) { + return instance.get('/api/geocode/intersections', { + params: { text, limit }, + signal, + }); + }, + intersectionsNearest (latitude, longitude, { n } = {}) { + return instance.get('/api/geocode/intersections/nearest', { + params: { latitude, longitude, n } + }).catch(handleError); + }, }, organizations: { index (page = 1) { @@ -503,8 +514,10 @@ const Api = { }, }, ai: { - transcribe (audio, mediaType) { - return instance.post('/api/ai/transcribe', { audio, mediaType }); + transcribe (audio, mediaType, { mode } = {}) { + return instance.post('/api/ai/transcribe', { audio, mediaType }, { + params: mode ? { mode } : undefined, + }); }, parseId (image, mediaType) { return instance.post('/api/ai/parse-id', { image, mediaType }); diff --git a/client/src/components/AudioRecorder.jsx b/client/src/components/AudioRecorder.jsx index 9ff2094d..c5398858 100644 --- a/client/src/components/AudioRecorder.jsx +++ b/client/src/components/AudioRecorder.jsx @@ -41,7 +41,7 @@ function formatTimer (seconds) { return `${m}:${s.toString().padStart(2, '0')}`; } -function AudioRecorder ({ onResult, onBusyChange, disabled }) { +function AudioRecorder ({ onResult, onBusyChange, disabled, mode = 'narrative', recordLabel }) { const mediaRecorderRef = useRef(null); const chunksRef = useRef([]); const timerRef = useRef(null); @@ -118,13 +118,17 @@ function AudioRecorder ({ onResult, onBusyChange, disabled }) { const pcmData = downsampleToInt16(audioBuffer); const base64 = int16ToBase64(pcmData); - const response = await Api.ai.transcribe(base64, 'audio/pcm'); + const response = await Api.ai.transcribe(base64, 'audio/pcm', { mode }); const text = (response.data.text || '').trim(); if (!text) { setError('No speech detected. Try again.'); return; } - onResult(text); + if (mode === 'location') { + onResult(response.data); + } else { + onResult(text); + } setError(null); } catch { setError('Transcription failed. Try again.'); @@ -189,7 +193,7 @@ function AudioRecorder ({ onResult, onBusyChange, disabled }) { onClick={startRecording} disabled={disabled} > - Record voice + {recordLabel ?? 'Record voice'} )} diff --git a/client/src/components/BetweenIntersectionsHint.jsx b/client/src/components/BetweenIntersectionsHint.jsx new file mode 100644 index 00000000..95f43062 --- /dev/null +++ b/client/src/components/BetweenIntersectionsHint.jsx @@ -0,0 +1,84 @@ +import { useEffect, useState } from 'react'; +import { Text } from '@mantine/core'; + +import Api from '@/Api'; + +/** + * Renders "between {Street1} and {Street2}" below an address, sourced from the + * two nearest SF intersection nodes per /api/geocode/intersections/nearest. + * + * Renders nothing when: + * - locationType is INTERSECTION (the intersection itself is the context) + * - lat/lng is missing (older records, manual entries) + * - the lookup fails or returns no useful pair + */ +function BetweenIntersectionsHint ({ record }) { + const [pair, setPair] = useState(null); + + useEffect(() => { + setPair(null); + if (!record) return; + if (record.locationType === 'INTERSECTION') return; + const lat = Number(record.latitude); + const lng = Number(record.longitude); + if (!Number.isFinite(lat) || !Number.isFinite(lng)) return; + + let cancelled = false; + Api.geocode.intersectionsNearest(lat, lng, { n: 3 }) + .then((response) => { + if (cancelled) return; + const rows = response?.data ?? []; + // Pick the first two intersections that share at least one street with each + // other — that's the "X between Y and Z" pattern. Fall back to first two if + // none share. + const matched = pickPair(rows); + if (matched) setPair(matched); + }) + .catch(() => { /* silent — between context is best-effort */ }); + return () => { cancelled = true; }; + }, [record?.latitude, record?.longitude, record?.locationType]); + + if (!pair) return null; + return ( + + between {pair.cross1} and {pair.cross2} + + ); +} + +function pickPair (rows) { + if (!rows || rows.length < 2) return null; + // The interesting case: two adjacent nodes that share one street (we want + // the cross-street name from each, NOT the shared street). Find a pair + // sharing a street. + for (let i = 0; i < rows.length; i++) { + for (let j = i + 1; j < rows.length; j++) { + const a = rows[i]; + const b = rows[j]; + const aStreets = [a.street1, a.street2]; + const bStreets = [b.street1, b.street2]; + const shared = aStreets.find(s => bStreets.includes(s)); + if (shared) { + const cross1 = aStreets.find(s => s !== shared); + const cross2 = bStreets.find(s => s !== shared); + if (cross1 && cross2 && cross1 !== cross2) { + return { + cross1: titleCase(cross1), + cross2: titleCase(cross2), + }; + } + } + } + } + return null; +} + +function titleCase (raw) { + const stripped = String(raw).replace(/\b0(\d(ST|ND|RD|TH))\b/gi, '$1'); + return stripped + .split(/\s+/) + .map(w => (w ? w[0].toUpperCase() + w.slice(1).toLowerCase() : w)) + .join(' '); +} + +export default BetweenIntersectionsHint; diff --git a/client/src/components/LocationAutocomplete.jsx b/client/src/components/LocationAutocomplete.jsx new file mode 100644 index 00000000..0360d65a --- /dev/null +++ b/client/src/components/LocationAutocomplete.jsx @@ -0,0 +1,162 @@ +import { forwardRef, useEffect, useRef, useState } from 'react'; +import { Autocomplete, Loader } from '@mantine/core'; +import { useDebouncedValue } from '@mantine/hooks'; + +import Api from '@/Api'; + +/** + * Unified address-or-intersection autocomplete for the incident Location field. + * + * Detects intersection-style input (e.g. "16th & Valencia", "Mission and 24th") + * from a connector token and switches the backend call between + * /api/geocode/search (address typeahead, AWS Location) + * /api/geocode/intersections (DataSF intersection index) + * + * On select: + * - address → form.setValues({ locationType:'ADDRESS', addressLine1, city, state, postalCode, latitude, longitude }) + * + clears street1/street2/intersectionId + * - intersection → form.setValues({ locationType:'INTERSECTION', street1, street2, intersectionId, latitude, longitude, city:'San Francisco', state:'CA' }) + * + clears addressLine1/addressLine2/postalCode + */ + +// Match a connector token between two non-empty sides. +const INTERSECTION_RE = /[&/@\\|]|\b(and|at)\b/i; + +function detectIntersectionMode (text) { + if (!text || text.length < 3) return false; + // Numeric leading digit + non-ordinal letter looks like an address ("425 Mission St") + if (/^\d+\s+[A-Za-z]/.test(text) && !/^\d+(st|nd|rd|th)\b/i.test(text)) return false; + return INTERSECTION_RE.test(text); +} + +const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, rightSection, ...props }, ref) { + const formValues = form.getValues(); + const initialDisplay = formValues.locationType === 'INTERSECTION' + ? [formValues.street1, formValues.street2].filter(Boolean).join(' & ') + : (formValues.addressLine1 ?? ''); + const [value, setValue] = useState(initialDisplay); + const [data, setData] = useState([]); + const [loading, setLoading] = useState(false); + const resultsRef = useRef({ mode: null, items: [] }); + const abortControllerRef = useRef(null); + const [debounced] = useDebouncedValue(value, 300); + + useEffect(() => { + if (!debounced || debounced.length < 3) { + setData([]); + resultsRef.current = { mode: null, items: [] }; + return; + } + + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + const controller = new AbortController(); + abortControllerRef.current = controller; + setLoading(true); + + const intersectionMode = detectIntersectionMode(debounced); + const apiCall = intersectionMode + ? Api.geocode.intersections(debounced, { signal: controller.signal }) + : Api.geocode.search(debounced, { signal: controller.signal }); + + apiCall + .then((response) => { + if (controller.signal.aborted) return; + const items = response?.data ?? []; + resultsRef.current = { mode: intersectionMode ? 'INTERSECTION' : 'ADDRESS', items }; + if (intersectionMode) { + setData(items + .filter((r) => r.cnn) + .map((r) => ({ value: r.cnn, label: `⌗ ${r.label}` }))); + } else { + setData(items + .filter((r) => r.placeId && r.addressLine1) + .map((r) => ({ + value: r.placeId, + label: r.neighborhood ? `📍 ${r.addressLine1} (${r.neighborhood})` : `📍 ${r.addressLine1}`, + }))); + } + }) + .catch((err) => { + if (controller.signal.aborted) return; + console.error('Location search failed:', err); + setData([]); + resultsRef.current = { mode: null, items: [] }; + }) + .finally(() => { + if (!controller.signal.aborted) setLoading(false); + }); + + return () => controller.abort(); + }, [debounced]); + + function handleChange (val) { + setValue(val); + } + + function handleFocus (e) { + e.target.select(); + } + + function handleBlur () { + // Free-text save: only persist into the form if we know the current mode. + // For unmatched text, leave the form's current addressLine1 alone — the + // explicit suggestion-pick is the path that writes through to the form. + } + + function handleOptionSubmit (key) { + const { mode, items } = resultsRef.current; + const match = items.find((r) => (mode === 'INTERSECTION' ? r.cnn : r.placeId) === key); + if (!match) return; + + if (mode === 'INTERSECTION') { + const label = `${match.street1Display} & ${match.street2Display}`; + setValue(label); + form.setValues({ + locationType: 'INTERSECTION', + street1: match.street1Display, + street2: match.street2Display, + intersectionId: match.cnn, + city: 'San Francisco', + state: 'CA', + latitude: match.latitude, + longitude: match.longitude, + addressLine1: null, + addressLine2: null, + postalCode: null, + }); + } else { + setValue(match.addressLine1); + form.setValues({ + locationType: 'ADDRESS', + addressLine1: match.addressLine1, + city: match.city ?? '', + state: match.state ?? '', + postalCode: match.postalCode?.split('-')[0] ?? '', + latitude: match.latitude ?? '', + longitude: match.longitude ?? '', + street1: null, + street2: null, + intersectionId: null, + }); + } + } + + return ( + options} + rightSection={loading ? : rightSection} + /> + ); +}); + +export default LocationAutocomplete; diff --git a/client/src/components/LocationVoiceButton.jsx b/client/src/components/LocationVoiceButton.jsx new file mode 100644 index 00000000..2d71967b --- /dev/null +++ b/client/src/components/LocationVoiceButton.jsx @@ -0,0 +1,182 @@ +import { useRef, useState } from 'react'; +import { ActionIcon, Box, Button, Group, Loader, Text } from '@mantine/core'; +import { IconMicrophone, IconPlayerStop, IconX } from '@tabler/icons-react'; + +import Api from '@/Api'; +import AudioWaveform from '@/components/AudioWaveform'; + +const TARGET_SAMPLE_RATE = 16000; +const MAX_RECORDING_SECONDS = 180; + +function downsampleToInt16 (audioBuffer) { + const inputData = audioBuffer.getChannelData(0); + const ratio = audioBuffer.sampleRate / TARGET_SAMPLE_RATE; + const outputLength = Math.floor(inputData.length / ratio); + const output = new Int16Array(outputLength); + for (let i = 0; i < outputLength; i++) { + const sample = inputData[Math.floor(i * ratio)]; + output[i] = Math.max(-32768, Math.min(32767, Math.floor(sample * 32767))); + } + return output; +} + +function int16ToBase64 (int16Array) { + const bytes = new Uint8Array(int16Array.buffer); + let binary = ''; + for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]); + return btoa(binary); +} + +/** + * Compact mic button for the Location field. When tapped, records audio, + * transcribes via /api/ai/transcribe?mode=location, and calls onResult with + * the structured response { text, matches, parsed }. + */ +function LocationVoiceButton ({ onResult, disabled, size = 'lg' }) { + const [recording, setRecording] = useState(false); + const [processing, setProcessing] = useState(false); + const [error, setError] = useState(null); + const [activeStream, setActiveStream] = useState(null); + const recorderRef = useRef(null); + const chunksRef = useRef([]); + + async function startRecording () { + setError(null); + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1 } }); + // eslint-disable-next-line no-undef + const recorder = new MediaRecorder(stream); + recorderRef.current = recorder; + chunksRef.current = []; + recorder.ondataavailable = (e) => { if (e.data.size > 0) chunksRef.current.push(e.data); }; + recorder.onstop = async () => { + stream.getTracks().forEach(t => t.stop()); + setActiveStream(null); + + const blob = new Blob(chunksRef.current); + await processAudio(blob); + }; + recorder.start(); + setActiveStream(stream); + setRecording(true); + + // Hard stop at max seconds. + setTimeout(() => { + if (recorderRef.current && recorderRef.current.state === 'recording') { + recorderRef.current.stop(); + setRecording(false); + } + }, MAX_RECORDING_SECONDS * 1000); + } catch { + setError('Could not access microphone.'); + } + } + + function stopRecording () { + if (recorderRef.current && recorderRef.current.state === 'recording') { + recorderRef.current.stop(); + setRecording(false); + } + } + + async function processAudio (blob) { + setProcessing(true); + try { + const arrayBuffer = await blob.arrayBuffer(); + // eslint-disable-next-line no-undef + const audioCtx = new AudioContext({ sampleRate: TARGET_SAMPLE_RATE }); + const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); + audioCtx.close(); + const pcm = downsampleToInt16(audioBuffer); + const base64 = int16ToBase64(pcm); + const response = await Api.ai.transcribe(base64, 'audio/pcm', { mode: 'location' }); + const text = (response.data.text || '').trim(); + if (!text) { + setError('No speech detected.'); + return; + } + onResult(response.data); + } catch { + setError('Transcription failed.'); + } finally { + setProcessing(false); + } + } + + if (processing) { + return ; + } + + if (recording) { + return ( + + + + + + + + + ); + } + + return ( + <> + + + + {error && {error}} + + ); +} + +/** + * Inline confirmation UI shown after voice transcription. Surfaces the heard + * text and up to N candidate matches; tapping one commits to the form via the + * provided callback. + */ +export function LocationConfirmationChip ({ result, onPick, onDismiss }) { + if (!result) return null; + const { text, matches, parsed } = result; + if (!matches || matches.length === 0) { + return ( + + + Heard: “{text}” + + + {parsed + ? 'Couldn\'t match those streets — try typing the intersection instead.' + : 'Didn\'t sound like a cross-street — try typing it instead.'} + + + + ); + } + const prompt = matches.length === 1 ? 'Tap to confirm:' : 'Did you mean:'; + return ( + + + Heard: “{text}”. {prompt} + + + {matches.slice(0, 4).map((m) => ( + + ))} + + + + + + ); +} + +export default LocationVoiceButton; diff --git a/client/src/components/facilityAddressLink/mapLinkUtils.js b/client/src/components/facilityAddressLink/mapLinkUtils.js index e9af64ad..d2f32a09 100644 --- a/client/src/components/facilityAddressLink/mapLinkUtils.js +++ b/client/src/components/facilityAddressLink/mapLinkUtils.js @@ -1,11 +1,24 @@ export function buildAddressQuery ({ + locationType, addressLine1, addressLine2, city, state, postalCode, zip, + street1, + street2, } = {}) { + if (locationType === 'INTERSECTION' && (street1 || street2)) { + const s1 = (street1 ?? '').toString().trim(); + const s2 = (street2 ?? '').toString().trim(); + const intersection = [s1, s2].filter(Boolean).join(' & '); + const cityValue = (city ?? 'San Francisco').toString().trim(); + const stateValue = (state ?? 'CA').toString().trim(); + const locality = [cityValue, stateValue].filter(Boolean).join(', '); + return [intersection, locality].filter(Boolean).join(', ') || null; + } + const hasLocalityInAddress = (value) => { const normalized = (value ?? '').toString().trim(); diff --git a/client/src/lesc/components/Holds.jsx b/client/src/lesc/components/Holds.jsx index 76b569c6..9437fa9c 100644 --- a/client/src/lesc/components/Holds.jsx +++ b/client/src/lesc/components/Holds.jsx @@ -43,11 +43,15 @@ function buildBlankIncident (facilityId) { cadNumber: null, caseNumber: null, encounteredVia: null, + locationType: 'ADDRESS', addressLine1: null, addressLine2: null, city: null, state: null, postalCode: null, + street1: null, + street2: null, + intersectionId: null, latitude: null, longitude: null, arrestedAt: DateTime.now().toISO(), diff --git a/client/src/lesc/components/Incident.jsx b/client/src/lesc/components/Incident.jsx index 21d58962..020c90c8 100644 --- a/client/src/lesc/components/Incident.jsx +++ b/client/src/lesc/components/Incident.jsx @@ -1,11 +1,15 @@ import { ActionIcon, Box, Group, Menu, Text } from '@mantine/core'; import { IconDots, IconFileExport, IconPencilMinus, IconTrash } from '@tabler/icons-react'; -import { formatSmartDateTime } from '@/utils/format'; +import { formatLocation, formatSmartDateTime } from '@/utils/format'; import { isValidIncident } from '@/utils/validators'; function Incident ({ incident, incidentId, onEditClick, onHandoffClick, onCancelClick }) { const isIncomplete = incident ? !isValidIncident(incident) : false; - const address = `${incident?.addressLine1 ?? ''}${incident?.addressLine2 ? `, ${incident.addressLine2}` : ''}`; + const address = incident + ? (incident.locationType === 'INTERSECTION' + ? formatLocation(incident) + : `${incident.addressLine1 ?? ''}${incident.addressLine2 ? `, ${incident.addressLine2}` : ''}`) + : ''; const displayId = incident?.id ?? incidentId ?? ''; const cancelLabel = (incident?.deflections?.length ?? 0) > 1 ? 'Cancel holds' : 'Cancel hold'; diff --git a/client/src/lesc/components/IncidentForm.jsx b/client/src/lesc/components/IncidentForm.jsx index 98b327f5..f5e8b6aa 100644 --- a/client/src/lesc/components/IncidentForm.jsx +++ b/client/src/lesc/components/IncidentForm.jsx @@ -20,13 +20,14 @@ import { DateTime } from 'luxon'; import { useTranslation } from 'react-i18next'; import Api from '@/Api'; -import AddressAutocomplete from '@/components/AddressAutocomplete'; import ChipInput from '@/components/ChipInput'; import Header from '@/components/Header'; import IconButtonLink from '@/components/IconButtonLink'; +import LocationAutocomplete from '@/components/LocationAutocomplete'; +import LocationVoiceButton, { LocationConfirmationChip } from '@/components/LocationVoiceButton'; import { useToast } from '@/components/ToastContext'; import { useFacilityContext } from '@/FacilityContext'; -import { formatAddress } from '@/utils/format'; +import { formatLocation } from '@/utils/format'; import { getCurrentLocationAddress } from '@/utils/geocoding'; import { validateIncident } from '@/utils/validators'; @@ -34,11 +35,15 @@ const initialValues = { cadNumber: '', caseNumber: '', encounteredVia: '', + locationType: 'ADDRESS', addressLine1: '', addressLine2: '', city: '', state: '', postalCode: '', + street1: '', + street2: '', + intersectionId: '', latitude: '', longitude: '', arrestedAt: '', @@ -52,11 +57,15 @@ function normalizeIncidentFormValues (values) { cadNumber: values?.cadNumber ?? '', caseNumber: values?.caseNumber ?? '', encounteredVia: values?.encounteredVia ?? '', + locationType: values?.locationType ?? 'ADDRESS', addressLine1: values?.addressLine1 ?? '', addressLine2: values?.addressLine2 ?? '', city: values?.city ?? '', state: values?.state ?? '', postalCode: values?.postalCode ?? '', + street1: values?.street1 ?? '', + street2: values?.street2 ?? '', + intersectionId: values?.intersectionId ?? '', latitude: values?.latitude ?? '', longitude: values?.longitude ?? '', arrestedAt: values?.arrestedAt ?? '', @@ -78,11 +87,15 @@ function buildIncidentPayload (values) { cadNumber: emptyStringToNull(values.cadNumber), caseNumber: emptyStringToNull(values.caseNumber), encounteredVia: emptyStringToNull(values.encounteredVia), + locationType: values.locationType ?? 'ADDRESS', addressLine1: emptyStringToNull(values.addressLine1), addressLine2: emptyStringToNull(values.addressLine2), city: emptyStringToNull(values.city), state: emptyStringToNull(values.state), postalCode: emptyStringToNull(values.postalCode), + street1: emptyStringToNull(values.street1), + street2: emptyStringToNull(values.street2), + intersectionId: emptyStringToNull(values.intersectionId), latitude: emptyStringToNull(values.latitude), longitude: emptyStringToNull(values.longitude), arrestedAt: arrestedAt.isValid ? arrestedAt.toISO() : null, @@ -110,6 +123,7 @@ function IncidentForm () { const { t } = useTranslation(); const [isInitialized, setInitialized] = useState(false); const [showAddressForm, setShowAddressForm] = useState(false); + const [voiceResult, setVoiceResult] = useState(null); const addressRef = useRef(); const autoGeolocationRequestedRef = useRef(false); const isEditing = !!incidentId; @@ -144,11 +158,11 @@ function IncidentForm () { setInitialized(true); // If this is a brand-new incident, and the incident doesn't already have an address, // then try using device location to fill in the address/location data - if (isConfirmIncidentFlow && !data.addressLine1 && !autoGeolocationRequestedRef.current) { + if (isConfirmIncidentFlow && !data.addressLine1 && !data.street1 && !autoGeolocationRequestedRef.current) { autoGeolocationRequestedRef.current = true; getCurrentLocationAddress().then((address) => { if (address) { - form.setValues(address); + form.setValues({ locationType: 'ADDRESS', ...address }); } }); } @@ -162,6 +176,7 @@ function IncidentForm () { .then((address) => { form.initialize({ ...initialValues, + locationType: 'ADDRESS', ...address, facilityId: facility.id, arrestedAt: now, @@ -192,11 +207,38 @@ function IncidentForm () { getCurrentLocationAddress().then((address) => { setInitialized(true); form.setValues({ + locationType: 'ADDRESS', ...address, + street1: null, + street2: null, + intersectionId: null, }); }); }; + function handleVoiceResult (data) { + // Always show the confirmation chip — even a single high-confidence match + // gets a tap-to-confirm step. Officer-entered evidence shouldn't silently + // commit; the chip is the place where voice → field becomes deliberate. + setVoiceResult(data); + } + + function applyIntersectionMatch (match) { + form.setValues({ + locationType: 'INTERSECTION', + street1: match.street1Display, + street2: match.street2Display, + intersectionId: match.cnn, + city: 'San Francisco', + state: 'CA', + latitude: match.latitude, + longitude: match.longitude, + addressLine1: null, + addressLine2: null, + postalCode: null, + }); + } + const onSubmitMutation = useMutation({ mutationFn: (formData) => isEditing @@ -260,75 +302,132 @@ function IncidentForm () { > {!showAddressForm && ( - - Location* - - } - rightSection={ - !isInitialized ? : - } - value={formatAddress(form.getValues())} - readOnly - onFocus={() => { - setShowAddressForm(true); - setTimeout(() => addressRef.current?.focus(), 100); - }} - /> + + + Location* + + } + rightSection={ + !isInitialized + ? + : ( + + + + + ) + } + rightSectionWidth={84} + value={formatLocation(form.getValues())} + readOnly + onFocus={() => { + setShowAddressForm(true); + setTimeout(() => addressRef.current?.focus(), 100); + }} + /> + { applyIntersectionMatch(m); setVoiceResult(null); }} + onDismiss={() => setVoiceResult(null)} + /> + )} {showAddressForm && ( <> - - Address line 1* + Location* } + placeholder='Address or cross-streets (e.g. "16th & Valencia")' rightSection={ - !isInitialized ? : + !isInitialized + ? + : ( + + + + + ) } + rightSectionWidth={84} /> - { applyIntersectionMatch(m); setVoiceResult(null); }} + onDismiss={() => setVoiceResult(null)} /> - - City* + + Street 1*} + /> + Street 2*} + /> + + City*} + /> - } - /> - - - State* - - } - /> - - + ) + : ( + <> + + + City* + + } + /> + + + State* + + } + /> + + + + )} )} {incidentAddress && ( - Address + {incident?.locationType === 'INTERSECTION' ? 'Cross-streets' : 'Address'} {incidentAddress} + )} {incident?.arrestedAt && ( diff --git a/client/src/lesc/components/custody/CustodyDetailContent.test.js b/client/src/lesc/components/custody/CustodyDetailContent.test.js index 419b6cd2..2b5aa833 100644 --- a/client/src/lesc/components/custody/CustodyDetailContent.test.js +++ b/client/src/lesc/components/custody/CustodyDetailContent.test.js @@ -59,6 +59,9 @@ vi.mock('@/FacilityContext', () => ({ vi.mock('@/utils/format', () => ({ formatAddress: (obj = {}) => [obj.addressLine1, obj.city].filter(Boolean).join(', '), + formatLocation: (obj = {}) => (obj.locationType === 'INTERSECTION' + ? [obj.street1, obj.street2].filter(Boolean).join(' & ') + : [obj.addressLine1, obj.city].filter(Boolean).join(', ')), formatDateTime: () => 'formatted-date-time', formatIntakeStartedAt: (date) => (date ? 'Apr 29, 11:24 AM' : null), formatTimelineTimestamp: (date) => (date ? 'timeline-timestamp' : null), @@ -105,6 +108,10 @@ vi.mock('@/components/LockedQRCode', () => ({ default: () => h('div', null, 'qr-code'), })); +vi.mock('@/components/BetweenIntersectionsHint', () => ({ + default: () => null, +})); + vi.mock('@tabler/icons-react', () => ({ IconAlertCircle: () => null, IconAlarm: () => null, diff --git a/client/src/utils/format.js b/client/src/utils/format.js index 431edde1..6c783030 100644 --- a/client/src/utils/format.js +++ b/client/src/utils/format.js @@ -8,6 +8,36 @@ export function formatAddress ({ addressLine1, addressLine2, city, state, postal return `${addressLine1 ?? ''}${addressLine2 ? `, ${addressLine2}` : ''}${city ? `, ${city}` : ''}${state ? `, ${state}` : ''}${postalCode ? ` ${postalCode}` : ''}`; } +/** + * Title-case a DataSF-style uppercase street name and strip zero-pad on + * numbered streets (e.g. "03RD ST" → "3rd St", "JUNIPERO SERRA BLVD" → + * "Junipero Serra Blvd"). Mirrors server/lib/streetNormalization.displayName. + */ +export function formatStreetName (raw) { + if (!raw) return ''; + const stripped = String(raw).replace(/\b0(\d(ST|ND|RD|TH))\b/gi, '$1'); + return stripped + .split(/\s+/) + .map(w => (w ? w[0].toUpperCase() + w.slice(1).toLowerCase() : w)) + .join(' '); +} + +/** + * Format the location of an incident (or any record carrying a locationType + * discriminator). For INTERSECTION mode renders "Street1 & Street2"; for + * ADDRESS mode delegates to formatAddress. Defaults to ADDRESS when no + * locationType is set, for backward compatibility with pre-migration records. + */ +export function formatLocation (record) { + if (!record) return ''; + if (record.locationType === 'INTERSECTION') { + const s1 = formatStreetName(record.street1); + const s2 = formatStreetName(record.street2); + return s1 && s2 ? `${s1} & ${s2}` : (s1 || s2); + } + return formatAddress(record); +} + /** * Convert date to DateTime object * @param {string|Date} date - Date to convert diff --git a/client/src/utils/validators.js b/client/src/utils/validators.js index d506386e..e85dd1f9 100644 --- a/client/src/utils/validators.js +++ b/client/src/utils/validators.js @@ -37,17 +37,32 @@ function isValidDobValue (value) { ); } -const IncidentSchema = z.object({ - addressLine1: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), - addressLine2: z.optional(z.nullable(z.string())), - city: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), - state: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), +const IncidentCommonShape = { arrestedAt: z.iso.datetime(ERROR_REQUIRED), encounteredVia: z.enum(['ON_VIEW', 'DISPATCHED'], ERROR_SELECT_ONE), cadNumber: z.string(ERROR_REQUIRED).check(z.refine((value) => hasMinimumAlphanumericChars(value, 2), ERROR_MIN_ALPHANUMERIC)), caseNumber: z.string(ERROR_REQUIRED).check(z.refine((value) => hasMinimumAlphanumericChars(value, 2), ERROR_MIN_ALPHANUMERIC)), supervisorBadgeNumber: z.string(ERROR_REQUIRED).check(z.minLength(1, ERROR_REQUIRED), z.maxLength(4, ERROR_REQUIRED)), -}); +}; + +const IncidentSchema = z.discriminatedUnion('locationType', [ + z.object({ + locationType: z.literal('ADDRESS'), + addressLine1: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), + addressLine2: z.optional(z.nullable(z.string())), + city: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), + state: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), + ...IncidentCommonShape, + }), + z.object({ + locationType: z.literal('INTERSECTION'), + street1: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), + street2: z.string(ERROR_REQUIRED).check(z.minLength(2, ERROR_REQUIRED)), + city: z.optional(z.nullable(z.string())), + intersectionId: z.optional(z.nullable(z.string())), + ...IncidentCommonShape, + }), +], ERROR_SELECT_ONE); const SEX_OPTIONS = ['MALE', 'FEMALE', 'OTHER', 'UNKNOWN']; const RACE_OPTIONS = ['WHITE', 'BLACK', 'HISPANIC', 'ASIAN', 'OTHER', 'UNKNOWN']; @@ -112,10 +127,17 @@ const DeflectionSchema = z.discriminatedUnion('drugUseEvidence', [ }), ], ERROR_SELECT_ONE); +// Records from the API or older fixtures may lack locationType; default it +// to ADDRESS so the discriminated union can branch. +function withLocationTypeDefault (obj) { + if (!obj || typeof obj !== 'object') return obj; + return obj.locationType ? obj : { ...obj, locationType: 'ADDRESS' }; +} + export const validateIncident = zod4Resolver(IncidentSchema); export const isValidIncident = (obj) => { - return !!IncidentSchema.safeParse(obj)?.success; + return !!IncidentSchema.safeParse(withLocationTypeDefault(obj))?.success; }; export const validateSubject = zod4Resolver(SubjectSchema); diff --git a/docs/cross-streets-implementation-plan.md b/docs/cross-streets-implementation-plan.md new file mode 100644 index 00000000..9984400d --- /dev/null +++ b/docs/cross-streets-implementation-plan.md @@ -0,0 +1,416 @@ +# Cross-Streets — Implementation Plan + +Companion to `docs/cross-streets.md` (investigation & design decisions). Read that first if you haven't. + +## Recap of locked-in decisions + +- **A1 — Storage**: discriminator + `street1`/`street2` columns on `Incident`, plus DataSF `cnn` as `intersectionId`. All existing rows backfilled to `locationType = 'ADDRESS'`. +- **B1 — UX**: single unified Location field, mode auto-detected from connector tokens (`&`, ` and `, ` at `, `/`, `@`). +- **C1 — Voice**: extend existing `AudioRecorder` onto the Location field, mic visible on the collapsed view. AWS Transcribe with DataSF-built Custom Vocabulary; server-side Double Metaphone fuzzy match against the SF street list. Confirmation chip before commit. +- **E — Between context**: included. After resolving an address, show "between X and Y" using a local DataSF lookup. +- **Intersection geocoding source**: DataSF `jfxm-zeee` vendored locally. **No AWS Location calls for intersections.** +- **Scope**: FIELD-role users only (today's `/incident` route gating). Incident location only — Subject/Facility address unchanged. SF-only — no multi-jurisdiction abstraction yet. + +## Phases at a glance + +| Phase | What | Depends on | +| --- | --- | --- | +| P1 | Data pipeline (vendored DataSF) | — | +| P2 | Schema migration | — | +| P3 | Server endpoints + voice route | P1, P2 | +| P4 | Client UI (LocationAutocomplete + mic) | P3 | +| P5 | Display sites (lists, details, map links, "between" context) | P2 | +| P6 | Tests (interleaved through P1–P5; called out explicitly here) | — | +| P7 | Rollout, instrumentation, follow-up gate | P4 | + +P1, P2 are independent and can start in parallel. P5 can start after P2 lands. P3 needs both. P4 follows P3. + +--- + +## Phase 1 — Data pipeline + +### Goal + +Produce a vendored, normalized JSON index of SF intersections, refreshable via script. Same data feeds the server endpoint (P3), the typeahead (P4), and the Transcribe Custom Vocabulary build (P3). + +### Files to create + +- `server/data/sf-intersections.json` — vendored output (~500 KB gzipped). Checked into the repo. +- `server/data/sf-streets.json` — derived street name list (with canonical suffix + aliases) for the Transcribe vocabulary and the phonetic matcher. +- `server/scripts/build-intersection-data.js` — fetches DataSF `jfxm-zeee` (paginated; ~21k rows), normalizes, writes both JSON files. Can be re-run quarterly. +- `server/lib/intersections.js` — load JSON at boot, expose `search(text)`, `findByCnn(cnn)`, `findNearest(lat, lng, n)`. +- `server/lib/streetNormalization.js` — pure functions: `normalizeStreet(input) → string[]` (returns candidate prefixes; see Part 9 in `cross-streets.md`), `displayName(rawName) → string` (title-case + canonical-suffix), `expandSuffix(input) → string`. + +### `sf-intersections.json` shape (proposal) + +```json +[ + { + "cnn": "24183000", + "street1": "16TH ST", + "street2": "VALENCIA ST", + "lat": 37.76491732, + "lng": -122.42188634, + "zip": "94103" + }, + ... +] +``` + +Single entry per intersection (dedupe by CNN at build time — DataSF stores both A→B and B→A orderings). Sorted by `street1` then `street2` for predictable diff in PRs. + +### `sf-streets.json` shape (proposal) + +```json +[ + { "name": "16TH ST", "display": "16th St", "base": "16th", "suffix": "St" }, + { "name": "VALENCIA ST", "display": "Valencia St", "base": "Valencia", "suffix": "St" }, + { "name": "JUNIPERO SERRA BLVD", "display": "Junipero Serra Blvd", "base": "Junipero Serra", "suffix": "Blvd" }, + { "name": "THE EMBARCADERO", "display": "The Embarcadero", "base": "Embarcadero", "suffix": null, "aliases": ["EMBARCADERO"] }, + { "name": "BAY SHORE BLVD", "display": "Bay Shore Blvd", "base": "Bay Shore", "suffix": "Blvd", "aliases": ["Bayshore"] } +] +``` + +Built by deduping all `street_name_1` values across `jfxm-zeee`. Aliases hand-curated (currently: `Embarcadero` → `THE EMBARCADERO`, `Bayshore` → `BAY SHORE BLVD`) and stored in a small `streetAliases.json` companion that the build script merges in. + +### Normalization rules (codified) + +From Part 9 findings: + +1. Long-form suffix → abbreviation: `STREET → ST`, `AVENUE → AVE`, `BOULEVARD → BLVD`, `LANE → LN`, `DRIVE → DR`, `ROAD → RD`, `WAY → WAY`, `HIGHWAY → HWY`. +2. Strip trailing suffix for prefix-match candidate. +3. Zero-pad single-digit numbered streets: `3RD → 03RD`, `7TH → 07TH`. +4. Try with and without leading `THE`. +5. Apply explicit alias map (`Bayshore → Bay Shore`). + +`normalizeStreet()` returns the union of all candidate prefixes so we can query multiple variants. + +### Acceptance + +- `node scripts/build-intersection-data.js` produces both JSON files; output is deterministic (same input → same JSON). +- `server/lib/intersections.js` loads the JSON synchronously at module import time (boot cost <100ms). +- All 30 intersections from the Part 9 bench resolve through `search()` to a correct match. +- `streetNormalization.test.js` covers each rule with a positive and negative case. + +### Open implementation questions + +- **Q-P1-a**: Where do the JSON files live — `server/data/` or a shared `data/` at repo root? Server-side use only in v1, so `server/data/` is fine. +- **Q-P1-b**: Refresh cadence — quarterly cron, manual, or CI? **Lean: manual for v1**; document the command in `CONTRIBUTING.md` (or a script comment). DataSF data changes slowly. + +--- + +## Phase 2 — Schema migration + +### Goal + +Extend `Incident` to carry intersection data alongside address data, with a discriminator and backward-compatible defaults. + +### Files to modify + +- `server/prisma/schema.prisma` (Incident model, ~line 706+): + ```prisma + model Incident { + // existing fields ... + locationType IncidentLocationType @default(ADDRESS) + street1 String? + street2 String? + intersectionId String? // DataSF CNN, when matched + } + + enum IncidentLocationType { + ADDRESS + INTERSECTION + } + ``` +- `server/models/incident.js` — extend Zod schema with the three new fields + enum. +- `client/src/utils/validators.js:40-50` — split the IncidentSchema into a discriminated union (or a conditional refinement): when `locationType = ADDRESS`, require `addressLine1`/`city`/`state`; when `INTERSECTION`, require `street1`/`street2`. `latitude`/`longitude` optional in both (best-effort). +- `client/src/utils/format.js:7-9` — `formatAddress(record)` branches: intersection → `"${title(street1)} & ${title(street2)}"`; address → existing behavior. + +### Migration + +Dev (Docker): `prisma db push --accept-data-loss` (per CLAUDE.md memory pattern). Existing rows get `locationType = 'ADDRESS'` automatically via the column default — verify with a sanity-check query post-push. + +Production: a generated migration with `ADD COLUMN locationType ... NOT NULL DEFAULT 'ADDRESS'` is safe — applies the default to existing rows in one statement. + +### Acceptance + +- Existing incidents read normally; new field defaults to `ADDRESS`. +- A new incident can be created via the API with `locationType: 'INTERSECTION'` carrying `street1`/`street2`/`intersectionId`/`latitude`/`longitude` and nullable address fields. +- Server Zod schema and client validators agree on the discriminated shape. + +### Open implementation questions + +- **Q-P2-a**: Do we ever need to clear address fields when transitioning a record from ADDRESS → INTERSECTION? **Lean: yes** — when the user changes the mode at edit time, blank the other side cleanly so we don't carry phantom data. + +--- + +## Phase 3 — Server endpoints + +### Goal + +Three server-side capabilities: (1) intersection typeahead, (2) the "between" context lookup, (3) voice transcription with location-mode post-processing. + +### Files to create + +- `server/routes/api/geocode/intersections.js` — `GET /api/geocode/intersections?text=…` and `GET /api/geocode/intersections/nearest?lat=…&lng=…&n=2`. + - The text endpoint: parse the input on connector tokens; for each side, normalize and prefix-match against `sf-streets.json`; return up to 10 candidate intersections from `sf-intersections.json` joining both sides. + - The nearest endpoint: given a lat/lng, return the N nearest intersection nodes for "between context." + - Auth: required (mirror existing `/api/geocode/search`). +- `server/lib/phoneticMatch.js` — exports `matchStreet(rawToken, streetList) → { candidate, score }[]`. Uses Double Metaphone (from `natural` npm) plus a Levenshtein tiebreak. +- `server/scripts/build-transcribe-vocabulary.js` — reads `sf-streets.json`, emits a CSV in AWS Transcribe Custom Vocabulary format. Optionally uses AWS SDK to create/update the vocab resource. Output: a vocabulary name to set in `AWS_TRANSCRIBE_VOCABULARY_NAME`. + +### Files to modify + +- `server/routes/api/ai/transcribe.js`: + - Accept an optional query param `mode` (`narrative` | `location`, default `narrative`). + - For `narrative`: existing behavior, returns the transcript string. + - For `location`: after transcription, run the result through a parse pipeline: + 1. Tokenize on `&`, ` and `, ` at `, `/`, `@`. + 2. For each side, call `phoneticMatch.matchStreet` against `sf-streets.json`, take top 3. + 3. For each (left-candidate × right-candidate) pair, look up in `sf-intersections.json`. + 4. Return `{ transcript, parsed: { side1Candidates, side2Candidates }, matches: [{ cnn, street1, street2, lat, lng, score }] }`. +- `server/lib/location.js` — no changes for v1 (AWS path stays on traditional address only). + +### Dependencies to add + +- `natural` (~2 MB; provides DoubleMetaphone, NYSIIS, JaroWinklerDistance). Server workspace only. + +### Acceptance + +- `curl '/api/geocode/intersections?text=16th%20%26%20Valencia'` returns one candidate at `(37.76491732, -122.42188634)`. +- `curl '/api/geocode/intersections?text=Junipero+%26+Monterey'` returns the Junipero Serra Blvd × Monterey Blvd match (verifies prefix-match handles a partial first street). +- `curl '/api/geocode/intersections/nearest?lat=37.7657&lng=-122.4194&n=2'` returns the two nearest intersection rows. +- `POST /api/ai/transcribe?mode=location` with a known audio file containing "Junipero Sarah and Monterey" returns `parsed.side1Candidates[0].name = 'JUNIPERO SERRA BLVD'` and `matches[0].cnn` set. (Junipero Serra fuzzy-match validation.) +- `node scripts/build-transcribe-vocabulary.js --upload` creates an AWS Transcribe Custom Vocabulary named `care-connect-sf-streets-v1`. Env var `AWS_TRANSCRIBE_VOCABULARY_NAME=care-connect-sf-streets-v1` activates it. + +### Open implementation questions + +- **Q-P3-a**: 2k-word vocabulary — submit as one resource, or split? **Lean: one** (per Part 5 Q4). Build the script to handle splitting trivially if needed later. +- **Q-P3-b**: Streaming Transcribe doesn't give us n-best. Do we fall back to batch transcribe (which does support n-best) when the streaming result's fuzzy-match score is low? **Lean: not in v1.** First try streaming + fuzzy match; measure. If accept rate is below ~80%, add the batch-retry path in P7. +- **Q-P3-c**: `intersectionId` resolution on submit — the client gets a list of `{ cnn, street1, street2, lat, lng }` candidates from the endpoint. We submit the one the user picks. **No separate resolve call needed.** Make sure the endpoint returns `cnn` so the client can pass it through to the incident write. + +--- + +## Phase 4 — Client UI + +### Goal + +A new `LocationAutocomplete` that handles both address and intersection input in one field, with the mic visible on the collapsed view. + +### Files to create + +- `client/src/components/LocationAutocomplete.jsx`: + - Accepts a Mantine form instance (same pattern as `AddressAutocomplete`). + - Internal debouncer (300ms, matching existing). + - Detects connector tokens client-side. If detected: query `/api/geocode/intersections`. Otherwise: query `/api/geocode/search` (today's address path). + - Suggestion list renders both result types with a small type chip (`⌗` intersection, `📍` address). + - On selection: dispatch a single `setValues` that writes either the address fields *or* the intersection fields (and clears the other), and sets `locationType`. Sets `intersectionId` to `cnn` when picking an intersection result. +- `client/src/components/LocationVoiceButton.jsx` — small button (Mantine `ActionIcon`) that wraps the existing `AudioRecorder` machinery but submits to `/api/ai/transcribe?mode=location`. On result, hand off to a candidate-picker callback. +- `client/src/components/LocationConfirmationChip.jsx` — inline chip displayed under the field after voice resolution: "Heard: '…'. Did you mean: [16th St & Valencia St ✓] [16th St & Mission St] [×]". Buttons trigger the same `setValues` flow as a manual selection. + +### Files to modify + +- `client/src/lesc/components/IncidentForm.jsx:262-333`: + - Replace `AddressAutocomplete` with `LocationAutocomplete` inside the expanded view. + - Collapsed view: add `LocationVoiceButton` to the `rightSection` next to the existing location icon. Mic is visible by default. + - Collapsed display: render either `formatAddress(record)` or `"${street1} & ${street2}"` based on `record.locationType`. + - Auto-fill on new incident open: when reverse-geocoding succeeds, set `locationType: 'ADDRESS'` and the address fields (today's flow). No intersection mode from device location (per the Decision-D-dropped resolution). +- `client/src/utils/format.js:7-9` — add `formatLocation(record)` that branches on `locationType`; refactor call sites in P5 to use it. +- `client/src/services/Api.js` (or wherever `Api.geocode.*` lives) — add `Api.geocode.intersections(text)` and `Api.geocode.intersectionsNearest(lat, lng, n)`. + +### UX details (compact specs) + +Collapsed view: +``` +┌─ Location ─────────────────────────────┬───┬───┐ +│ │ 🎤 │ ⌖ │ +└────────────────────────────────────────┴───┴───┘ +``` + +Tap mic → recording UI replaces the chrome inline (existing `AudioRecorder` pattern, no modal). On result → confirmation chip below: +``` +┌─ Location ─────────────────────────────┬───┬───┐ +│ 16th St & Valencia St │ 🎤 │ ⌖ │ +├────────────────────────────────────────┴───┴───┤ +│ Heard: "16th and Valencia" │ +│ Matches: [16th St & Valencia St ✓] │ +│ [16th St & Mission St] [✗ retry] │ +└────────────────────────────────────────────────┘ +``` + +Picking a match commits to the form. Pressing × clears the chip and keeps the prior value. + +Expanded view (tap the field to expand): +- When `locationType=INTERSECTION`: show `street1` + `street2` as two TextInputs, plus an optional `city` (defaulted to "San Francisco"). Hide `addressLine1`/`addressLine2`/`state`/`postalCode`. +- When `locationType=ADDRESS`: today's layout exactly. No changes. +- A small mode toggle link at the top of the expanded view: "Switch to intersection" / "Switch to address" — covers the rare case where auto-detect was wrong and the officer wants to override. + +### Acceptance + +- Type `16th & Valencia` → suggestion list shows intersection matches; selecting one populates form with `locationType=INTERSECTION`, `street1='16TH ST'`, `street2='VALENCIA ST'`, `intersectionId='24183000'`, lat/lng. +- Type `425 16th St` → behaves exactly like today: address suggestions, address fields populated. +- Tap mic, say "Junipero Sarah and Monterey" → confirmation chip shows "Junipero Serra Blvd & Monterey Blvd" as the top match; tapping commits. +- Existing incidents with `locationType=ADDRESS` render correctly in the collapsed view. +- Form submit produces a valid incident record on the server for both modes. + +### Open implementation questions + +- **Q-P4-a**: When auto-detect fires mid-typing (user types `16th &` between keystrokes), do we re-query on every keystroke or wait for the user to stop typing? **Lean: same 300ms debounce as today** — re-query the appropriate endpoint each debounce-tick. +- **Q-P4-b**: Should the mode toggle link in the expanded view be eliminated in favor of pure auto-detection? **Lean: keep it as a safety valve for v1**; remove after we observe whether it gets used. +- **Q-P4-c**: Recording UI when the field is collapsed — does the field expand to show the chip, or does the chip appear *under* the still-collapsed field? **Lean: chip under the collapsed field** (matches the user's stated preference in Q6 — don't force expansion). + +--- + +## Phase 5 — Display sites + 5150 + map link + "between" context + +### Goal + +Every place that reads the incident location needs to handle both modes. Plus the "between" context UX freebie (Decision E). + +### Files to modify + +- `client/src/lesc/components/Incident.jsx:8,23` — replace inline `addressLine1 + addressLine2` concat with `formatLocation(incident)`. +- `client/src/lesc/components/custody/CustodyDetailContent.jsx:188` — same. +- `client/src/utils/format.js` — implement `formatLocation()` (created in P4); keep `formatAddress()` for legacy call sites that genuinely want only the address part. +- `client/src/components/facilityAddressLink/` — generate a Google/Apple Maps URL for intersection mode. Google Maps accepts `?q=16th+St+%26+Valencia+St,+San+Francisco,+CA` and pins the intersection. Apple Maps similar. +- `server/lib/forms/5150/generate.js` — no change for v1 (form uses subject address, not incident location). **If** that changes later, add a branch: when `subject` has only intersection-mode incident location, render the intersection string into the "and residing at" field. Per Q3 in `cross-streets.md`, render in human-readable form (`16th St & Valencia St`). + +### "Between" context + +- New UI element under the Location field on the **incident detail page** (and possibly the collapsed Location chrome): "between Valencia St and Guerrero St." +- Fetched via `/api/geocode/intersections/nearest?lat=…&lng=…&n=2` (created in P3). Server walks `sf-intersections.json` for nodes on the same street as the address, sorted by distance. +- Only fires when `locationType=ADDRESS` and lat/lng is populated. No "between" context for intersection-mode records (the intersection itself is the context). +- Cached per incident — once resolved, store the result alongside the incident (small string, no schema change needed — could be a transient client-side computation). + +### Acceptance + +- Incident list rows render both address and intersection records correctly. +- Custody detail page shows the location in both modes. +- Google/Apple Maps deep link works for an intersection (verify by tapping). +- Address-mode incidents show a "between X and Y" line; intersection-mode incidents do not. +- 5150 PDF generation unchanged (no incident location involved). + +### Open implementation questions + +- **Q-P5-a**: "Between" — compute on client (using the same DataSF JSON if shipped to client) or server? **Lean: server.** Client doesn't need the full intersection index; the lookup is fast on the server (~10k entries, in-memory). +- **Q-P5-b**: When the address resolution has a null lat/lng (older records, or a manually-entered address), skip the "between" line silently. Easy. + +--- + +## Phase 6 — Tests + +Interleaved with each phase. Called out explicitly so nothing falls through. + +### Server + +- `server/test/lib/streetNormalization.test.js` — every normalization rule, plus negative cases (no over-normalization). +- `server/test/lib/intersections.test.js` — load fixture, search for the 30 bench intersections, expect 29/30 + 1 multi. +- `server/test/lib/phoneticMatch.test.js` — Double Metaphone matches: "Junipero Sarah" → "Junipero Serra"; "Cesar Charvez" → "Cesar Chavez"; "Embarcadiro" → "The Embarcadero". Include negative cases. +- `server/test/routes/api/geocode/intersections.test.js` — endpoint behavior, including "no match," multi-match, and prefix-match cases. +- `server/test/routes/api/ai/transcribe.test.js` — extend with a `mode=location` case using a mocked Transcribe response; verify the response shape. +- `server/test/routes/api/incidents/create.test.js` — extend to cover an intersection-mode create. + +### Client + +- `client/src/components/LocationAutocomplete.test.jsx` — connector detection, mode switching, selection writes correct fields. +- `client/src/lesc/components/IncidentForm.test.jsx` — extend to cover intersection-mode submit; mode toggle link. +- Don't bother snapshot-testing the confirmation chip rendering — covered by manual UX testing. + +### Manual / browser + +- Voice path end-to-end on a real mobile device — desktop browser will work but iOS Safari is the high-risk environment per the existing Web Speech API memory note (we're not using Web Speech, but iOS audio capture has its own quirks). +- Print a 5150 PDF for an incident whose subject lives at an intersection address — confirm the "and residing at" field still renders sensibly (no change expected; verify). + +--- + +## Phase 7 — Rollout, instrumentation, follow-up gate + +### Instrumentation (added during P3/P4) + +Log structured events (PostHog or whatever the project uses; check existing patterns): + +- `location_input_mode`: `address` | `intersection`, on every incident submit. +- `voice_transcribe`: `{ mode, raw_transcript, top_match, score, user_action: 'accepted'|'edited'|'rejected' }`, on every voice flow completion. +- `intersection_lookup_miss`: when an officer enters a cross-street that has no match in `sf-intersections.json`. Watch this for SF-data gaps that need the alias map. +- `between_context_resolved`: lat/lng → nearest-intersection success/fail. + +### Review gate (2–4 weeks after FIELD rollout) + +Three metrics drive the next decision: + +1. **Adoption**: what % of new incidents use intersection mode? If <10%, did we expose the feature well enough? +2. **Voice accept rate**: of voice-initiated entries, what % does the officer accept the top match? Target ≥80%. +3. **Lookup miss rate**: what % of intersection submits had no DataSF match? Target ≤5%. If higher, expand the alias map. + +If voice accept rate is below 80% and the failure mode is dominated by proper-noun street names: **escalate to Deepgram Nova-3 with Keyterm Prompting** for the location path only (keep AWS Transcribe for narrative). Concrete next-task: add a Deepgram client behind a feature flag, A/B against AWS Transcribe on the same audio for a week. + +If lookup misses cluster around specific street name aliases: extend `streetAliases.json` and re-build. + +### Out of scope for v1 (deferred) + +- Subject address intersection support. +- Facility address intersection support. +- Multi-jurisdiction street data (anything beyond SF). +- what3words integration. +- LLM/Bedrock natural-language parsing ("near 16th and the BART"). +- AWS Autocomplete sanity-check layer. +- N-best AWS Transcribe batch retry on low-confidence streaming results. +- Refactoring address out of `Incident` into a polymorphic `Location` table. + +--- + +## File-level summary (delivery checklist) + +### New files + +| Path | Phase | +| --- | --- | +| `server/data/sf-intersections.json` | P1 | +| `server/data/sf-streets.json` | P1 | +| `server/data/streetAliases.json` (hand-curated) | P1 | +| `server/scripts/build-intersection-data.js` | P1 | +| `server/scripts/build-transcribe-vocabulary.js` | P3 | +| `server/lib/intersections.js` | P1 | +| `server/lib/streetNormalization.js` | P1 | +| `server/lib/phoneticMatch.js` | P3 | +| `server/routes/api/geocode/intersections.js` | P3 | +| `server/test/lib/streetNormalization.test.js` | P6/P1 | +| `server/test/lib/intersections.test.js` | P6/P1 | +| `server/test/lib/phoneticMatch.test.js` | P6/P3 | +| `server/test/routes/api/geocode/intersections.test.js`| P6/P3 | +| `client/src/components/LocationAutocomplete.jsx` | P4 | +| `client/src/components/LocationVoiceButton.jsx` | P4 | +| `client/src/components/LocationConfirmationChip.jsx` | P4 | +| `client/src/components/LocationAutocomplete.test.jsx` | P6/P4 | + +### Modified files + +| Path | Phase | +| --- | --- | +| `server/prisma/schema.prisma` | P2 | +| `server/models/incident.js` | P2 | +| `server/routes/api/ai/transcribe.js` | P3 | +| `server/test/routes/api/ai/transcribe.test.js` | P6/P3 | +| `server/test/routes/api/incidents/create.test.js` | P6/P2 | +| `client/src/utils/validators.js` | P2 | +| `client/src/utils/format.js` | P2/P5 | +| `client/src/services/Api.js` (or equivalent geocode client) | P4 | +| `client/src/lesc/components/IncidentForm.jsx` | P4 | +| `client/src/lesc/components/Incident.jsx` | P5 | +| `client/src/lesc/components/custody/CustodyDetailContent.jsx` | P5 | +| `client/src/components/facilityAddressLink/` | P5 | +| `client/src/lesc/components/IncidentForm.test.jsx` | P6/P4 | + +### Dependencies added + +- `server`: `natural` (Double Metaphone, NYSIIS, Levenshtein) — ~2 MB. + +### PR split (suggested) + +Three PRs feels right; this is a feature with three natural seams. + +1. **PR 1 — Data + schema** (P1 + P2). Pure foundation. Reviewable on its own, low blast radius. +2. **PR 2 — Server + display** (P3 + P5). Backend endpoints, display branching, "between" context. Visible behavior change for existing addresses (the "between" line) but no UX-breaking change. +3. **PR 3 — Client UX + voice** (P4). The user-facing feature. Lands behind a measure-and-iterate gate (P7). + +Alternative: one bundled PR if review velocity favors it. Lean: three PRs for review tractability — but if reviewer prefers bundling, easy to combine. diff --git a/docs/cross-streets-research.md b/docs/cross-streets-research.md new file mode 100644 index 00000000..3e989c08 --- /dev/null +++ b/docs/cross-streets-research.md @@ -0,0 +1,492 @@ +# Cross-Streets Location Input: External Research Report + +**Audience:** Care Connect design team +**Purpose:** Inform the design doc for adding cross-street ("16th & Valencia") location entry to the field-officer incident report flow. +**Scope:** External research only — no Care Connect code changes proposed here. + +--- + +## TL;DR + +1. **Intersection geocoding is a first-class concept across the major CAD/911 vendors and consumer maps.** The de-facto entry convention is `Street1 & Street2` (or `Street1 and Street2`). Google Maps, Mapbox, Apple Maps, Uber, Lyft, Mark43, PremierOne, Hexagon, Tyler all support this, but the geographic coverage and exact behavior differ. +2. **AWS Location Service's forward `Geocode` and `Autocomplete` APIs *do* accept intersection input — but the documented "intersections" capability is reverse-geocoding-centric.** The response schema has a first-class `Intersection: [string]` array and `PlaceType: "Intersection"`, and the `AdditionalFeatures: ["Intersections"]` flag on `ReverseGeocode` is the way to get "nearest intersection" lookups. Forward intersection support in AWS Location v2 is real but underdocumented; expect to validate with empirical SF testing before committing. +3. **DataSF has authoritative, free, lat/lng-indexed lists of every SF street, intersection, and centerline node.** This is the strongest fallback if AWS intersection coverage is patchy in SF. (`pu5n-qu5c`, `ctsg-7znq`, `sw2d-qfup`, `3psu-pn9h`.) +4. **For STT, the right combination is: AWS Transcribe Custom Vocabulary (with display forms) + alternative transcriptions (n-best up to 10) + post-hoc fuzzy/phonetic matching against an SF street name dictionary.** Custom vocabulary alone won't fix "Junipero Sarah" because IPA/SoundsLike pronunciation hints are no longer honored in non-medical custom vocabulary. The fix is **n-best + Double-Metaphone fuzzy match against the SF street list.** +5. **If voice accuracy on proper-noun street names becomes a recurring blocker, Deepgram Nova-3 "Keyterm Prompting"** (up to 100 terms, advertised 90% keyword-recall improvement, preserves casing) **is materially stronger than AWS Transcribe custom vocabulary** for biasing recognition toward a fixed lexicon. +6. **Detecting intersection-vs-address input is trivially regexable on the client** (the presence of `&`, `/`, `@`, or the word `and`/`at` between two non-numeric tokens is enough to flip into intersection mode). NENA and ArcGIS treat `& | @ \` (and `and`/`at`) as the canonical connectors. +7. **Surprising finding:** Apple/Google use intersections silently for *driver privacy*, not officer ergonomics — but the underlying UX (single text field, mode auto-detected from connector token) is exactly what officers want. We should copy that pattern. + +--- + +## 1. How other applications handle cross-streets vs addresses + +### 1.1 Computer-Aided Dispatch (CAD) and RMS vendors + +CAD systems have treated cross-streets as a first-class location type for decades. Industry best-practice documents (e.g., Billerica PD CAD SOP) explicitly require an intersection to be entered in the **street name field** in the form `BOSTON & COOK` — with **no street suffix abbreviations** (no `RD`, `ST`, `AV`) when in intersection mode. The system then validates the intersection against its centerline GIS, snaps to the nearest node, and may auto-append the *next* nearest cross street as a confirming detail to the dispatcher. + +Sources: +- [BJA / LEITSC — Law Enforcement CAD Systems Functional Specification](https://bja.ojp.gov/sites/g/files/xyckuh186/files/media/document/leitsc_law_enforcement_cad_systems.pdf) — describes intersection validation, address ranges, and X/Y/Z translation as core CAD functions. +- [Billerica Police Department — COM-05 CAD SOP](https://public.powerdms.com/BillericaPD/documents/1846604) — concrete formatting rules ("`BOSTON & COOK`", drop suffixes for intersections). + +**Vendor-by-vendor:** + +| Vendor | Intersection support | Notes | +| --- | --- | --- | +| Mark43 CAD | Yes, via Esri-based location entry and what3words integration. | Marketed around "pinpoint exact locations" with Esri partnership; also integrates RapidSOS for verified caller coordinates. ([Mark43 CAD overview](https://mark43.com/platform/cad/)) | +| Motorola PremierOne | Yes — built on a "location-based, GIS-data map." | Optimized for multi-agency PSAPs; intersection and common-place-name lookup are baseline features. ([Motorola PremierOne CAD User Guide](https://www.motorolasolutions.com/content/dam/msi/docs/support/manuals/CAD6_7_8UserGuide.pdf)) | +| Hexagon Intergraph I/CAD | Yes — integrated mapping/dispatch with intersection lookup; what3words integration available. ([Hexagon OnCall what3words integration](https://what3words.com/business/emergency)) | +| Tyler New World CAD | Yes — NCIC/RMS lookups and mobile CAD support intersection entry. ([Tyler / FirstArriving integration article](https://support.firstarriving.com/support/solutions/articles/36000485333-tyler-technologies-new-world-cad)) | +| TriTech / CentralSquare | Yes — long-standing intersection geocoding via local centerline. (Industry standard, no single linkable doc found in this research pass.) | +| RapidSOS UNITE | "GIS module automatically uses your agency's local geocoder first for address search, **cross streets**, and common place names." ([RapidSOS GIS Mapping](https://rapidsos.com/public-safety/gis-mapping/)) | + +**RMS systems** (Versaterm, Caliber, Axon, ARMS, Omnigo) typically receive location data via CAD-to-RMS hand-off, so the intersection format originates in CAD. ([Axon — Complete Guide to Police RMS](https://www.axon.com/resources/police-rms), [ARMS RMS](https://arms.com/products/rms/)) + +### 1.2 Ride-share apps + +Both Uber and Lyft support intersection input in the same text box as addresses, with the explicit UX goal of protecting rider home addresses. From [Engadget — "Shield your address from Uber by using cross streets"](https://www.engadget.com/2017-04-13-uber-cross-streets-for-pickup-and-dropoff.html): + +> "Both Uber and Lyft now allow users to enter two cross streets when typing in a destination or pickup location." + +Lyft's mapping is powered by Google Maps, which is why intersection support arrived there first. Uber explicitly added it at user request. The interesting engineering insight from Uber: **"a rider's pin may drop in the middle of the street, creating confusion about which side the rider is on"** — i.e., intersection pinpoints are slightly ambiguous about the corner ([Uber Newsroom — pickup experience](https://www.uber.com/newsroom/smooth-pickup-experience/)). For incident reporting this matters less, but worth noting. + +### 1.3 Consumer maps + +| Service | Intersection input | Notes | +| --- | --- | --- | +| **Google Maps** | Yes. Format `Street A & Street B, City` returns a coordinate. The Geocoding API documents a `RANGE_INTERPOLATED` `location_type` for interpolated points between intersections. | "Google Maps is aware of street intersections in the US. The geocoder does not currently know any countries outside the US that use street intersections as addresses." ([Google Groups: Geocoding street intersections](https://groups.google.com/g/google-maps-api-web-services/c/UhvCp2yyx-M); [Geocoding requests/responses](https://developers.google.com/maps/documentation/geocoding/requests-geocoding)) Doesn't handle complex constructions like "Between Pine St and Church St" or "SW Corner of Main & Elm." | +| **Apple Maps / MapKit** | Yes in the consumer Maps app (the typed search box accepts "Market and Castro"). MapKit's `MKLocalSearch` / `MKLocalSearchCompleter` will return intersection POIs. ([Apple Community: intersection search](https://discussions.apple.com/thread/7666887); [MapKit](https://developer.apple.com/documentation/mapkit/)) No first-class `placemark.kind == intersection` type, but the search completer surfaces them. | +| **Waze** | Yes in the search box; Waze's data model is OSM-derived. ([Waze community thread](https://support.google.com/waze/thread/126493485/is-there-a-way-to-key-in-an-intersection-of-two-roads?hl=en)) | +| **Mapbox** | Yes — Geocoding v5/v6 supports "intersection search" with `and` or `&` between street names. *"Intersection search is not available in all countries."* ([Mapbox Geocoding API](https://docs.mapbox.com/api/search/geocoding-v5/), [Mapbox Help](https://docs.mapbox.com/help/getting-started/geocoding/)) Mapbox also published (now deprecated) [`cross-street-indexer`](https://github.com/mapbox/cross-street-indexer) — a tile-based geocoder specifically for OSM intersections; useful as a reference architecture. | + +### 1.4 Established UX patterns for "address or intersection" + +The dominant pattern across all consumer/CAD systems is a **single free-text field with auto-detection**, not a mode toggle: + +- User types `16th and Valencia` → suggestion list switches to intersection-typed suggestions. +- User types `425 16th St` → suggestion list returns point addresses. +- The connector tokens (`&`, `and`, `/`, `@`) are the trigger. + +Esri's ArcGIS geocoder formalizes this: **"By default, the symbols `&`, `|`, `@`, and `\` are recognized as intersection connectors,"** plus locale-specific words like `and` and `at` in the US ([Esri docs](https://doc.esri.com/en/arcgis-pro/latest/help/data/geocoding/fundamentals-of-intersection-geocoding.html)). + +CAD systems are slightly different — they tend to expose an explicit "intersection" choice in the address-type radio button, because dispatchers also need other types (mile-marker, common-place name, lat/long). For a **field-officer** app, the consumer pattern (no toggle, infer from input) is closer to what officers want. + +--- + +## 2. AWS Location Service — what's actually supported + +### 2.1 Endpoint landscape (v2 / `geo-places`) + +The v2 Places APIs are split into the following relevant endpoints ([Places APIs overview](https://docs.aws.amazon.com/location/latest/developerguide/places-choose-api.html)): + +| Endpoint | What it does | Intersection-relevant? | +| --- | --- | --- | +| `Geocode` | Forward geocoding: free-text or structured → lat/lng + components. | **Yes** — `QueryText` accepts intersection strings; response includes `Address.Intersection: [string]` and `MatchScores.Components.Address.Intersection`. | +| `ReverseGeocode` | lat/lng → address. Supports `AdditionalFeatures: ["Intersections"]` to return nearby intersections with `Heading` and street-type filters. | **Yes — this is the best-documented path.** | +| `Autocomplete` | Typeahead suggestions from partial input. | **Yes** — response carries `Address.Intersection: [string]` and `PlaceType: "Intersection"` is a valid result type. | +| `Suggest` | Like Autocomplete but returns search-term suggestions (cheaper). | Yes for the response shape, similar to Autocomplete. | +| `SearchText` | Text query for places/POIs. | Less suited to intersections; biased toward POIs. | +| `SearchNearby` | Radius search around a point. | N/A. | +| `GetPlace` | Detail lookup by `PlaceId`. | Used to expand an intersection PlaceId into full coordinates. | + +Reference: [Places choose-api table](https://docs.aws.amazon.com/location/latest/developerguide/places-choose-api.html). + +### 2.2 Forward-geocoding an intersection + +The `Geocode` API ([API reference](https://docs.aws.amazon.com/location/latest/APIReference/API_geoplaces_Geocode.html)) accepts: + +```json +{ + "QueryText": "16th and Valencia, San Francisco, CA", + "BiasPosition": [-122.4194, 37.7749] +} +``` + +The response shape includes: + +```json +{ + "ResultItems": [{ + "PlaceType": "Intersection", // <-- valid value + "Address": { + "Intersection": ["16th St", "Valencia St"], + "Label": "16th St & Valencia St, San Francisco, CA, USA", + ... + }, + "Position": [-122.4214, 37.7651], + "MatchScores": { + "Components": { + "Address": { "Intersection": [0.98, 0.99] } + } + } + }] +} +``` + +Notes: +- `Address.Intersection` is documented as `array of strings` in every relevant Places type (Geocode, Autocomplete, ReverseGeocode result items). +- `MatchScores.Components.Address.Intersection` is an array of per-street confidence scores — useful for ranking ambiguous matches. +- The `ParsedQuery.Address.Street` array exposes what the parser identified, which is useful for showing "we read this as: 16th St + Valencia St" UI confirmation. + +**Caveat:** the AWS documentation centers intersection examples around `ReverseGeocode + AdditionalFeatures: ["Intersections"]`. The forward path is implicit ("the API supports flexible queries, including free-form text"), and there is no AWS-published example of `QueryText = "16th & Valencia"`. **Plan to verify empirically in SF before relying on it.** + +### 2.3 Reverse-geocoding to the nearest intersection + +This is **explicitly documented** ([How to get intersections](https://docs.aws.amazon.com/location/latest/developerguide/reverse-how-to-get-intersections.html)): + +```json +POST /v2/reverse-geocode +{ + "QueryPosition": [-122.4214, 37.7651], + "AdditionalFeatures": ["Intersections"], + "Heading": 45, // optional, for directional bias + "Filter": { "IncludePlaceTypes": ["Street"] } +} +``` + +Response includes a top-level `Intersections: [...]` array per `ResultItem`, each with `PlaceId`, `Position`, `Distance`, and `Address.Intersection: [street1, street2]`. AWS specifically calls out **"emergency services and delivery couriers"** as the target use case for this endpoint. + +This makes a great Care Connect feature in itself: from a captured GPS fix, propose `Address: 425 16th St` AND `Nearest intersection: 16th St & Valencia St` so the officer can pick whichever is operationally clearer. + +### 2.4 Data providers + +AWS Location v2 has unified onto Esri + HERE + Grab + OpenData under the hood, but the provider is no longer surfaced per-place-index in v2 as it was in v1 ([v1 data-provider features](https://docs.aws.amazon.com/location/previous/developerguide/data-provider-features.html)). Provider differences known from v1 docs: + +- **Esri**: Best US street coverage, returns unit information. +- **HERE**: Returns time-zone info; strong globally. +- **Grab**: Southeast Asia only. + +Esri is the historical workhorse for US intersection data, and ArcGIS's intersection-geocoding code path is mature ([Esri intersection geocoding](https://doc.esri.com/en/arcgis-pro/latest/help/data/geocoding/fundamentals-of-intersection-geocoding.html)). Since AWS Location v2 packages Esri data, intersection support should be reasonable for SF, but we should still **bench-test against a sample of SF intersections** (especially weird ones: Junipero Serra & Monterey, Geary & Masonic, the Panhandle). + +### 2.5 Storage / pricing gotcha + +The `Geocode` API's `IntendedUse` defaults to `SingleUse`. **If we want to persist the resolved coordinates on the incident record, we must set `IntendedUse: "Storage"`**, which is charged at a higher rate ([Geocode API ref](https://docs.aws.amazon.com/location/latest/APIReference/API_geoplaces_Geocode.html)). Autocomplete results **cannot be stored at all** in v2. Practical pattern: use Autocomplete for typeahead UI, then re-Geocode (with `IntendedUse: Storage`) on the final selection to persist. + +--- + +## 3. Alternative geocoding sources for SF intersections + +### 3.1 DataSF (recommended fallback / local cache) + +DataSF publishes multiple closely-related datasets via Socrata. All are free, JSON via SODA API, no key required for low-volume use. + +| Dataset | ID | What it is | +| --- | --- | --- | +| [Street Intersections](https://data.sfgov.org/Geographic-Locations-and-Boundaries/Street-Intersections/ctsg-7znq/data) | `ctsg-7znq` | One row per (intersection × street) — multiple rows per intersection node, one per intersecting street. Includes CNN (Centerline Network Number). | +| [List of Streets and Intersections](https://data.sfgov.org/Geographic-Locations-and-Boundaries/List-of-Streets-and-Intersections/pu5n-qu5c/data) | `pu5n-qu5c` | Combined list of segments + intersections, sorted by street name and ascending address number. | +| [List of Intersections only](https://data.sfgov.org/Geographic-Locations-and-Boundaries/List-of-Intersections-only/sw2d-qfup/data) | `sw2d-qfup` | Just intersections, sorted by street name. | +| [Streets — Active and Retired](https://data.sfgov.org/Geographic-Locations-and-Boundaries/Streets-Active-and-Retired/3psu-pn9h) | `3psu-pn9h` | Authoritative street name list, including aliases and historical names. | +| [Street Centerlines and Nodes (handbook entry)](https://datasf.gitbook.io/draft-publishing-standards/standard-reference-data/basemap/street-centerlines-nodes) | n/a | Documents how SF models the centerline graph. | + +Architectural angle: we could **vendor a static `streets.json` + `intersections.json`** into the client (or server) at build time. For SF this is small — on the order of ~10k–15k intersections and ~2k unique street names — and gives us: + +- A canonical, complete street-name dictionary for AWS Transcribe custom vocabulary. +- A canonical intersection list for client-side autocomplete with zero API latency / cost. +- The lat/lng of every intersection node for direct pin placement. +- A safety net for AWS Location v2 misses. + +DataSF base portal: [data.sfgov.org](https://www.sf.gov/understanding-san-franciscos-street-level-data). Socrata API docs: [dev.socrata.com](https://dev.socrata.com/). + +### 3.2 OpenStreetMap / Nominatim / Photon + +- **Nominatim** has [poor intersection support](https://github.com/osm-search/Nominatim/issues/123). Long-standing open issue: searching `Street1, Street2` returns no result. Several community workarounds exist but none are robust. +- **Photon** (Komoot) inherits OSM data and the same limitation; no first-class intersection-search documented. +- **OSM Overpass API** can return intersection nodes by querying `node[highway=intersection]` or by intersecting `way` geometries, but this is a low-level GIS query, not a user-facing geocoder. +- **R-bloggers walkthrough** ([Geocoding An Intersection with OSM data](https://www.r-bloggers.com/2020/08/geocoding-an-intersection-with-open-street-map-data/)) shows manual approach. + +**Verdict:** OSM-based geocoding is **not a good direct backend for intersection input.** Useful only as a data source for a custom-built intersection index (similar to how Mapbox built `cross-street-indexer`). + +### 3.3 Google Maps + +- Native intersection support in Geocoding API ([requests-geocoding docs](https://developers.google.com/maps/documentation/geocoding/requests-geocoding)). +- Pricing: ~$5.00 per 1,000 requests under Essentials SKU, with 10,000 free events/month ([Maps Platform pricing](https://developers.google.com/maps/billing-and-pricing/pricing)). +- Limitations: US-only intersection awareness, can't parse natural-language modifiers ("end of Main St", "SW corner of…"). +- Practical issue for Care Connect: ToS-restricted (can't be cached/stored persistently without a license tier upgrade — same general issue as AWS `IntendedUse`). + +### 3.4 Mapbox + +- Intersection search available in [Geocoding v5/v6](https://docs.mapbox.com/api/search/geocoding-v5/) with `and` or `&` syntax. +- Country availability is restricted but **the US is supported**. +- Pricing competitive with Google. +- Deprecated [`cross-street-indexer`](https://github.com/mapbox/cross-street-indexer) is a reference for building an OSM-derived intersection index. + +### 3.5 Canonical "list of SF intersections" — yes, it exists + +DataSF's `ctsg-7znq` + `pu5n-qu5c` + `sw2d-qfup` collectively constitute that canonical list, free, lat/lng-tagged. **This is the strongest argument for a vendored local index.** + +--- + +## 4. Voice STT for street names — accuracy techniques + +### 4.1 The problem, restated + +User dictates `Junipero Serra and Monterey` → AWS Transcribe outputs `Junipero Sarah and Monterey` → exact-string geocoder lookup fails → officer sees "no match" and reverts to manual entry. + +The fix is one (or a combination) of: +1. Bias the recognizer toward `Junipero Serra` (and the rest of SF's ~2,000 street names) ahead of time. +2. Generate n-best alternatives and pick the one that geocodes. +3. Fuzzy/phonetic match the literal transcript against a known SF street dictionary post-hoc. + +In practice, (1) + (3) combined is the industry pattern. + +### 4.2 AWS Transcribe options + +**Custom Vocabulary** ([Custom vocabularies docs](https://docs.aws.amazon.com/transcribe/latest/dg/custom-vocabulary.html)): + +- Designed exactly for "domain-specific terms, brand names, acronyms, proper nouns" — street names are textbook examples. +- Supports a `DisplayAs` column so you can normalize the rendered string (e.g., recognize "saint francis" → write `St Francis`). +- **Important and not in the user's memory yet**: the `IPA` and `SoundsLike` pronunciation-hint columns are **no longer honored** in non-medical custom vocabulary tables — values are ignored ([AWS docs note](https://docs.aws.amazon.com/transcribe/latest/dg/custom-vocabulary.html)). This is the single biggest gotcha. Pronunciation control survives only in AWS Transcribe Medical. +- 300-word soft limit per vocabulary for best results; for ~2k SF streets, split into a small set of focused vocabularies (downtown, Sunset, etc.) or accept the larger vocabulary's diminishing returns. + +**Custom Language Model (CLM)** ([Custom language models](https://docs.aws.amazon.com/transcribe/latest/dg/custom-language-models.html)): + +- Captures *context* (e.g., "officer responding to a call at...") rather than individual word recognition. +- Requires training data and meaningfully more setup. +- Verdict: **overkill for street names alone**; revisit if Care Connect ends up dictating full narratives with consistent radio-style language. + +**Alternative Transcriptions (n-best)** ([Alternative transcriptions](https://docs.aws.amazon.com/transcribe/latest/dg/alternatives.html)): + +- `ShowAlternatives=true, MaxAlternatives=N` (1–10). Returns N transcripts per segment with confidence scores. +- **Batch only**, not streaming. (If we want streaming, this technique is unavailable in Transcribe.) +- Combined with fuzzy match: try alt 1 against the street dictionary; if no match within distance threshold, try alt 2, etc. + +**Vocabulary Filtering**: not relevant here (it's for redaction). + +### 4.3 Cross-vendor comparison for "bias toward a known lexicon" + +| Vendor / feature | Mechanism | Limit | Strength for street names | +| --- | --- | --- | --- | +| **AWS Transcribe Custom Vocabulary** | Pre-built named resource, `Phrase` + `DisplayAs`. | ~300 words recommended; no pronunciation hints (non-medical). | Decent — but no way to teach `Junipero` = `hoo-NEE-pear-o` for the standard model. | +| **Deepgram Keywords (Nova-2/Enhanced/Base)** | Per-request `keywords=TERM:INTENSIFIER`, runtime, no pre-step. | 100 keywords per request; single-word only. | Good for typeahead-feel use cases. | +| **Deepgram Keyterm Prompting (Nova-3 / Flux)** | Per-request, in-context learning at inference time. | Up to 100 terms; multi-word supported; preserves capitalization & formatting. | **Strongest in class.** Deepgram quotes up to 90% improvement in keyword-recall rate; one customer reported 625% improvement on veterinary terms after switching from Nova-2 keywords → Nova-3 keyterms. Multilingual variant supports Spanish street names. ([Deepgram Keyterm Prompting](https://developers.deepgram.com/docs/keyterm); [Deepgram blog](https://deepgram.com/learn/deepgram-expands-nova-3-with-10-new-languages-and-multilingual-keyterm-prompting)) | +| **Google Cloud Speech-to-Text — Model Adaptation / Speech Context** | `phrases` (up to 5,000) with `boost`; supports `Class Tokens` like `$ADDRESSNUM`. | 5k phrases, 125+ languages. | Strong, and the **only one with a built-in `$ADDRESSNUM` / address class token**. Worth a real bake-off if AWS doesn't get us there. ([Google STT speech adaptation](https://cloud.google.com/speech-to-text/docs/adaptation-model)) | +| **Azure Speech — Custom Speech** | Train a customized acoustic + language model on Azure-hosted text/audio corpora. | Largest customization surface, slowest setup. | Strongest "deep" customization but heavy operationally; probably not needed for SF streets alone. | + +**Recommendation hierarchy:** +1. Start with AWS Transcribe + Custom Vocabulary (we're already on AWS) + n-best (if batch) + post-hoc fuzzy match. +2. If proper-noun street names remain a recurring failure (Junipero Serra, Geary, Bayshore), evaluate **Deepgram Nova-3 with Keyterm Prompting** — that one feature is significantly better-targeted at this exact problem than anything AWS offers in the non-medical Transcribe family. Streaming-friendly too. +3. Google STT's `$ADDRESSNUM` class is the best-fit primitive for the *address* path; less so for intersections. + +### 4.4 On-device / Web Speech API + +Web Speech API on mobile has confirmed, documented unreliability: + +- Safari/iOS: interim results duplicate, `isFinal` never fires, recognition gets throttled and switches to cloud mid-stream. ([WebKit issue #120](https://github.com/WebKit/Documentation/issues/120); [Apple Developer Forums thread](https://developer.apple.com/forums/thread/694847)) +- Requires online connectivity on Chrome/Safari — recognition runs on Google/Apple servers. +- General industry sentiment ([addpipe deep dive](https://blog.addpipe.com/a-deep-dive-into-the-web-speech-api/), [Medium "Taming the Web Speech API"](https://webreflection.medium.com/taming-the-web-speech-api-ef64f5a245e1)): **don't ship it in production for anything user-critical on mobile.** + +The user's memory note ("Web Speech API abandoned — unreliable on mobile") is squarely consistent with current industry sentiment in 2026. + +**Apple Speech (`SFSpeechRecognizer`)** is much more reliable than Web Speech on iOS but requires a native iOS shell. Worth noting in case Care Connect ships a wrapper app — Apple Speech now supports on-device recognition for most languages without server round-trip. + +### 4.5 Phonetic / fuzzy matching against a known vocabulary (post-hoc) + +This is the **highest leverage technique** for the specific Junipero Serra → Junipero Sarah failure mode, because the transcription's phonetic content is right; the spelling is just wrong. + +**Algorithms** ([Phonetic Matching Algorithms — Medium](https://medium.com/@ievgenii.shulitskyi/phonetic-matching-algorithms-50165e684526), [Babel Street — Fuzzy Name Matching Techniques](https://www.babelstreet.com/blog/fuzzy-name-matching-techniques)): + +| Algorithm | Best for | Note | +| --- | --- | --- | +| **Soundex** | Surnames, simple cases | Fixed-length key (4 chars), English-centric, weak for modern names. | +| **NYSIIS** | Street names | Babel Street article explicitly recommends NYSIIS for street names. Worth trying. | +| **Metaphone** | English proper nouns | Better than Soundex; variable-length key. | +| **Double Metaphone** | General-purpose, recommended default | Returns *primary* + *secondary* code, handles non-English origins (Spanish, Italian, Slavic) — important for SF (Cesar Chavez, Junipero, Bernal). | +| **Levenshtein / edit distance** | Typos | Doesn't help when the mis-spelling is phonetically faithful ("Sarah" vs "Serra"). | + +**Recommended approach for Care Connect:** +1. Pre-compute Double Metaphone (primary + secondary) for every SF street name from DataSF `3psu-pn9h`. +2. After transcription, compute Double Metaphone of each token in the transcript, find best matches. +3. Combine with Levenshtein/token-set ratio as tiebreak. + +**JavaScript libraries** (server-side Node.js): +- [`fuzzball`](https://github.com/nol13/fuzzball.js) — port of Python's fuzzywuzzy; `extract` for top-N matches with cutoff. Mature and battle-tested. +- [`fuse.js`](https://www.fusejs.io/) — most popular browser-friendly fuzzy library; client-side viable for SF's ~2k street list. +- [`fuzzysort`](https://github.com/farzher/fuzzysort) — fastest for "match against list" use cases, handles diacritics. +- [`natural`](https://github.com/NaturalNode/natural) (npm) — provides `Metaphone`, `DoubleMetaphone`, `SoundEx`, `NYSIIS` out of the box. **Best fit for the phonetic step.** +- [`double-metaphone`](https://github.com/words/double-metaphone) — focused, well-maintained. + +**Suggested pipeline** for Junipero-Sarah failure: +``` +transcript = "Junipero Sarah and Monterey" +tokens = split on connectors -> ["Junipero Sarah", "Monterey"] +for each token: + candidates = sfStreets.byDoubleMetaphone(token) + .sortBy(combined(DoubleMetaphoneMatch, Levenshtein, NYSIIS)) + .top(5) +present candidates as confirmation chips: [Junipero Serra] [June Pereira] ... +``` + +--- + +## 5. UX patterns for unified address/intersection input + +### 5.1 Detecting intent from input + +Trivially regexable on the client. The connectors AWS, Esri, Google, and Mapbox all recognize are: **`&`, `/`, `@`, `\`, `|`, the word `and`, and the word `at`** (US locale) ([Esri docs](https://doc.esri.com/en/arcgis-pro/latest/help/data/geocoding/fundamentals-of-intersection-geocoding.html)). + +A pragmatic detection heuristic: + +``` +isIntersection(input): + normalized = input.trim().toLowerCase() + // 1. explicit connector symbols + if /[&/@\\|]/.test(normalized): return true + // 2. " and " or " at " between two non-numeric tokens + if / (and|at) /.test(normalized) AND no leading street number: return true + // 3. otherwise, address (if starts with digit) or POI/place name + if /^\d/.test(normalized): return "address" + return "ambiguous" // route to general search +``` + +### 5.2 Suggest behavior + +The dominant pattern (Google, Apple, Uber, Lyft, Mapbox): **single field, mode auto-detected from the connector token, suggestion list re-types itself when the connector is typed.** Concretely: + +- User types `16th` → suggestions show "16th St", "16th Ave", "16th & Mission", "16th & Valencia", "16th BART"… +- User types `16th &` → suggestions collapse to intersection-only. +- User types `16th & V` → suggestions show "16th & Valencia", "16th & Vermont", "16th & Van Ness". + +AWS Location `Autocomplete` supports this directly — you can `Filter.IncludePlaceTypes: ["Intersection"]` once we detect the connector token, and use `BiasPosition` to keep results inside SF. + +### 5.3 Natural-language input ("near 16th & Valencia", "outside the 16th St BART") + +This is largely **unsolved** in shipping geocoders: + +- Google explicitly doesn't parse "End of Main St" or "SW Corner of Main & Elm" ([Nodal Bits article](https://www.nodalbits.com/bits/google-maps-intersections/)). +- There is academic and patent work on it ([USPTO 11,341,334 — "Method and apparatus for evaluating natural language input to identify actions and landmarks"](https://image-ppubs.uspto.gov/dirsearch-public/print/downloadPdf/11341334); [USPTO 7,983,913 — "Understanding spoken location information based on intersections"](https://image-ppubs.uspto.gov/dirsearch-public/print/downloadPdf/7983913)), but no off-the-shelf product reliably handles arbitrary natural-language location. +- **Modern workaround**: an LLM pass (Claude / Bedrock) over the raw transcript can extract a structured `{ street1, street2, modifier: "near"|"outside"|"end of" }` triple, then call AWS Location with the extracted streets. This is novel territory but plausible given Care Connect is already an AWS/Bedrock shop (per project memory on the ID Scanner feature). + +### 5.4 "Cross street as context" pattern (worth stealing) + +Several CAD systems show the *address* as primary and the *nearest cross street* as secondary context, even when the user entered just an address. For an officer reading back a location over radio, "425 16th St — between Valencia and Guerrero" is materially more useful than just "425 16th St." Achievable in Care Connect by: + +1. After resolving an address, call `ReverseGeocode` on its coordinates with `AdditionalFeatures: ["Intersections"]` to get the nearest two intersections. +2. Surface them as read-only "between X and Y" metadata under the address. + +This is a cheap, high-value UX improvement orthogonal to the main "let officers enter intersections" feature. + +--- + +## 6. Standards, accessibility, and miscellaneous findings + +### 6.1 NIEM / APCO / NENA standards + +The applicable standard for location representation in incident records is **NENA-STA-004.2-2024** — the [NG9-1-1 US Civic Location Data Exchange Format (CLDXF-US)](https://www.nena.org/news/675079/NENA-NG9-1-1-United-States-Civic-Location-Data-Exchange-Format-CLDXF-US-Standard-Now-Available.htm). It defines a structured civic address schema (`HNO`, `HNS`, `RD`, `STS`, `POD`, `MP`, etc.). + +For inter-system incident exchange, **NENA/APCO EIDD (NENA-STA-021.1a / APCO 2.105.1-2017)** — [Emergency Incident Data Document](https://www.nena.org/page/EIDD) — is the NIEM-conformant XML schema used between disparate CAD/RMS/PSAP systems. It carries location both as civic address and as geometric/lat-lng. + +For Care Connect, **the practical implication is small but worth noting**: if the incident record ever needs to be exchanged with a PSAP/CAD via EIDD, store **both** the structured intersection (`{ street1, street2 }`) *and* a normalized lat/lng. EIDD doesn't have a clean "intersection-only" civic-address mode — intersection-resolved data is typically transmitted as `LocationByValue` with geodetic coordinates plus a `LocationDescription` text containing "between X & Y." + +References: +- [NENA Standard for NG9-1-1 GIS Data Model (NENA-STA-006.2-2022)](https://cdn.ymaws.com/www.nena.org/resource/resmgr/standards/nena-sta-006.2-2022_ng9-1-1_.pdf) — defines Road Centerlines layer schema. +- [NENA GIS Data Transition Info Document](https://cdn.ymaws.com/www.nena.org/resource/resmgr/standards/NENA-INF-046.1-2024_GIS_Data.pdf). + +### 6.2 what3words + +Worth a one-paragraph mention because it's gained real PSAP traction: + +- Pre-integrated with Mark43, Versaterm, Hexagon OnCall, Sun Ridge RIMS, RapidDeploy, RapidSOS, and built into 4,800+ ECCs ([what3words.com — emergency adoption](https://what3words.com/business/emergency)). +- LAFD officially integrated it into CAD in 2021; Hayward PD and FD also adopted. +- Strength: 3-word strings (`///filled.count.soap`) are easier to dictate over voice than coordinates, and resolve to a 3m square. +- **Relevance to Care Connect**: not a substitute for intersection entry (officers won't memorize w3w words for their beat), but a *third* input modality alongside address + intersection that costs ~0 to add via the free what3words API. Could be a "if you can read me a w3w from a caller's phone, paste it here" affordance. + +Ref: [Police1 — How what3words is changing PSAPs](https://www.police1.com/police-products/police-technology/software/cad/articles/what-if-you-could-save-a-life-in-three-words-well-theres-an-app-for-that-dRWTFl21O4Hqg96a/). + +### 6.3 Accessibility considerations for voice + text address input + +- **WCAG 2.2**: voice input is itself an accommodation; we must also ensure typed entry is fully equivalent (no voice-only flows). +- **Screen-reader compatibility**: when toggling between address-mode and intersection-mode suggestions, announce the change via `aria-live="polite"` so VoiceOver/TalkBack users know the result set has switched semantics. +- **Voice control users (Apple Voice Control, Dragon)** rely on stable accessibility labels for fields; the intersection auto-detect must not change the field's accessible name mid-interaction. +- **Field conditions**: bright sun, dark patrol cars, gloved hands all interact with input ergonomics. Auto-detection of intent (no toggle) reduces taps and is broadly an accessibility win. +- **Cognitive load**: officers are dictating under stress. Showing a confirmation chip (`16th St & Valencia St — tap to confirm`) before geocoding is preferable to silently snapping to a coordinate. CAD systems do exactly this in the dispatcher view. + +### 6.4 Surprising / counterintuitive findings + +1. **AWS Location v2's intersection support is real but the forward path is documentation-light.** The reverse-geocode "nearest intersection" feature is well-documented because that's how emergency services use it; the forward case is implicit. **Don't trust the docs alone — bench-test SF intersections.** +2. **Custom Vocabulary pronunciation hints (IPA / SoundsLike) were silently deprecated for non-medical use.** This is the single biggest "thing you'd expect to work, but doesn't" landmine. If we need real phonetic control over `Junipero`, we'll get it only by switching STT vendor (Deepgram Nova-3 Keyterm Prompting) or by accepting a fuzzy-match post-processing step. +3. **NYSIIS is specifically recommended for street names** over Double Metaphone (Babel Street article). I'd still default to Double Metaphone for SF because so many SF street names are Spanish-origin, but NYSIIS in combination would be worth measuring. +4. **Uber's intersection support was added primarily for *driver privacy*, not navigation efficiency.** This is a useful counter-data-point for the design doc — even though our motivation is different (officer ergonomics), the UX we want to copy is well-validated in a totally different problem domain. +5. **The dispatcher convention is to *drop* street suffixes for intersections** (`BOSTON & COOK`, not `BOSTON ST & COOK ST`). If officers learn this convention from radio traffic, our autocomplete should accept the suffix-less form transparently. +6. **DataSF makes a "vendor a local intersection index" approach genuinely cheap.** SF is small (~2k streets, ~10–15k intersections). A static index gives sub-100ms typeahead, zero AWS cost per keystroke, and a permanent fallback when AWS Location coverage misses an alley intersection. +7. **The phonetic failure mode is mostly about transliterated proper nouns.** `Junipero Serra` (Spanish), `Cesar Chavez` (Spanish), `Geary` (Irish), `Masonic` (English), `Embarcadero` (Spanish) — these break STT. Pure-English street names (`Market`, `Mission`, `Valencia` (as English) `Castro`, `Folsom`) generally transcribe fine. Suggests an 80/20: a curated *correction dictionary* of the ~50 most-mistranscribed SF street names could outperform a generic custom vocabulary. + +--- + +## Appendix: Source index + +### AWS Location Service +- [Places APIs (choose-api)](https://docs.aws.amazon.com/location/latest/developerguide/places-choose-api.html) +- [Geocode API reference](https://docs.aws.amazon.com/location/latest/APIReference/API_geoplaces_Geocode.html) +- [How to geocode an address](https://docs.aws.amazon.com/location/latest/developerguide/how-to-geocode-address.html) +- [How to get intersections (ReverseGeocode)](https://docs.aws.amazon.com/location/latest/developerguide/reverse-how-to-get-intersections.html) +- [Autocomplete API reference](https://docs.aws.amazon.com/location/latest/APIReference/API_geoplaces_Autocomplete.html) +- [v1 data-provider features](https://docs.aws.amazon.com/location/previous/developerguide/data-provider-features.html) + +### AWS Transcribe +- [Custom vocabularies](https://docs.aws.amazon.com/transcribe/latest/dg/custom-vocabulary.html) +- [Custom language models](https://docs.aws.amazon.com/transcribe/latest/dg/custom-language-models.html) +- [Alternative transcriptions](https://docs.aws.amazon.com/transcribe/latest/dg/alternatives.html) +- [Build a custom vocabulary blog post](https://aws.amazon.com/blogs/machine-learning/build-a-custom-vocabulary-to-enhance-speech-to-text-transcription-accuracy-with-amazon-transcribe/) + +### Other STT vendors +- [Deepgram Keywords](https://developers.deepgram.com/docs/keywords) +- [Deepgram Keyterm Prompting](https://developers.deepgram.com/docs/keyterm) +- [Deepgram Nova-3 launch](https://deepgram.com/learn/introducing-nova-3-speech-to-text-api) +- [Deepgram multilingual Keyterm Prompting](https://deepgram.com/learn/deepgram-expands-nova-3-with-10-new-languages-and-multilingual-keyterm-prompting) +- [Best STT APIs 2026 — Deepgram blog](https://deepgram.com/learn/best-speech-to-text-apis-2026) +- [WebKit Documentation issue #120 — Web Speech API on iOS](https://github.com/WebKit/Documentation/issues/120) +- [Apple Developer Forums — Web Speech bugs in iOS 15.1](https://developer.apple.com/forums/thread/694847) +- [addpipe — A Deep Dive into the Web Speech API](https://blog.addpipe.com/a-deep-dive-into-the-web-speech-api/) + +### Geocoders (consumer / commercial) +- [Google Geocoding API requests](https://developers.google.com/maps/documentation/geocoding/requests-geocoding) +- [Google Maps Platform pricing](https://developers.google.com/maps/billing-and-pricing/pricing) +- [Google Groups — Geocoding street intersections](https://groups.google.com/g/google-maps-api-web-services/c/UhvCp2yyx-M) +- [Nodal Bits — Google Maps & Intersections](https://www.nodalbits.com/bits/google-maps-intersections/) +- [Mapbox Geocoding v5](https://docs.mapbox.com/api/search/geocoding-v5/) +- [Mapbox Help — Search products](https://docs.mapbox.com/help/getting-started/geocoding/) +- [Mapbox cross-street-indexer (deprecated)](https://github.com/mapbox/cross-street-indexer) +- [Esri / ArcGIS Pro — Fundamentals of intersection geocoding](https://doc.esri.com/en/arcgis-pro/latest/help/data/geocoding/fundamentals-of-intersection-geocoding.html) +- [Nominatim issue #123 — intersection search](https://github.com/osm-search/Nominatim/issues/123) +- [OSM Help — finding intersection](https://help.openstreetmap.org/questions/16381/find-road-intersectiton-using-nominatim) +- [R-bloggers — Geocoding intersections with OSM](https://www.r-bloggers.com/2020/08/geocoding-an-intersection-with-open-street-map-data/) + +### DataSF +- [Understanding SF's street-level data](https://www.sf.gov/understanding-san-franciscos-street-level-data) +- [Street Intersections — `ctsg-7znq`](https://data.sfgov.org/Geographic-Locations-and-Boundaries/Street-Intersections/ctsg-7znq/data) +- [List of Streets and Intersections — `pu5n-qu5c`](https://data.sfgov.org/Geographic-Locations-and-Boundaries/List-of-Streets-and-Intersections/pu5n-qu5c/data) +- [List of Intersections only — `sw2d-qfup`](https://data.sfgov.org/Geographic-Locations-and-Boundaries/List-of-Intersections-only/sw2d-qfup/data) +- [Streets — Active and Retired — `3psu-pn9h`](https://data.sfgov.org/Geographic-Locations-and-Boundaries/Streets-Active-and-Retired/3psu-pn9h) +- [Street Centerlines and Nodes (handbook)](https://datasf.gitbook.io/draft-publishing-standards/standard-reference-data/basemap/street-centerlines-nodes) +- [Socrata SODA API docs](https://dev.socrata.com/) + +### CAD / RMS vendors and standards +- [BJA / LEITSC — Law Enforcement CAD Systems](https://bja.ojp.gov/sites/g/files/xyckuh186/files/media/document/leitsc_law_enforcement_cad_systems.pdf) +- [Billerica PD — COM-05 CAD SOP](https://public.powerdms.com/BillericaPD/documents/1846604) +- [Mark43 CAD](https://mark43.com/platform/cad/) +- [Motorola PremierOne User Guide](https://www.motorolasolutions.com/content/dam/msi/docs/support/manuals/CAD6_7_8UserGuide.pdf) +- [Tyler New World CAD — FirstArriving article](https://support.firstarriving.com/support/solutions/articles/36000485333-tyler-technologies-new-world-cad) +- [RapidSOS GIS Mapping](https://rapidsos.com/public-safety/gis-mapping/) +- [Axon — Complete Guide to Police RMS](https://www.axon.com/resources/police-rms) +- [NENA NG9-1-1 CLDXF-US Standard announcement](https://www.nena.org/news/675079/NENA-NG9-1-1-United-States-Civic-Location-Data-Exchange-Format-CLDXF-US-Standard-Now-Available.htm) +- [NENA Standard for NG9-1-1 GIS Data Model](https://cdn.ymaws.com/www.nena.org/resource/resmgr/standards/nena-sta-006.2-2022_ng9-1-1_.pdf) +- [NENA/APCO EIDD](https://www.nena.org/page/EIDD) + +### Phonetic / fuzzy matching +- [Babel Street — Fuzzy Name Matching Techniques](https://www.babelstreet.com/blog/fuzzy-name-matching-techniques) +- [Medium — Phonetic Matching Algorithms](https://medium.com/@ievgenii.shulitskyi/phonetic-matching-algorithms-50165e684526) +- [fuzzball.js](https://github.com/nol13/fuzzball.js) +- [Fuse.js](https://www.fusejs.io/) +- [fuzzysort](https://github.com/farzher/fuzzysort) +- [natural (Node.js NLP, includes Double Metaphone / NYSIIS)](https://github.com/NaturalNode/natural) + +### Ride-share / consumer maps +- [Engadget — Shield your address from Uber by using cross streets](https://www.engadget.com/2017-04-13-uber-cross-streets-for-pickup-and-dropoff.html) +- [Uber — Pickup Spots](https://www.uber.com/us/en/ride/how-it-works/pickup-spots/) +- [Uber Newsroom — Smooth pickup experience](https://www.uber.com/newsroom/smooth-pickup-experience/) +- [Apple Community — Intersection search](https://discussions.apple.com/thread/7666887) +- [MapKit documentation](https://developer.apple.com/documentation/mapkit/) +- [Waze community — intersection input](https://support.google.com/waze/thread/126493485/is-there-a-way-to-key-in-an-intersection-of-two-roads?hl=en) + +### what3words +- [what3words emergency adoption](https://what3words.com/business/emergency) +- [Police1 — How what3words is changing PSAPs](https://www.police1.com/police-products/police-technology/software/cad/articles/what-if-you-could-save-a-life-in-three-words-well-theres-an-app-for-that-dRWTFl21O4Hqg96a/) diff --git a/docs/cross-streets-spot-fixes.md b/docs/cross-streets-spot-fixes.md new file mode 100644 index 00000000..65157b89 --- /dev/null +++ b/docs/cross-streets-spot-fixes.md @@ -0,0 +1,178 @@ +# Cross-Streets — Spot-Fix Log + +Running log of in-the-weeds issues encountered while shipping the cross-streets +feature, with diagnosis and fix. Kept so we can spot recurring patterns and +decide if/when to revise the underlying strategy rather than continue +whack-a-mole. + +Each entry: what was observed → what the root cause was → what we changed → +optional pattern tag. + +Pattern tags emerging so far: +- `data-shape-assumption` — code that assumed the address-only shape and broke for intersection-mode records. +- `stt-fusion` — speech-to-text errors that collide tokens. +- `ranking-bug` — matcher returned candidates that didn't include the intended one inside the result limit. +- `ux-feedback` — affordance missing or feedback unclear. + +--- + +## #1 — 422 on "Hold a chair" + +**Observed:** Clicking "Hold a chair" failed with a server 422: +``` +"errors": [ + {"path": "street1", "message": "Invalid input: expected string, received undefined"}, + {"path": "street2", "message": "Invalid input: expected string, received undefined"}, + {"path": "intersectionId", "message": "Invalid input: expected string, received undefined"} +] +``` + +**Diagnosis:** `client/src/lesc/components/Holds.jsx:40` defines `buildBlankIncident(facilityId)` which posts a fully-blank incident payload to `/api/incidents` for the "first hold of an incident" flow. After P2 added `locationType`/`street1`/`street2`/`intersectionId` to the schema, this helper still only set the *original* fields explicitly. The new fields went to the server as `undefined`. Zod's `.nullable()` accepts `string | null` but not `undefined` — hence 422. + +**Decision (Option A vs B):** debated whether to loosen the schema (`.nullish()`) or keep schema strict and update the client payload-builder. Went with **B** to stay consistent with the existing convention in this codebase: server schema is strict, client lists every field explicitly. See conversation thread. + +**Fix:** added `locationType: 'ADDRESS'`, `street1: null`, `street2: null`, `intersectionId: null` to `buildBlankIncident` (`client/src/lesc/components/Holds.jsx:40-58`). Reverted the temporary `.nullish()` change in `server/models/incident.js`. + +**Pattern tag:** `data-shape-assumption` + +**Notes:** This is the *only* other client-side payload builder for incidents I found (the other path is `IncidentForm.jsx:buildIncidentPayload`, which I already updated). If future fields get added, both builders need updating — worth considering a shared helper or default factory. + +--- + +## #2 — Intersection-mode incidents always "details incomplete" + +**Observed:** While auditing fix #1, realized that *if* an intersection-mode incident were created, every downstream gate (handoff, transfer, cancel, my-holds, arrived) would treat it as incomplete because `isIncidentDetailsComplete` checks `incident.addressLine1 && incident.city && incident.state`. None of those are set for intersection mode. + +**Diagnosis:** `server/lib/incidentPermissions.js:22-33` and `server/lib/hospitalCancellation647f.js:32-42` both encode the same address-only completeness check. + +**Fix:** extracted a `isIncidentLocationComplete(incident)` helper in `incidentPermissions.js` that branches on `locationType`. INTERSECTION mode needs `street1 && street2`; ADDRESS mode keeps the original check. Both call sites updated. + +**Pattern tag:** `data-shape-assumption` + +**Notes:** This was the same shape of bug as #1 (downstream code didn't know about the discriminator) but in a different layer. Worth watching: if we find a third such case in unrelated code, the right move may be a typed "is this an incident with a usable location?" helper that becomes the single source of truth, rather than reimplementing the branch each time. + +--- + +## #3 — Arrests report & 849b PDF would show blank location for intersection-mode + +**Observed:** While auditing for #2, found two more places with the same address-only assumption: +- `server/routes/api/arrests.js` — Prisma `select { addressLine1, city, state }`, then `streetCityState(i)` for the response `address` field. +- `server/lib/forms/849b/generate.js:62` — `arrestLocation = [addressLine1, city, state].filter(...).join(', ')` for the SFSO PDF. + +**Diagnosis:** Same pattern as #2 — code that assumes the address shape. + +**Fix:** added `incidentLocationText(incident)` helper in `server/lib/forms/shared/formUtils.js` that branches on `locationType`. Added `locationType`/`street1`/`street2` to the arrests-route Prisma select. Both call sites now call the new helper. Per the Q3 decision in `cross-streets.md`, intersection-mode incidents render as e.g. "16th St & Valencia St" with city/state appended. + +**Pattern tag:** `data-shape-assumption` + +**Notes:** Three address-shape bugs in a row (#1, #2, #3). If we find one more, I'd argue for a single canonical "render an incident's location text" helper used everywhere, and a typed read-side wrapper that surfaces the discriminator. For now, deliberately leaving the per-call branching since the call sites are few. + +--- + +## #4 — Confirmation chip never appeared after a successful voice match + +**Observed:** User reported: "I don't see a chip anywhere after I successfully enter an intersection." + +**Diagnosis:** Mid-implementation I'd added a "if exactly 1 match, auto-fill and skip the chip" optimization in `IncidentForm.jsx:handleVoiceResult`. For the *common* case (e.g. "16th and Valencia" → 1 clean match), this meant zero feedback that voice was processed — the field silently updated. This directly contradicted what the implementation plan called for ("show a confirmation chip before commit. CAD systems do this. It's cheap, prevents silent misroutes."). + +**Fix:** removed the shortcut. Chip always shows when there's at least one match. Tightened the wording: single match reads "Tap to confirm:", multi reads "Did you mean:" (`client/src/components/LocationVoiceButton.jsx:LocationConfirmationChip`). + +**Pattern tag:** `ux-feedback` + +**Notes:** This was a self-inflicted optimization that I added without thinking through the implications for evidence integrity (officer-entered data). Worth a reminder to myself: don't optimize away verification steps in officer/evidence flows. + +--- + +## #5 — Mic disappeared in expanded view + +**Observed:** User reported the mic was visible on the collapsed location field but vanished when the field expanded into the address form. No way to use voice once expanded. + +**Diagnosis:** The expanded view's `LocationAutocomplete` only had `` (current-location icon) in its `rightSection`. The mic was only wired up on the collapsed view. + +**Fix:** mirrored the same `` into the expanded view's `rightSection`, plus rendered the confirmation chip in both layouts. + +**Pattern tag:** `ux-feedback` + +**Notes:** Worth a UX sweep when voice support extends to other fields — if we add voice to a new field, both states (collapsed/expanded, narrative/edit, etc.) need the affordance, not just the default one. + +--- + +## #6 — "Mission Street" couldn't match in the typeahead/voice path + +**Observed:** Voice path returned "Couldn't match those streets" for "14th Street and Mission Street" — even though both streets clearly exist and the parser split correctly. + +**Diagnosis:** `findStreetCandidates('Mission Street')` returned the first 5 streets starting with prefix "MISSION" sorted alphabetically: +``` +MISSION BAY BLVD NORTH +MISSION BAY BLVD SOUTH +MISSION BAY CIR +MISSION BAY DR +MISSION CREEK +``` +`MISSION ST` was alphabetically further down and got cut off. The intersection-lookup step then had no `MISSION ST` candidate to pair with `14TH ST`. Same risk for any common street name that has compound-name neighbors. + +**Fix:** added `rankedStreetCandidates()` in `server/lib/intersections.js`. Fetches a larger candidate pool, then ranks: streets whose `base` *exactly equals* one of the input's prefix candidates come first, then by base length ascending. So `MISSION ST` (base = "MISSION") now ranks ahead of `MISSION BAY BLVD ...` (base = full compound). Applied to both the typed path (`search`) and the voice path (`findStreetCandidates`). + +**Pattern tag:** `ranking-bug` + +**Notes:** This kind of bug is invisible in single-street-name tests — only surfaces when a real intersection lookup hits a popular base name with many compound siblings. **Worth adding integration tests** for the popular cases (Mission, Market, Geary, Van Ness — streets that have lots of named neighbors). + +--- + +## #7 — "14th admission" — STT fusion of "and Mission" + +**Observed:** User said "14th and Mission" → Transcribe heard "14th admission" → parser found no connector → chip said "Didn't sound like a cross-street." + +**Diagnosis:** AWS Transcribe is collapsing the bigram "and Mission" into the single token "admission" because they're phonetically near-identical (`/ænd ˈmɪʃən/` ≈ `/ædˈmɪʃən/`). Worst case among SF streets because the fusion produces a real English word — Transcribe's language model prefers "admission" over the unusual "and Mission." + +**Fix:** added `parseOrRecover()` in `server/lib/intersections.js`. When the direct connector parse fails, for each token try peeling a 1–3 char connector-like prefix (`AD`, `AND`, `AN`, `AT`, `N`); if the remainder is a known SF street base, split there. "14th admission" → token "admission" → peel "ad" → "mission" is in the street list → split as side1="14th", side2="MISSION". Wired into the typed `search` and the transcribe-route location-mode handler. Conservative — only triggers if the remainder is a *known* street base, so random words like "addiction"/"addition" don't false-positive. + +**Pattern tag:** `stt-fusion` + +**Notes (potential pattern → strategy revision):** This is the first STT-specific fusion fix. If more such cases pile up, the right strategy revision is probably **switching the location path to Deepgram Nova-3 with Keyterm Prompting** (per the Part 7 escalation rule in `cross-streets-implementation-plan.md`). Specifically, the threshold to revisit: if voice accept rate drops below ~80% in a measurable sample and the failures are dominated by proper-noun fusion, that's the cue. We don't have measurement yet — adding it is in Phase 7. Until then, the `parseOrRecover` recovery is a stopgap that covers the most common case (vowel-starting major streets after "and"). + +Other STT-fusion cases I'd watch for next: +- "X and Octavia" → ? +- "X at Irving" → ? +- "X and Owens" → ? +- "Howard and Sixth" → "Howard inserts" or similar +- "X & Y" said as "X 'n Y" → ? + +--- + +## #8 — "Third & Folsom" → "Third and fulsome" — ordinal word + Folsom misheard + +**Observed:** Voice "Third & Folsom" came through as `"Third and fulsome"`. The chip showed "Couldn't match those streets." + +**Diagnosis:** Two independent issues in one transcript: +1. AWS Transcribe outputs the English word "Third" (not "3rd"). The lookup couldn't bridge it to the DataSF canonical `03RD ST` — the digit-prefixed name has no meaningful phonetic encoding (Double Metaphone treats `0`/`3` as silent). So side1 failed with no candidates. +2. "Folsom" was heard as "fulsome" — but this side actually *would* have worked, because phonetic match catches it (`FLSM ↔ FLSM`, score 0.89). The failure was attributable entirely to side1. + +**Fix:** added `expandOrdinalWords()` in `server/lib/streetNormalization.js` mapping `First/Second/Third/.../Thirtieth` and compound ordinals (`Twenty-First`, `Twenty Fourth`) to their digit forms (`1ST/2ND/3RD/.../30TH/21ST/24TH`). Hooked into `toPrefixes` as the first step, so the existing zero-pad regex (`3RD → 03RD`) catches it next. Result: `findStreetCandidates('Third')` now returns `['03RD AVE', '03RD ST', '03RD TI ST']`. End-to-end: `"Third and fulsome"` → `3rd St & Folsom St`. + +Tests added for ordinal expansion (`expandOrdinalWords`) and ordinal-aware `toPrefixes`. + +**Pattern tag:** `stt-fusion` (specifically a "wrong-form transcription" subtype — the word is correctly recognized but in a form the lookup can't bridge) + +**Notes:** This was actually two latent bugs in one transcript, and only the visible side (Third) was blocking. The Folsom→fulsome part already worked via phonetic fallback — quiet confirmation that the matcher pipeline does handle vowel-shift cases when the prefix layer fails. Good sign for the strategy: phonetic fallback is doing real work, just gated by digit-vs-word in this case. + +**Strategy implication:** SF's numbered streets/avenues are everywhere. This won't be a one-off — *any* voice input that names a numbered street as a word ("Twenty Fourth and Mission", "Nineteenth and Taraval") would have failed before this fix. Worth verifying with usage data that the ordinal map covers what officers actually say. + +--- + +## Pattern summary so far + +- **`data-shape-assumption`** (3 occurrences): #1, #2, #3. Each was a code path that assumed address-only and didn't know about the locationType discriminator. **If another shows up, lean on a typed read-side wrapper** (e.g. `getIncidentLocation(incident) → { type: 'address'|'intersection', text, lat, lng }`) and migrate call sites to it rather than continuing inline branching. +- **`ranking-bug`** (1 occurrence): #6. Test coverage gap — popular base names with compound siblings need integration tests. +- **`stt-fusion`** (2 occurrences): #7, #8. Both handled with focused fixes in the normalization/recovery pipeline. **If 3+ more such fixes pile up, escalate the STT strategy** to Deepgram Nova-3 per the implementation plan's Part 7 gate. #8 in particular suggests a more general principle: anywhere a canonical name has a non-phonetic transformation from speech (digits ↔ words, abbreviations, leading "THE"), we need explicit normalization — phonetic match alone won't bridge it. +- **`ux-feedback`** (2 occurrences): #4, #5. Both were my misses, not systemic. Manual UX review would have caught both before user found them. + +--- + +## Strategy reconsiderations parked for later + +(These are *not* changes to make now — they're flagged for revisit once the running log shows enough signal to act.) + +- **Single source-of-truth read wrapper for incident location.** Promote `incidentLocationText` and friends into a typed helper that all consumers (server formatters, completeness checks, PDFs, list responses) use. Currently they each branch on `locationType` themselves. If we collect a 4th `data-shape-assumption` entry, do this. +- **Switch location path STT to Deepgram Nova-3.** Currently using AWS Transcribe (shared with the narrative path). If we collect 3+ `stt-fusion` entries OR get measurement showing <80% voice accept rate on locations, escalate. +- **Integration test fixture for popular base-name lookups.** Add tests for "& Mission", "& Market", "& Geary", "& Van Ness", "& Castro" — all streets with many compound siblings. diff --git a/docs/cross-streets.md b/docs/cross-streets.md new file mode 100644 index 00000000..d0727ad2 --- /dev/null +++ b/docs/cross-streets.md @@ -0,0 +1,548 @@ +This is a research/investigation project. +- Currently, the app asks for an Incident address. The user must ultimately input a street address (though we offer a few kinds of assistance: we try to reverse-geocode your device location to produce an address automatically; and we offer type-ahead assistance against a database of addresses in SF.) +- Our users in the FIELD role are police officers. They are very accustomed to reporting a location in terms of cross-streets, rather than a numbered street address. E.g. "16th & Valencia". Having to find and input a nearby street address adds friction to the workflow. + +So the question is: is there an effective and delightful way to honor this preference? + +This change would be non-trivial. We'd first need to investigate the current codebase: +- Do we currently store address information in a way that assumes a street address? +- What are all the ways that we use/validate the stored address? If we changed the storage, those might need to change. + +This is also not a novel problem, so we should explore existing best practices used by other applications that need to support 'address' input and need to support both traditional addressses and cross-streets. + +Then there are some UX questions that arise: +- If we're giving the user the option to input either a street address or cross-streets, what does the UX look like? Do we offer an explicit choice? Is there a way to offer a unified input and simply interpret the user's input in an intelligent way? (Note that the input is already a bit complex here - I think it's sometimes displayed a single text field, but sometimes broken out into into one field per address-part; I'm not sure exactly what triggers the change.) +- We'd probably want to use voice input to allow the officer to speak cross-streets. If we do this, is there a technique we can use to improve accuracy by matching against real cross-streets? For example, a really naive approach would be to first translate the speech to text, then use the text to search a cross-street database of some kind. But this might result in a transcription where the officer says "Junipero Serra & Monterey" and this gets transcribed incorrectly as "Junipero Sarah & Monterey", which could subsequently fail when we search it. Are there techniques for matching this against a list of known cross-streets? (I can imagine that either you get the transcription service to return multiple candidates, and you pick the best one; or maybe we can provide a "vocab" list of all the street names in SF, and ask the transcriber to try and match to those? Or open to other ideas.) + +Just noting that this project already uses a couple of external services that could be relevant. We use voice transcription (through AWS, I think) and geolocation/reverse-geocoding (also from AWS IIRC). So we may get some mileage out of those existing services when tackling this problem. + +Please investigate this question thoroughly, both investigating the current codebase and also doing any public research that will help us understand best practices and technical options here. Then put on your designer hat and present design options (you can share which way you lean on each decision, if you have a strong preference.) Then produce a detailed written summary that we can review together in order to build an implementation plan for the feature. + +--- + +# Investigation & Design Summary + +This section is the findings synthesis. It is written to feed an implementation-planning conversation, not as the plan itself. Companion document with full external sources and citations: `docs/cross-streets-research.md`. + +Author's POV: where I have a strong preference I say so and explain why. Treat the "recommended" calls as starting positions to push back on, not foregone conclusions. + +--- + +## Part 1 — How the current app handles location + +### 1.1 Storage (`server/prisma/schema.prisma`) + +The `Incident` model embeds a single address inline (no separate `Address` table). All seven fields are nullable in the DB but several are required by the client validator: + +| Field | Type | Client-required | Notes | +| --- | --- | --- | --- | +| `addressLine1` | `String?` | ✅ (min 2) | Street address, e.g. "425 16th St" | +| `addressLine2` | `String?` | optional | Apt/suite | +| `city` | `String?` | ✅ (min 2) | | +| `state` | `String?` | ✅ (min 2) | | +| `postalCode` | `String?` | | 5-digit, stripped from ZIP+4 | +| `latitude` | `Decimal(9,6)?` | | Stored as decimal | +| `longitude` | `Decimal(9,6)?` | | Stored as decimal | + +The same shape is used on `Facility` (with neighborhood/district extras) and `Subject` (PII-flagged). + +Server-side Zod schema (`server/models/incident.js`) mirrors the columns but does not enforce required-ness — that lives only in the client (`client/src/utils/validators.js:40-50`). + +### 1.2 Address input UI + +- `client/src/components/AddressAutocomplete.jsx` — single-text-field typeahead. Debounces 300ms, hits `Api.geocode.search()` → `/api/geocode/search`. On selection, fills six fields at once via `form.setValues({ addressLine1, city, state, postalCode, latitude, longitude })` (lines 75-82). +- `client/src/lesc/components/IncidentForm.jsx:262-333` — two visual states: + - **Collapsed**: read-only `TextInput` showing `formatAddress(...)` plus a "use current location" button. + - **Expanded**: `AddressAutocomplete` for `addressLine1`, plus separate `TextInput`s for `addressLine2`, `city`, and a side-by-side `Group` for `state`/`postalCode`. + - Switching is triggered by clicking/focusing the collapsed field (`setShowAddressForm(true)`). +- Note: the "single field vs split fields" the prompt asked about is *not* a different input mode — it's the same form, collapsed vs expanded. The collapsed view just *displays* the comma-joined parts. + +### 1.3 Geocoding backend + +- `server/lib/location.js` — wraps `@aws-sdk/client-geo-places` (AWS Location Service v2). +- Endpoints: + - `GET /api/geocode/search?text=…` (`server/routes/api/geocode/search.js`) — calls AWS `Suggest` with a hard-coded SF bounding box `[-122.5155, 37.7080, -122.3570, 37.8120]`, max 5 results, returns `{ placeId, label, addressLine1, city, state, postalCode, neighborhood, latitude, longitude }`. Auth required. + - `GET /api/geocode/reverse?latitude=…&longitude=…` — calls AWS `ReverseGeocode`, returns `{ addressLine1, city, state, postalCode }`. Currently does NOT require auth. +- Reverse-geocoding from device location: `client/src/utils/geocoding.js` → `getCurrentLocationAddress()` calls `navigator.geolocation.getCurrentPosition()` then hits `/api/geocode/reverse`. Auto-fires once on new-incident open if there is no existing address (`IncidentForm.jsx:147-154`). +- **There is no vendored local address database.** Everything is AWS. + +### 1.4 Read/use sites for the address + +| Site | What it does | +| --- | --- | +| `Incident.jsx` | Renders `addressLine1` + `, addressLine2` on summary cards | +| `CustodyDetailContent.jsx:188` | Renders `formatAddress(incident)` on detail page | +| `utils/format.js:7-9` | Joins all six visible parts with commas | +| `components/facilityAddressLink/` | Builds Google/Apple Maps URL from `formatAddress(...)` | +| `server/lib/forms/5150/generate.js:20-22` | Fills "and residing at" on 5150 PDF using **subject** address only | +| `server/lib/forms/849b/` | Does **not** use address fields | +| Handoff (`routes/api/deflections/handoff.js`) | Does not include address | + +There is currently **no address-based search/filter** anywhere in the app, and **no use of lat/lng beyond storage and map links**. + +### 1.5 Voice transcription precedent + +- Server: `server/routes/api/ai/transcribe.js` uses **`TranscribeStreamingClient` + `StartStreamTranscriptionCommand`**, English, 16kHz PCM, region `us-west-2`. Already has a hook for custom vocabulary: `commandParams.VocabularyName = process.env.AWS_TRANSCRIBE_VOCABULARY_NAME` (lines 53-64). Currently takes only the first alternative (line 71). No n-best. +- Client: `client/src/components/AudioRecorder.jsx` — captures 16kHz mono PCM, downsamples to `Int16Array`, base64-encodes, POSTs to `/api/ai/transcribe`. Max 180s. +- Used today only for the **Narrative field** (`DeflectionForm.jsx:146-150`). Append-to-field semantics: subsequent recordings concatenate with a space. + +This is the precedent we'd extend for cross-street voice input. + +### 1.6 What this tells us before designing + +1. **The schema is not deeply opinionated about "street address."** The seven columns happen to model one, but nothing downstream other than the 5150 PDF (which uses the *subject* address, not the incident address) requires a fully-formed civic address. PDFs, displays, and map links all use string concatenation. **This means we have real freedom in how we represent intersections in storage.** +2. **The address autocomplete is a single text field already.** UX-wise, that's the same affordance you want for an "address-or-intersection" unified input. The expanded edit view is the only place that assumes structured-address parts. +3. **AWS Transcribe is already wired with a vocabulary hook** but the n-best response path is not. The custom vocabulary table itself does not exist yet (env var is unset by default). +4. **No tests pin down cross-street behavior** — that's a clean slate. + +--- + +## Part 2 — External landscape (condensed) + +Full sources in `docs/cross-streets-research.md`. The five findings that matter most for design: + +1. **Intersection geocoding is a first-class concept everywhere except OSM.** Google Maps, Mapbox, Apple Maps, Uber, Lyft, Mark43, PremierOne, Hexagon, Tyler, RapidSOS all support it. Esri formalizes the connectors: `&`, `/`, `@`, `\`, `|`, plus `and`/`at` (US locale). +2. **AWS Location Service v2 supports intersections — but the *forward* path is documentation-light.** Response shapes have first-class `Address.Intersection: [string]` and `PlaceType: "Intersection"`. The well-documented case is the reverse-geocode "nearest intersection" feature (`AdditionalFeatures: ["Intersections"]`), which AWS explicitly markets for emergency services. **We need to empirically bench-test forward intersection queries on SF before relying on them.** +3. **DataSF publishes free, authoritative SF intersection + street-name datasets** (`ctsg-7znq`, `pu5n-qu5c`, `sw2d-qfup`, `3psu-pn9h`). ~2k street names, ~10–15k intersections. Small enough to vendor a static index. This is the strongest fallback if AWS coverage is patchy, and it doubles as the source of the STT custom vocabulary. +4. **AWS Transcribe Custom Vocabulary IPA/SoundsLike columns are silently deprecated for non-medical use.** This is the gotcha. We cannot teach Transcribe that "Junipero" sounds like "hoo-NEE-pear-o" directly. The fix is post-hoc phonetic match. If proper-noun accuracy stays bad after that, **Deepgram Nova-3 with Keyterm Prompting** is materially stronger (100 multi-word terms, ~90% keyword-recall improvement, streaming-friendly). +5. **The dominant UX pattern is a single text field with mode auto-detected from the connector.** No mode toggle. User types `&` or ` and ` → suggestion list flips to intersection results. Critically, CAD systems do this with **no street suffixes** in intersection mode (`BOSTON & COOK`, not `BOSTON ST & COOK ST`) — our matcher must handle that. + +Two surprises worth flagging: + +- **Uber added cross-streets for *privacy*, not navigation** — the UX we want to copy was validated against a completely different motivation, which is actually a reassuring signal that it's robust. +- **The phonetic failure mode is concentrated.** Most SF streets transcribe fine. The problems are the ~50 Spanish-/Irish-/historic-origin proper nouns: Junipero Serra, Cesar Chavez, Embarcadero, Geary, Masonic, Bayshore. A small curated correction dictionary may outperform the full custom-vocabulary approach. + +--- + +## Part 3 — Design options & recommendations + +There are four orthogonal decisions. I lay out options for each, then assemble the recommended end-to-end design. + +### Decision A — Storage shape + +**Option A1 — Discriminator + structured intersection columns** *(my recommendation)* + +Add two nullable columns and a discriminator: + +``` +addressLine1 String? (unchanged) +addressLine2 String? (unchanged) +city String? (unchanged) +state String? (unchanged) +postalCode String? (unchanged) +latitude Decimal? (unchanged) +longitude Decimal? (unchanged) + +-- new -- +locationType enum('ADDRESS','INTERSECTION') DEFAULT 'ADDRESS' +street1 String? +street2 String? +``` + +Semantics: + +- `locationType = ADDRESS` (default, including all historical rows): existing six fields used. `street1`/`street2` null. +- `locationType = INTERSECTION`: `street1`, `street2`, `city`, `state`, `latitude`, `longitude` populated. `addressLine1`/`addressLine2`/`postalCode` null. + +Pros: +- Migration is a no-op for existing rows — they already mean "address" because `addressLine1` is set. We can backfill `locationType = 'ADDRESS'` with a single SQL update; intent of historical rows is preserved verbatim (the user's stated constraint). +- Structured intersection data is queryable, exportable, and easy to render. We can match `street1`/`street2` against the SF street dictionary for normalization. +- Display code can branch cleanly: `if locationType=INTERSECTION then "${street1} & ${street2}" else formatAddress(...)`. +- Lat/lng remains the universal join point for maps and EIDD-style export (CLDXF-US doesn't have a clean "intersection-only" mode anyway — store both). +- The 5150 PDF problem (which uses *subject* address, not incident) is unaffected. If incident location ever feeds a PDF, the formatter has an easy branch. + +Cons: +- Schema migration required. ~3 new fields, 1 enum. +- `formatAddress()` and every display site need branching logic. Manageable — `format.js` is one file. + +**Option A2 — Reuse `addressLine1` as a free-text "location" field** + +Stop treating `addressLine1` as a structured street address. Let it hold either `"425 16th St"` or `"16th & Valencia"`, with `latitude`/`longitude` as the source of truth for downstream consumers. + +Pros: Zero schema change. + +Cons: Lossy. Loses the ability to programmatically distinguish address from intersection, to canonicalize street names, or to ask "is this intersection valid?" The intent of historical rows is preserved literally but ambiguously. Every downstream consumer that wants to do anything beyond "render a string" has to re-parse `addressLine1`. **I'd avoid this.** + +**Option A3 — Polymorphic `Location` table** + +Extract a separate `Location` table with a discriminator and join from `Incident`/`Facility`/`Subject`. + +Pros: Cleanest long-term shape. + +Cons: Large refactor across three models for a feature that only needs new shape on `Incident`. Premature. We can do this later if needed. + +**My call:** A1. Minimal schema change, no data loss, queryable, backward compatible. + +### Decision B — Input UX + +**Option B1 — Unified single field with auto-detected mode** *(my recommendation)* + +The existing collapsed/expanded pattern stays. The differences: + +- **Collapsed view**: still read-only. If location is an intersection, render `16th St & Valencia St` (with a small chip indicating "intersection"); if address, render existing `formatAddress(...)`. Same "use my location" button on the right. +- **Expanded view, single primary input**: replace `AddressAutocomplete` with a slightly smarter `LocationAutocomplete` that: + - Accepts free-text input. + - Detects the intersection connectors (`&`, ` and `, ` at `, `/`, `@`) client-side. + - When detected, switches the backend call to intersection-search and the suggestion list to intersection results. + - When not detected and input starts with a digit, behaves exactly like today's address autocomplete. + - On selection, fills *either* the address fields *or* the intersection fields, and sets `locationType` accordingly. The other set is cleared. +- **Expanded view, contextual extras**: + - If intersection mode is detected: show optional `city` (defaulted to "San Francisco") and a read-only confirmation chip showing the two normalized street names. + - If address mode: show today's `addressLine2`/`city`/`state`/`postalCode` row. + +ASCII sketch: + +``` +Collapsed (address): +┌─ Location ──────────────────────────────────────────┬───┐ +│ 425 16th St, San Francisco, CA 94103 │ ⌖ │ +└─────────────────────────────────────────────────────┴───┘ + +Collapsed (intersection): +┌─ Location ──────────────────────────────────────────┬───┐ +│ 16th St & Valencia St · San Francisco [✕] │ ⌖ │ +│ ↑ small "intersection" chip │ │ +└─────────────────────────────────────────────────────┴───┘ + +Expanded — typing "16th &": +┌─ Location ──────────────────────────────────────────┬───┐ +│ 16th &| │ ⌖ │ +├─────────────────────────────────────────────────────┴───┤ +│ Suggestions (intersection mode auto-detected): │ +│ ⌗ 16th St & Valencia St │ +│ ⌗ 16th St & Mission St │ +│ ⌗ 16th St & Guerrero St │ +│ ⌗ 16th St & Folsom St │ +└─────────────────────────────────────────────────────────┘ + +Expanded — typing "425 ": +┌─ Location ──────────────────────────────────────────┬───┐ +│ 425 16| │ ⌖ │ +├─────────────────────────────────────────────────────┴───┤ +│ Suggestions (address mode): │ +│ 📍 425 16th St, San Francisco, CA │ +│ 📍 425 16th Ave, San Francisco, CA │ +│ 📍 4251 16th St, San Francisco, CA │ +└─────────────────────────────────────────────────────────┘ +``` + +Pros: Echoes the validated Google/Uber/Mapbox pattern. Zero new taps for officers. Auto-detection is trivial client-side regex, no ML. + +Cons: Officers need to know the convention (`&` or `and`). That's the universal radio/CAD convention so it's already familiar; we can show a one-line hint under the field on first use. + +**Option B2 — Explicit toggle (Address | Intersection)** + +A segmented control switches the form between address-input and intersection-input modes. + +Pros: Less magic. No ambiguity. + +Cons: Adds a tap and a decision before typing. Diverges from every consumer-grade pattern. **I'd avoid this** unless usability testing shows officers struggle with B1. + +**Option B3 — "Add intersection" as a secondary affordance** + +Address is primary; intersection is an "+ add intersection" link below. + +Cons: Tells the officer "your preferred input is the secondary one." Backwards. + +**My call:** B1. Mirror the consumer pattern, keep the existing collapsed/expanded chrome. + +### Decision C — Voice input + +**Option C1 — Extend the existing AudioRecorder onto the Location field, with a vocabulary + post-hoc fuzzy match** *(my recommendation)* + +Concretely: + +1. **Build an AWS Transcribe Custom Vocabulary** from DataSF `3psu-pn9h` (all SF street names, current + retired aliases). Upload it via the Transcribe console or a one-shot script. Set `AWS_TRANSCRIBE_VOCABULARY_NAME` in env. +2. **Reuse `AudioRecorder` on the Location field**, swapping the `onTranscription` handler so the result feeds the location parser instead of appending to a narrative. +3. **Server-side post-processing** (in `routes/api/ai/transcribe.js` or a new route specialized for location): after Transcribe returns, tokenize on connector words/symbols, then for each token compute Double Metaphone (or NYSIIS) and match against the pre-computed phonetic index of SF streets. Pick the best candidate per side; return the structured `{ street1, street2 }` *plus* the raw transcript. +4. **Show a confirmation chip** before geocoding: `Heard: "Junipero Serra & Monterey" → 16th St… no wait → Junipero Serra Blvd & Monterey Blvd · tap to confirm`. CAD systems do this. It's cheap, prevents silent misroutes. +5. **N-best path**: not available in Transcribe streaming. If the fuzzy-match top candidate's confidence is low, fall back to a synchronous batch retranscription request with `ShowAlternatives=true, MaxAlternatives=5`, then try each alternative through the same matcher. Only do this on low-confidence first pass. + +Pros: +- We keep AWS Transcribe; one streaming round-trip in the happy path. +- DataSF vocab + Double Metaphone fixes the *exact* failure mode the prompt called out ("Junipero Sarah" → "Junipero Serra"). +- Zero new vendor relationship, no new credentials. +- Built on `AudioRecorder`, which is already in production for narrative dictation. + +Cons: +- IPA/SoundsLike hints don't help us (deprecated). The fuzzy step is where the magic happens, not in Transcribe itself. +- Real-world accuracy on the hard street names is unknown until measured. + +**Option C2 — Switch the location flow to Deepgram Nova-3 with Keyterm Prompting** + +Pros: Strongest in-class biasing toward a known lexicon (claimed ~90% keyword-recall lift; multi-word terms; preserves casing). Streaming-friendly. + +Cons: New vendor, new credentials, separate SDK alongside AWS Transcribe (because we'd keep AWS for narrative). + +**My call:** C1 first. Instrument it (log every transcribe → match → confirm/correct cycle). If we hit a measured failure-rate floor we can't get below — say >15% of voice-entered locations need manual correction — escalate to C2 for the location path only. + +**Option C3 — Skip voice for v1** + +Pros: Smaller scope. + +Cons: The whole point of the feature is field ergonomics; voice is the highest-leverage piece. **I'd avoid this.** + +### Decision D — Reverse-geocoding from device location + +We currently auto-reverse-geocode on new incident open and offer a button to re-fetch. With intersections in the picture: + +**Option D1 — Offer "nearest intersection" alongside "nearest address" as picker** *(my recommendation)* + +When the location button is tapped, fire two AWS calls in parallel: +1. `ReverseGeocode` → nearest address (today). +2. `ReverseGeocode` with `AdditionalFeatures: ["Intersections"]` → nearest intersection(s). + +Show a small picker: + +``` +We located you near: + ○ 425 16th St [address] + ● 16th St & Valencia St [intersection] + ↑ default-selected for FIELD users; default-selected per user preference otherwise +``` + +The default-selected option respects user role: FIELD users default to intersection, others to address. After v1, learn from individual user behavior. + +Pros: Acknowledges that officers prefer intersections without forcing the choice every time on every other user. Costs one extra cheap AWS call. + +Cons: One extra UI element. We can keep it visually minimal. + +**Option D2 — Always reverse-geocode to intersection for FIELD role** + +Pros: Lowest UI friction. + +Cons: Forecloses on cases where the officer wants a precise address (e.g., a specific apartment building). + +**My call:** D1. Cheap, respects preference, easy to evolve. + +### Decision E — "Between" context (free win) + +Independent of the above: whenever we resolve an address, we can also fetch the nearest two intersections and surface them as read-only context. + +``` +425 16th St +between Valencia St and Guerrero St +``` + +This is a CAD convention, costs one extra reverse-geocode per address, and is materially useful for officers reading a location over radio. **Recommend including in v1.** + +--- + +## Part 4 — Recommended end-to-end design (the picks, assembled) + +> **⚠ Revised after bench testing.** The original recommendation here had AWS Location Service as primary with DataSF as fallback. Empirical testing (see **Part 8** below) showed AWS forward intersection geocoding is unreliable for SF — silent-fail rate ≥27%, with dangerous misparses ("16th & Mission" → "house number 16 on Mission St"). **DataSF is now primary; AWS is out of the cross-street path entirely.** The other decisions are unchanged. + +Storage: **Option A1** — discriminator + `street1`/`street2`. Backfill all existing rows to `locationType = 'ADDRESS'`. + +UX: **Option B1** — unified single field, mode auto-detected from connectors. Same collapsed/expanded chrome as today. + +Voice: **Option C1** — extend `AudioRecorder` onto the Location field, **visible on the collapsed view** so officers discover it. AWS Transcribe with a DataSF-derived Custom Vocabulary plus server-side Double-Metaphone fuzzy match against the SF street list. Show a confirmation chip before commit. Measure; reserve Deepgram as the escalation path. + +Reverse geocoding from device location: **dropped** — when the officer uses device location to populate the field, we record a traditional address (today's behavior, unchanged). No two-result picker. + +"Between" context: include in v1. + +Geocoding source (**revised**): **vendored DataSF intersection index as primary.** No AWS Location calls for intersection input. AWS Location continues to serve the existing traditional-address autocomplete and current-location reverse geocoding paths. + +Scope: this feature changes the form for **FIELD-role users only** (confirmed: `/incident` is gated to `UserRole.FIELD` in `AppRedirectsConfig.jsx:14-16`). No settings toggle; the unified-field UX is the new default for everyone who can reach the form. + +Standards posture: store both the structured `{ street1, street2 }` and lat/lng on every intersection-mode incident. That's all we need to be EIDD-compatible later, without committing to EIDD now. + +--- + +## Part 5 — Open questions for the implementation plan + +These are the calls I'd want to make jointly before writing code, in rough priority: + +1. **Does AWS Location v2 actually forward-geocode SF intersections well?** This is the single biggest unknown. Concrete test: pick 30 SF intersections (a mix of major arterials, alley intersections, Spanish-name corners, Mission grid, Sunset numbered grid) and bench-test AWS `Geocode` + `Autocomplete` against them. If hit rate is below ~95%, lean on DataSF as primary rather than fallback. +2. **Does FIELD role get a different default, or does every user get a "show me intersections too" toggle in settings?** I leaned role-based; if the role split matters operationally, this becomes a tickbox rather than a default. +3. **5150 PDF behavior when incident location is an intersection.** Today the 5150 uses subject address (not incident). If that changes — or if SFSO wants the incident location on a future form — we need to decide: render `16th St & Valencia St` literally, or refuse to fill the field and show a warning, or require the officer to add a `addressLine1` even when the primary location is an intersection. **My lean:** render the intersection string literally; PDF fields are free-text, and "intersection of 16th & Valencia" is acceptable to dispatch downstream. +4. **Vocabulary scope.** DataSF lists ~2k SF street names plus ~hundreds of aliases. AWS Transcribe's "best results" target is ~300 words; we'd need to either split into focused sub-vocabularies (downtown, west side, etc.) or accept that all 2k go in one table and measure. The blog-post precedent suggests 2k is fine in practice — the 300 number is a soft recommendation, not a hard limit. **My lean:** single vocabulary with all street base names (no suffix variants), measure, split if needed. +5. **Where does the DataSF data live?** Options: (a) check the JSON into the repo, refresh via a script; (b) server-side cache populated at deploy time; (c) S3-hosted JSON the client fetches once and caches. **My lean:** (a) for v1 — simplest, no infra, file is small (well under 1 MB gzipped). +6. **Voice on the collapsed-Location chrome.** Do we expose the mic from the collapsed view, or only when expanded? **My lean:** collapsed view shows the location button (current location) only; expanding reveals the mic. Officers will want voice but should see the field they're dictating into, for the confirmation chip. +7. **Subject address.** This research focused on the incident location. The same officer ergonomics arguably apply to the *subject's* address ("they live near 16th & Valencia"). My lean: out of scope for v1 — subject address is more often a literal residence than a known intersection. Revisit after launch. +8. **Tests.** No existing tests pin down cross-street behavior, which is freeing. We should add: validator tests for both modes, autocomplete-detection tests, a Transcribe-mock test that proves the fuzzy-matcher fixes the Junipero Sarah case. + +--- + +## Part 6 — Risk register + +| Risk | Likelihood | Impact | Mitigation | +| --- | --- | --- | --- | +| AWS forward intersection geocoding has poor SF coverage | Medium | High | Pre-flight bench test. If poor, swap primary to DataSF index. | +| Custom Vocabulary doesn't materially improve "Junipero Serra" recognition | High | Medium | Phonetic fuzzy-match step is the actual fix; vocab is supporting. | +| Officers don't discover the `&` / `and` convention | Low | Low | Single-line hint under the field on first incident; help-tip icon. | +| Existing data quietly relied on `addressLine1` always being set | Low | Medium | Add a query check during migration; backfill enum to ADDRESS for all rows. | +| Mantine uncontrolled-form gotcha re-bites us (memory note) | Medium | Low | Same defaults-vs-values pattern applies; reuse the pattern that worked. | +| 5150 PDF expects addressLine1 (it does, for *subject* only — incident is unaffected) | Low | Low | Behavior already in scope of Part 5 Q3. | + +--- + +## Part 7 — What's NOT in this proposal (deliberate omissions) + +- **Multi-jurisdiction support.** SF-only per the scoping conversation. The pluggable seams (vocabulary source, intersection index, bounding box) are visible in the design but not built. +- **what3words.** Mentioned in the research as a real PSAP modality, but adds a third input type with marginal value over address+intersection for our user base. Defer. +- **LLM/Bedrock natural-language parsing** ("near 16th and the BART"). Promising future enhancement; not v1. +- **Refactoring address out of `Incident` into a polymorphic `Location` table.** Right long-term shape, wrong scope for this feature. +- **Subject/Facility address cross-street support.** v2 candidate. + +--- + +End of initial investigation. See Part 8 for the bench-test follow-up that revised Part 4. + +--- + +## Part 8 — Bench-test findings: AWS forward intersection geocoding for SF + +Bench script: `server/scripts/bench-intersections.js`. Run via `node scripts/bench-intersections.js` from the server workspace. + +### Method + +- 30 SF intersections, hand-selected across 6 categories: major arterial (English), Mission/SoMa numbered grid, west-side avenues, Spanish-origin names, alley/minor streets, edge/weird (Panhandle, Lombard curvy block, etc.). +- 4 input format variations per intersection: `X & Y, San Francisco, CA`, `X and Y, San Francisco, CA`, `X St & Y St, San Francisco, CA` (suffix appended where bare), and `X & Y` (bare, no city). +- 2 AWS endpoints per query: `GeocodeCommand` and `AutocompleteCommand`. 240 queries total. +- Bias position: SF center. Bounding box: SF only. + +### Results + +**`AutocompleteCommand` could not be tested** — the `care-connect-location-dev` IAM user lacks `geo-places:Autocomplete` permission. Not blocking the architecture decision (see analysis), but noted as a known gap. + +**`GeocodeCommand` returns *something* on 100% of queries.** That's the problem — the something is silently wrong on 27–73% of queries depending on format. Failure modes seen: + +| Failure mode | Example | +| --- | --- | +| Returns one street only | `Maiden Lane & Grant, SF` → "Maiden Ln" (no intersection) | +| Reads numbered street as house number | `16th & Mission, SF` → "16 Mission St, SF 94105-1227" (~3 mi off) | +| Returns nearby POI | `Market & Powell, SF` → "BART/MUNI - Powell Street Station, 899 Market St" | +| Returns wrong city | `Lombard & Hyde, SF` with `&` → "Lombard district, American Canyon, CA" | +| Returns wrong street | `Bayshore St & Cortland St` → "Bayshore Fwy, South San Francisco" | + +**Intersection-typed hit rate by input format (across all 30):** + +| Format | Intersection hits | +| --- | --- | +| `X & Y, SF, CA` | 8 / 30 (27%) | +| `X and Y, SF, CA` | 11 / 30 (37%) | +| `X St & Y St, SF, CA` | **22 / 30 (73%)** | +| `X & Y` (no city) | 9 / 30 (30%) | + +**By category, using the best-case `withSuffix` format:** + +| Category | Intersection hits | Notes | +| --- | --- | --- | +| Mission/SoMa | 5/5 (100%) | The numbered-grid case. Suffix fix is decisive. | +| Arterial | 3/5 (60%) | Powell BART POI wins over Market & Powell; Geary/Masonic breaks when "St" appended (canonical is Blvd/Ave). | +| Spanish | 4/5 (80%) | Bayshore Blvd → "Bayshore St" breaks it. Junipero Serra works fine. | +| Alley | 4/5 (80%) | Maiden Lane never resolves as intersection. | +| Edge | 4/5 (80%) | Crestline & Twin Peaks Blvd never resolves (AWS has no node). | +| West-side avenues | 2/5 (40%) | Worst category. "California & 25th Ave" → "25 California St." | + +### Why 73% isn't a usable ceiling + +1. **Silent failures are dangerous.** Officers see a plausible address; nothing flags that it's wrong. "16 Mission St" is several miles from the actual "16th & Mission." Lombard & Hyde → American Canyon is the canonical example. +2. **The 73% rate requires suffix normalization that AWS itself doesn't provide.** "St" only works for some streets. To get there we need a per-street canonical-suffix dictionary — which means we need DataSF anyway. At that point, doing the actual intersection lookup in DataSF instead of round-tripping to AWS is strictly simpler. +3. **The `&` vs `and` distinction is non-deterministic.** Lombard & Hyde fails with `&`, succeeds with `and`. We can't pick a single connector and trust it. +4. **Some categories are structurally broken.** West-side avenues (40%) and Crestline-type edge cases never resolve regardless of normalization. AWS doesn't model them. + +### Architecture impact + +The original Part 4 recommendation had AWS as primary intersection source with DataSF as fallback. **Revised:** + +- **DataSF as primary.** Vendor a static intersection index (`ctsg-7znq` / `sw2d-qfup` from DataSF Socrata; ~10–15k entries, lat/lng-tagged, canonical street names + suffixes). Client-side typeahead matches directly. Zero AWS cost in the cross-street path. No silent failures — if the intersection isn't in the index, we surface "no match" honestly. +- **AWS Location stays on the traditional-address path only** — today's autocomplete behavior and reverse-geocoding from device location are unchanged. +- **No AWS Geocode for intersections at all.** Eliminates a class of silent geographic errors. + +This is also simpler architecturally: no suffix-normalize-and-retry pipeline against AWS, no new IAM permission, no Storage-tier pricing concerns for AWS Geocode results, no cross-validation logic between two sources of truth. + +### Remaining unknowns + +1. **Does DataSF cover all 30 of our test intersections with sensible lat/lng?** Next step. If DataSF also misses Crestline / Twin Peaks Blvd, we have a bigger problem and may need to fall back to OSM-derived data for that long tail. +2. **Should AWS Autocomplete play a sanity-check role?** Open. We could enable the IAM permission later and add a non-blocking "AWS disagrees" warning. Park for now. +3. **What canonical name should we display?** DataSF data uses uppercase, suffix-included form ("MARKET ST", "VALENCIA ST", "JUNIPERO SERRA BLVD"). For display we'd title-case and may want to strip suffixes in compact contexts ("16th & Valencia") while preserving them in detail views ("16th St & Valencia St"). Format choice to nail down in implementation. + +--- + +## Part 9 — Bench-test findings: DataSF coverage of SF intersections + +Bench script: `server/scripts/bench-datasf-intersections.js`. No credentials needed — DataSF Socrata is open. + +### Method + +Same 30 intersections as Part 8. For each, query the **`jfxm-zeee`** dataset ("Intersections by Each Cross Street Permutation") via the Socrata SODA API: + +``` +https://data.sfgov.org/resource/jfxm-zeee.json? + $where=starts_with(upper(street_name_1), 'X') AND starts_with(upper(street_name_2), 'Y') +``` + +Query both `(a, b)` and `(b, a)` orderings, dedupe by CNN. Total dataset size: **21,058 permutation rows** (~10–11k unique intersections). + +### Normalization gotchas discovered + +Per-intersection failures on the first pass revealed three normalization rules that any production implementation needs to handle: + +1. **Long-form suffix expansion.** DataSF uses abbreviated suffixes: `LANE → LN`, `STREET → ST`, `AVENUE → AVE`, `BOULEVARD → BLVD`. User input may use either form. Map long forms to short before query. +2. **Leading "THE".** DataSF stores **"THE EMBARCADERO"** as the canonical name, not "EMBARCADERO". When user types "Embarcadero" we need to try both prefixes. +3. **Compound vs split spelling.** **"BAYSHORE"** is stored as **"BAY SHORE"** (two words) in DataSF. This is the kind of edge case that needs an explicit alias map; "Bayshore" → "Bay Shore" is a known SF-specific quirk. + +Also discovered: DataSF zero-pads single-digit numbered streets (`3RD → 03RD`, `7TH → 07TH`). Predictable but needs the transformation. + +### Schema reference (`jfxm-zeee`) + +| Column | Example | Notes | +| --- | --- | --- | +| `id` | `236753` | Row ID | +| `cnn` | `24183000` | Centerline Network Number (SF canonical intersection ID) | +| `street_name_1` | `16TH ST` | Full name with suffix | +| `street_name_2` | `VALENCIA ST` | Full name with suffix | +| `latitude` | `37.76491732…` | Direct lat/lng — no separate geocode needed | +| `longitude` | `-122.4218863…` | | +| `zip_code` | `94103` | | +| `x_coord`, `y_coord` | … | NAD83 / state plane; ignore | + +Every intersection appears twice in the table (A→B and B→A) — useful when joining, but in this query layer we dedupe on `cnn` to count one intersection once. + +### Results (after normalization fix) + +**28/30 exact match (93%) · 1 multi-candidate (both correct) · 1 genuine miss.** + +| Category | Coverage | Notes | +| --- | --- | --- | +| Arterial | 5/5 | | +| Mission/SoMa | 5/5 | | +| West-side avenues | 5/5 | The category that AWS choked on. DataSF handles it cleanly. | +| Spanish-origin | 5/5 | After "THE" + "BAY SHORE" fixes. Junipero Serra resolves directly. | +| Alley | 5/5 | | +| Edge | 3/5 + 1 multi | Fell & Stanyan returns two adjacent nodes (FELL ST & FELL ACCESS RD). Crestline & Twin Peaks Blvd genuinely doesn't exist. | + +### The one genuine miss + +**Crestline & Twin Peaks Blvd**: DataSF shows Crestline Dr's intersections as Parkridge Dr, Burnett Ave, Vista Ln, and a dead end — *not* Twin Peaks Blvd. The two roads run near each other on Twin Peaks but appear not to share a node in the SF centerline graph. **My test case was wrong**, not DataSF's data. If officers ever report a location on Crestline near Twin Peaks Blvd, the typeahead would surface the actual Crestline intersections (Parkridge, Burnett, Vista) — which is the correct behavior. Reverse-geocoding the lat/lng to an address would still work as a fallback. + +This is reassuring: **the data appears to be both complete and authoritative for actual SF intersections.** + +### The one multi-candidate + +**Fell & Stanyan** returns both "FELL ST & STANYAN ST" (37.77143, -122.45398) and "FELL ACCESS RD & STANYAN ST" (37.77193, -122.45408) — adjacent nodes ~50m apart at the Panhandle/Golden Gate Park entrance. This is genuinely ambiguous and **the right behavior is to surface both candidates in the typeahead** for the officer to pick. The UX must handle multi-candidate results gracefully. + +### Implications for the implementation plan + +1. **DataSF is the right primary source.** 93% exact + 3% multi-candidate (still correct) + 3% genuine geographic gap = effectively complete for SF. +2. **A small normalization layer is required** (suffix expansion, "THE" handling, "BAY SHORE" alias, zero-padding numbered streets). This is a single function, well-defined, easy to unit-test. +3. **Multi-candidate results are real and must be designed for.** ~3% of intersections in our test set returned 2+ valid nodes. The typeahead should not auto-pick. +4. **The "Crestline & Twin Peaks Blvd" type miss is actually the desired behavior** — officers won't type non-existent intersections. If they do, the typeahead saying "no match" is correct. +5. **Vendoring the data is realistic.** 21,058 permutation rows = ~10–11k unique intersections. At ~80 bytes per row (CNN + 2 street names + lat/lng), the full index is ~1.5–2 MB raw, comfortably <500 KB gzipped. Static JSON checked into the repo works fine. +6. **The `cnn` (Centerline Network Number) is a natural canonical key.** Worth storing alongside `street1`/`street2`/`lat`/`lng` on the Incident model — gives us a stable identifier that survives DataSF refresh, useful for de-duplication and downstream joins. + +### Suggested schema addendum + +Adding to the Part 4 storage recommendation: in addition to `street1`/`street2`/`locationType`, store the DataSF `cnn` when available: + +``` +locationType enum('ADDRESS','INTERSECTION') DEFAULT 'ADDRESS' +street1 String? +street2 String? +intersectionId String? -- DataSF CNN (e.g., "24183000"); nullable for free-text entries +``` + +If an officer enters a cross-street that isn't in our vendored index (the 3% case), `street1`/`street2`/`latitude`/`longitude` are still populated but `intersectionId` is null. Downstream consumers can branch on null-vs-present if they need a canonical reference. \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index c7690ecd..29d4caf3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6505,6 +6505,15 @@ "make-plural": "^7.0.0" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.4.11", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.4.11.tgz", + "integrity": "sha512-o9rAHc0IpIjuPSxRutWpE1F62x7n+4mVS4rCNHkzhIUMQcc18bb6xEq5wd2NdN0WjepIyXIppRshYI2kQDOZVA==", + "license": "MIT", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -7007,6 +7016,78 @@ "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" } }, + "node_modules/@redis/bloom": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.12.1.tgz", + "integrity": "sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/client": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.12.1.tgz", + "integrity": "sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@redis/json": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-5.12.1.tgz", + "integrity": "sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/search": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-5.12.1.tgz", + "integrity": "sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/time-series": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-5.12.1.tgz", + "integrity": "sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-beta.35", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.35.tgz", @@ -9618,6 +9699,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "license": "MIT" + }, + "node_modules/@types/whatwg-url": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz", + "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==", + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@types/yauzl": { "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", @@ -10899,6 +10995,26 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/afinn-165": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-2.0.2.tgz", + "integrity": "sha512-mJ/RLUfpXfQA6bzugv+bBsc/QYkVrKaLYeS8fWBpKbTCsonv4iuV9ET0fgReEunm9vKLkaNgnekuSNlTC3WQ1Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/afinn-165-financialmarketnews": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/afinn-165-financialmarketnews/-/afinn-165-financialmarketnews-3.0.0.tgz", + "integrity": "sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -11049,6 +11165,18 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/apparatus": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.10.tgz", + "integrity": "sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==", + "license": "MIT", + "dependencies": { + "sylvester": ">= 0.0.8" + }, + "engines": { + "node": ">=0.2.6" + } + }, "node_modules/archiver": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", @@ -11915,6 +12043,15 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/bson": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.2.0.tgz", + "integrity": "sha512-YCEo7KjMlbNlyHhz7zAZNDpIpQbd+wOEHJYezv0nMYTn4x31eIUM2yomNNubclAt63dObUzKHWsBLJ9QcZNSnQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -12545,6 +12682,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "license": "MIT", @@ -18844,6 +18990,15 @@ "node": ">= 6" } }, + "node_modules/kareem": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-3.3.0.tgz", + "integrity": "sha512-kpSuLD3/7RenBnjnJdOHXCKC8dTd1JzeOiJhN0necWWci6cC+qX+VuwPnMVgb+a4+KNJSfgqahpnfWaeDXCimw==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/katex": { "version": "0.16.25", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.25.tgz", @@ -19389,6 +19544,21 @@ "node": ">= 0.6" } }, + "node_modules/memjs": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/memjs/-/memjs-1.3.2.tgz", + "integrity": "sha512-qUEg2g8vxPe+zPn09KidjIStHPtoBO8Cttm8bgJFWWabbsjQ9Av9Ky+6UcvKx6ue0LLb/LEhtcyQpRyKfzeXcg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "license": "MIT" + }, "node_modules/mensch": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/mensch/-/mensch-0.3.4.tgz", @@ -19650,6 +19820,120 @@ "pathe": "^2.0.1" } }, + "node_modules/mongodb": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.2.0.tgz", + "integrity": "sha512-F/2+BMZtLVhY30ioZp0dAmZ+IRZMBqI+nrv6t5+9/1AIwCa8sMRC3jBf81lpxMhnZgqq8CoUD503Z1oZWq1/sw==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^7.2.0", + "mongodb-connection-string-url": "^7.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.806.0", + "@mongodb-js/zstd": "^7.0.0", + "gcp-metadata": "^7.0.1", + "kerberos": "^7.0.0", + "mongodb-client-encryption": ">=7.0.0 <7.1.0", + "snappy": "^7.3.2", + "socks": "^2.8.6" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.1.tgz", + "integrity": "sha512-h0AZ9A7IDVwwHyMxmdMXKy+9oNlF0zFoahHiX3vQ8e3KFcSP3VmsmfvtRSuLPxmyv2vjIDxqty8smTgie/SNRQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^13.0.0", + "whatwg-url": "^14.1.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/mongoose": { + "version": "9.6.2", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-9.6.2.tgz", + "integrity": "sha512-7m8HntjkoRnwEmuPC0kdlwcZXJOQf4twumFj+PNzg/anqqZE2Er7hQslqyzy07mP3JcFjoTSgH5765PyqOXsxw==", + "license": "MIT", + "dependencies": { + "kareem": "3.3.0", + "mongodb": "~7.2", + "mpath": "0.9.0", + "mquery": "6.0.0", + "ms": "2.1.3", + "sift": "17.1.3" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mongoose" + } + }, "node_modules/moo": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.2.tgz", @@ -19703,6 +19987,24 @@ "node": ">= 0.8" } }, + "node_modules/mpath": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz", + "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/mquery": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/mquery/-/mquery-6.0.0.tgz", + "integrity": "sha512-b2KQNsmgtkscfeDgkYMcWGn9vZI9YoXh802VDEwE6qc50zxBFQ0Oo8ROkawbPAsXCY1/Z1yp0MagqsZStPWJjw==", + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -19828,6 +20130,31 @@ "url": "https://opencollective.com/napi-postinstall" } }, + "node_modules/natural": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/natural/-/natural-8.1.1.tgz", + "integrity": "sha512-Ucb+lsUcGxUqu3rn8cwHjT6gJQosO63nIX/aBQXB3+IDkNbFV7PuviysO+Rzz3aKn7PZhPj3bNF4PS9gDVjYCQ==", + "license": "MIT", + "dependencies": { + "afinn-165": "^2.0.2", + "afinn-165-financialmarketnews": "^3.0.0", + "apparatus": "^0.0.10", + "dotenv": "^17.3.1", + "memjs": "^1.3.2", + "mongoose": "^9.2.1", + "pg": "^8.18.0", + "redis": "^5.11.0", + "safe-stable-stringify": "^2.5.0", + "stopwords-iso": "^1.1.0", + "sylvester": "^0.0.21", + "underscore": "^1.13.0", + "uuid": "^13.0.0", + "wordnet-db": "^3.1.14" + }, + "engines": { + "node": ">=0.4.10" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -19835,6 +20162,31 @@ "dev": true, "license": "MIT" }, + "node_modules/natural/node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/natural/node_modules/uuid": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.2.tgz", + "integrity": "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -23126,6 +23478,22 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-5.12.1.tgz", + "integrity": "sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==", + "license": "MIT", + "dependencies": { + "@redis/bloom": "5.12.1", + "@redis/client": "5.12.1", + "@redis/json": "5.12.1", + "@redis/search": "5.12.1", + "@redis/time-series": "5.12.1" + }, + "engines": { + "node": ">= 18.19.0" + } + }, "node_modules/reflect-metadata": { "version": "0.1.14", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz", @@ -24124,6 +24492,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sift": { + "version": "17.1.3", + "resolved": "https://registry.npmjs.org/sift/-/sift-17.1.3.tgz", + "integrity": "sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==", + "license": "MIT" + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -24275,6 +24649,15 @@ "dev": true, "license": "MIT" }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "license": "MIT", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -24391,6 +24774,15 @@ "node": ">= 0.4" } }, + "node_modules/stopwords-iso": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stopwords-iso/-/stopwords-iso-1.1.0.tgz", + "integrity": "sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/storybook": { "version": "10.3.5", "resolved": "https://registry.npmjs.org/storybook/-/storybook-10.3.5.tgz", @@ -24799,6 +25191,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sylvester": { + "version": "0.0.21", + "resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.21.tgz", + "integrity": "sha512-yUT0ukFkFEt4nb+NY+n2ag51aS/u9UHXoZw+A4jgD77/jzZsBoSDHuqysrVCBC4CYR4TYvUJq54ONpXgDBH8tA==", + "engines": { + "node": ">=0.2.6" + } + }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", @@ -26862,6 +27262,15 @@ "node": ">=0.10.0" } }, + "node_modules/wordnet-db": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/wordnet-db/-/wordnet-db-3.1.14.tgz", + "integrity": "sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==", + "license": "MIT", + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/workbox-background-sync": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/workbox-background-sync/-/workbox-background-sync-7.4.0.tgz", @@ -27651,6 +28060,7 @@ "lodash": "^4.17.21", "luxon": "^3.7.2", "mime-types": "^3.0.1", + "natural": "^8.1.1", "nodemailer": "^7.0.6", "pdf-lib": "^1.17.1", "pg-boss": "^12.14.0", diff --git a/server/lib/forms/849b/generate.js b/server/lib/forms/849b/generate.js index 7343cb8b..01f3c5cc 100644 --- a/server/lib/forms/849b/generate.js +++ b/server/lib/forms/849b/generate.js @@ -1,7 +1,7 @@ import { readFile } from 'fs/promises'; import { join } from 'path'; import prismaPkg from '@prisma/client'; -import { FORM_TIMEZONE, firstInitialLastName, formatDateTime24 } from '../shared/formUtils.js'; +import { FORM_TIMEZONE, firstInitialLastName, formatDateTime24, incidentLocationText } from '../shared/formUtils.js'; import { fill849b } from './fill849b.js'; import { build849bReleaseNarrative } from './releaseNarrative.js'; import i18n from '#lib/i18n.js'; @@ -59,9 +59,7 @@ export function transformData (deflection) { const officerName = firstInitialLastName(incidentCreator); const officerBadge = incident?.createdByBadgeNumber || incidentCreator?.badgeNumber || ''; const reportingDeputy = deflection.releasedBy || (deflection.exitDestination === 'JAIL' ? deflection.exitedBy : null); - const arrestLocation = [incident?.addressLine1, incident?.city, incident?.state] - .filter(Boolean) - .join(', '); + const arrestLocation = incidentLocationText(incident); const subjectAddress = [subject?.addressLine1, subject?.city, subject?.state] .filter(Boolean) diff --git a/server/lib/forms/shared/formUtils.js b/server/lib/forms/shared/formUtils.js index 1a4b0492..12f70773 100644 --- a/server/lib/forms/shared/formUtils.js +++ b/server/lib/forms/shared/formUtils.js @@ -65,3 +65,33 @@ export function streetCityState (obj) { export function streetCityStateZip (obj) { return [streetCityState(obj), obj?.postalCode].filter(Boolean).join(' '); } + +/** + * Human-readable rendering of an incident's location. Branches on + * locationType to render "Street1 & Street2" for intersection mode while + * falling back to the address shape (addressLine1, city, state) otherwise. + * + * Title-cases the DataSF-uppercase street names (e.g. "16TH ST" → "16th St"), + * stripping the zero-pad on numbered streets. + */ +export function incidentLocationText (incident) { + if (incident?.locationType === 'INTERSECTION') { + const s1 = formatDataSfStreet(incident.street1); + const s2 = formatDataSfStreet(incident.street2); + const cross = [s1, s2].filter(Boolean).join(' & '); + const cityState = [incident.city, incident.state].filter(Boolean).join(', '); + return [cross, cityState].filter(Boolean).join(', '); + } + return streetCityState(incident); +} + +function formatDataSfStreet (raw) { + if (!raw) return ''; + // Already title-cased input (e.g. "16th St") — leave untouched. + if (/[a-z]/.test(raw)) return raw; + const stripped = String(raw).replace(/\b0(\d(ST|ND|RD|TH))\b/gi, '$1'); + return stripped + .split(/\s+/) + .map(w => (w ? w[0].toUpperCase() + w.slice(1).toLowerCase() : w)) + .join(' '); +} diff --git a/server/lib/hospitalCancellation647f.js b/server/lib/hospitalCancellation647f.js index 84dae0cc..7193299c 100644 --- a/server/lib/hospitalCancellation647f.js +++ b/server/lib/hospitalCancellation647f.js @@ -1,5 +1,7 @@ import { DateTime } from 'luxon'; +import { isIncidentLocationComplete } from './incidentPermissions.js'; + export const HOSPITAL_CANCEL_REASON = 'HOSPITAL'; export const HOSPITAL_CANCELLATION_INCOMPLETE_DETAILS_ERROR = 'SFPD policy requires person details to be completed before a medical-related cancelation'; export const HOSPITAL_CANCELLATION_ELIGIBLE_SUBJECT_STATUSES = new Set([ @@ -30,9 +32,7 @@ export function hasCompleteHospitalCancellationDetails (deflection) { ); const incidentComplete = Boolean( - incident?.addressLine1 && - incident?.city && - incident?.state && + isIncidentLocationComplete(incident) && incident?.arrestedAt && incident?.encounteredVia && hasMinimumAlphanumericChars(incident?.cadNumber, 2) && diff --git a/server/lib/incidentPermissions.js b/server/lib/incidentPermissions.js index 5ca14399..22cae0e1 100644 --- a/server/lib/incidentPermissions.js +++ b/server/lib/incidentPermissions.js @@ -16,14 +16,25 @@ const CUSTODY_PROPERTY_EDIT_STATUSES = [ 'RELEASED', ]; +/** + * Is the incident's location filled in? Handles both an address-style + * location (addressLine1 + city + state) and an intersection-style location + * (street1 + street2). Pre-migration rows without locationType default to + * the address shape. + */ +export function isIncidentLocationComplete (incident) { + if (incident?.locationType === 'INTERSECTION') { + return !!(incident.street1 && incident.street2); + } + return !!(incident?.addressLine1 && incident?.city && incident?.state); +} + /** * Are all required incident fields filled in? */ export function isIncidentDetailsComplete (incident) { return !!( - incident.addressLine1 && - incident.city && - incident.state && + isIncidentLocationComplete(incident) && incident.arrestedAt && incident.encounteredVia && incident.cadNumber && diff --git a/server/lib/intersections.js b/server/lib/intersections.js new file mode 100644 index 00000000..5caecdfe --- /dev/null +++ b/server/lib/intersections.js @@ -0,0 +1,259 @@ +// SF intersection lookup, backed by a vendored DataSF index. +// +// Two flat JSON files are loaded at boot: +// server/data/sf-intersections.json — one row per intersection (deduped by CNN) +// server/data/sf-streets.json — one row per canonical street name +// +// Build them with: node server/scripts/build-intersection-data.js + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + toPrefixes, + parseCrossStreetInput, + displayName, +} from './streetNormalization.js'; +import { buildMatcher } from './phoneticMatch.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const DATA_DIR = path.resolve(__dirname, '..', 'data'); + +let cache = null; + +function load () { + if (cache) return cache; + const intersections = JSON.parse( + fs.readFileSync(path.join(DATA_DIR, 'sf-intersections.json'), 'utf8') + ); + const streets = JSON.parse( + fs.readFileSync(path.join(DATA_DIR, 'sf-streets.json'), 'utf8') + ); + // Build a prefix index for streets: first letter → array of street records. + // Small enough at ~2k entries that linear scan within a bucket is fine. + const streetsByLetter = new Map(); + for (const street of streets) { + const letter = street.name[0]; + if (!streetsByLetter.has(letter)) streetsByLetter.set(letter, []); + streetsByLetter.get(letter).push(street); + } + const phoneticMatcher = buildMatcher(streets); + cache = { intersections, streets, streetsByLetter, phoneticMatcher }; + return cache; +} + +/** Test-only: clear the in-memory cache so a fresh load() re-reads the files. */ +export function _resetCache () { + cache = null; +} + +/** + * Return all street records whose `name` begins with any of the given prefixes. + * Prefixes are uppercase. Returns at most `limit` matches. + */ +function matchStreets (prefixes, limit = 10) { + const { streets, streetsByLetter } = load(); + const hits = []; + const seen = new Set(); + for (const prefix of prefixes) { + const bucket = streetsByLetter.get(prefix[0]) ?? streets; + for (const street of bucket) { + if (street.name.startsWith(prefix) && !seen.has(street.name)) { + seen.add(street.name); + hits.push(street); + if (hits.length >= limit) return hits; + } + } + } + return hits; +} + +/** + * Find intersection rows matching street1 × street2 in either ordering. + * Both side1 and side2 are arrays of candidate canonical names (full names + * with suffix, uppercase). + */ +function lookupIntersections (side1Names, side2Names, limit = 10) { + const { intersections } = load(); + const set1 = new Set(side1Names); + const set2 = new Set(side2Names); + const hits = []; + for (const row of intersections) { + const matches = + (set1.has(row.street1) && set2.has(row.street2)) || + (set2.has(row.street1) && set1.has(row.street2)); + if (matches) { + hits.push(row); + if (hits.length >= limit) break; + } + } + return hits; +} + +/** + * Search the intersection index for a free-text query like "16th & Valencia". + * Returns up to `limit` candidate intersections. + * + * If the query doesn't parse as a cross-street, returns []. + */ +export function search (text, { limit = 10 } = {}) { + const parsed = parseOrRecover(text); + if (!parsed) return []; + + const side1Streets = rankedStreetCandidates(parsed.side1, 25); + const side2Streets = rankedStreetCandidates(parsed.side2, 25); + if (side1Streets.length === 0 || side2Streets.length === 0) return []; + + const side1Names = side1Streets.map(s => s.name); + const side2Names = side2Streets.map(s => s.name); + const rows = lookupIntersections(side1Names, side2Names, limit); + return rows.map(toApiResult); +} + +/** + * Like `parseCrossStreetInput`, but if no connector is found, attempt a + * "fused-connector" recovery: STT often collapses "and Mission" into + * "admission" (a real English word). For each token whose direct parse + * failed, try peeling a 1–3 char connector-like prefix off; if the remainder + * is a known SF street base, split there. Returns null only if recovery also + * fails. + */ +export function parseOrRecover (text) { + const direct = parseCrossStreetInput(text); + if (direct) return direct; + return recoverFusedConnector(text); +} + +const FUSED_CONNECTOR_PREFIXES = new Set(['AD', 'AND', 'AN', 'AT', 'N']); + +function recoverFusedConnector (text) { + const trimmed = String(text || '').trim(); + if (!trimmed) return null; + // Strip a trailing city/state tail, same as parseCrossStreetInput. + const cleaned = trimmed.replace(/,?\s*(san francisco|sf)\s*(,\s*ca\b.*)?$/i, '').trim(); + const tokens = cleaned.split(/\s+/); + if (tokens.length < 2) return null; + + const { streets } = load(); + const knownBases = new Set(streets.map(s => s.base)); + + for (let i = 0; i < tokens.length; i++) { + const upper = tokens[i].toUpperCase(); + for (let cut = 1; cut <= 3 && cut < upper.length; cut++) { + const prefix = upper.slice(0, cut); + const suffix = upper.slice(cut); + if (!FUSED_CONNECTOR_PREFIXES.has(prefix)) continue; + if (!knownBases.has(suffix)) continue; + const before = tokens.slice(0, i).join(' ').trim(); + const after = [suffix, ...tokens.slice(i + 1)].join(' ').trim(); + if (before && after) return { side1: before, side2: after }; + } + } + return null; +} + +// Prefix-match a free-text token and rank so that streets whose `base` +// exactly equals one of the prefix candidates come first. This keeps the +// common "MISSION ST" from being crowded out by compound names like +// "MISSION BAY BLVD" within the result limit. +function rankedStreetCandidates (rawToken, limit) { + const prefixes = toPrefixes(rawToken); + const hits = matchStreets(prefixes, Math.max(limit * 4, 50)); + if (hits.length === 0) return []; + const exactBases = new Set(prefixes); + return [...hits].sort((a, b) => { + const aExact = exactBases.has(a.base); + const bExact = exactBases.has(b.base); + if (aExact !== bExact) return aExact ? -1 : 1; + if (a.base.length !== b.base.length) return a.base.length - b.base.length; + return a.name < b.name ? -1 : 1; + }).slice(0, limit); +} + +/** + * Find the `n` nearest intersections to a given lat/lng, sorted ascending by + * straight-line distance. Used for "between X and Y" address context. + */ +export function findNearest (lat, lng, { n = 2 } = {}) { + const { intersections } = load(); + // Compute squared planar distance — fine at the scale of SF (~12 km across). + // 1 degree latitude ≈ 111 km; 1 degree longitude at SF ≈ 88 km. We scale + // longitude to make the metric roughly isotropic so the ordering is correct. + const lngScale = Math.cos((lat * Math.PI) / 180); + const scored = intersections.map(row => { + const dLat = row.lat - lat; + const dLng = (row.lng - lng) * lngScale; + return { row, d2: dLat * dLat + dLng * dLng }; + }); + scored.sort((a, b) => a.d2 - b.d2); + return scored.slice(0, n).map(({ row, d2 }) => ({ + ...toApiResult(row), + // Convert squared-degrees back to meters (rough): 1 deg lat ≈ 111000 m. + distanceMeters: Math.round(Math.sqrt(d2) * 111000), + })); +} + +/** + * Find candidate streets for a free-text token. Tries direct prefix match + * first; if none, falls back to Double Metaphone phonetic matching. Returns + * the matching street records (name + base + display) without intersection + * lookup — call searchByStreets() to find intersections between two sides. + */ +export function findStreetCandidates (rawToken, { limit = 5 } = {}) { + const { phoneticMatcher } = load(); + const ranked = rankedStreetCandidates(rawToken, limit); + if (ranked.length > 0) { + return ranked.map(s => ({ + name: s.name, + base: s.base, + display: s.display, + score: 1, + via: 'prefix', + })); + } + // Fall back to phonetic. + const phoneticHits = phoneticMatcher.match(rawToken, { limit }); + return phoneticHits.map(h => ({ ...h, via: 'phonetic' })); +} + +/** + * Search for intersections given two arrays of candidate street names + * (full canonical names with suffix). Used by the voice path after phonetic + * matching has produced candidates per side. + */ +export function searchByStreets (side1Names, side2Names, { limit = 10 } = {}) { + const rows = lookupIntersections(side1Names, side2Names, limit); + return rows.map(toApiResult); +} + +/** Look up a single intersection by its DataSF CNN. */ +export function findByCnn (cnn) { + const { intersections } = load(); + const row = intersections.find(r => r.cnn === cnn); + return row ? toApiResult(row) : null; +} + +function toApiResult (row) { + return { + cnn: row.cnn, + street1: row.street1, + street2: row.street2, + street1Display: displayName(row.street1), + street2Display: displayName(row.street2), + label: `${displayName(row.street1)} & ${displayName(row.street2)}`, + latitude: row.lat, + longitude: row.lng, + zipCode: row.zip ?? null, + }; +} + +export default { + search, + findNearest, + findByCnn, + findStreetCandidates, + searchByStreets, + parseOrRecover, + _resetCache, +}; diff --git a/server/lib/phoneticMatch.js b/server/lib/phoneticMatch.js new file mode 100644 index 00000000..a6c507ea --- /dev/null +++ b/server/lib/phoneticMatch.js @@ -0,0 +1,99 @@ +// Phonetic matching of free-text street tokens against the SF street list. +// +// This is the fallback layer when direct prefix-match (intersections.search) +// fails — e.g., when voice transcription mishears "Junipero Serra" as +// "Junipero Sarah." Double Metaphone collapses both to the same phonetic key +// "JNPRSR" / "ANPRSR", so we can recover the intended street. +// +// Usage: +// const matcher = buildMatcher(streets); +// matcher.match("Junipero Sarah") // → [{ name: "JUNIPERO SERRA BLVD", score: 0.92 }, ...] + +import natural from 'natural'; + +const { DoubleMetaphone, LevenshteinDistance } = natural; +const dm = new DoubleMetaphone(); + +/** + * Build a matcher over a list of street records. + * Each record must have `name` and `base` fields. + * + * Pre-computes the Double Metaphone codes once. The matcher returned exposes + * `match(token, { limit })` which returns candidates ranked by a combined + * phonetic + Levenshtein score. + */ +export function buildMatcher (streets) { + // Pre-compute phonetic codes for each street's `base` (the part without suffix). + // We index by primary code, with secondary code as fallback. + const byPrimary = new Map(); + const bySecondary = new Map(); + const enriched = streets.map(street => { + const [primary, secondary] = dm.process(street.base || street.name); + const entry = { ...street, primary, secondary }; + if (!byPrimary.has(primary)) byPrimary.set(primary, []); + byPrimary.get(primary).push(entry); + if (secondary && secondary !== primary) { + if (!bySecondary.has(secondary)) bySecondary.set(secondary, []); + bySecondary.get(secondary).push(entry); + } + return entry; + }); + + function match (rawToken, { limit = 5 } = {}) { + const token = String(rawToken).trim(); + if (!token) return []; + const [tPrimary, tSecondary] = dm.process(token); + + // Collect candidates with exact phonetic match (primary↔primary, + // primary↔secondary, etc.). + const candidates = new Map(); + const collect = (entries, codeMatched) => { + for (const entry of entries ?? []) { + if (!candidates.has(entry.name)) { + candidates.set(entry.name, { entry, codeMatched }); + } + } + }; + collect(byPrimary.get(tPrimary), 'primary'); + collect(byPrimary.get(tSecondary), 'secondary-input'); + collect(bySecondary.get(tPrimary), 'secondary-street'); + collect(bySecondary.get(tSecondary), 'secondary-both'); + + // If no exact phonetic matches, fall back to a broader phonetic-prefix + // sweep — streets whose code starts with the input's code, or vice versa. + // Useful for partial inputs like "Junipero" against "JUNIPERO SERRA". + if (candidates.size === 0 && tPrimary) { + for (const entry of enriched) { + if ( + entry.primary.startsWith(tPrimary) || + tPrimary.startsWith(entry.primary) + ) { + candidates.set(entry.name, { entry, codeMatched: 'prefix' }); + } + } + } + + // Score each candidate: phonetic-equality weight + Levenshtein-based bonus. + const tokenUpper = token.toUpperCase(); + const scored = Array.from(candidates.values()).map(({ entry, codeMatched }) => { + const lev = LevenshteinDistance(tokenUpper, entry.base, { insertion_cost: 1, deletion_cost: 1, substitution_cost: 1 }); + const denominator = Math.max(tokenUpper.length, entry.base.length, 1); + const levSim = 1 - lev / denominator; + const codeBonus = codeMatched === 'primary' + ? 0.6 + : codeMatched === 'secondary-input' || codeMatched === 'secondary-street' + ? 0.45 + : codeMatched === 'secondary-both' + ? 0.4 + : 0.25; + // Combined score in [0, 1]. + const score = Math.min(1, codeBonus + 0.4 * levSim); + return { name: entry.name, base: entry.base, display: entry.display, score }; + }); + + scored.sort((a, b) => b.score - a.score); + return scored.slice(0, limit); + } + + return { match }; +} diff --git a/server/lib/streetNormalization.js b/server/lib/streetNormalization.js new file mode 100644 index 00000000..86c75d42 --- /dev/null +++ b/server/lib/streetNormalization.js @@ -0,0 +1,176 @@ +// Normalize free-text SF street names to DataSF's canonical form. +// +// DataSF (`jfxm-zeee`) stores street names uppercase with abbreviated suffixes: +// "16TH ST", "VALENCIA ST", "JUNIPERO SERRA BLVD", "THE EMBARCADERO", +// "BAY SHORE BLVD" (split, not "BAYSHORE"), "01ST ST" (zero-padded). +// +// Officers will type informally ("16th and Valencia", "Junipero Serra", "Bayshore"). +// `toPrefixes(name)` returns the set of DataSF prefix strings to try in a +// prefix-match query (covering suffix expansion, THE handling, compound→split, +// zero-pad). + +const SUFFIX_EXPANSIONS = [ + [/\sSTREET$/, ' ST'], + [/\sAVENUE$/, ' AVE'], + [/\sBOULEVARD$/, ' BLVD'], + [/\sLANE$/, ' LN'], + [/\sDRIVE$/, ' DR'], + [/\sROAD$/, ' RD'], + [/\sHIGHWAY$/, ' HWY'], + [/\sPLACE$/, ' PL'], + [/\sTERRACE$/, ' TER'], + [/\sCOURT$/, ' CT'], + [/\sFREEWAY$/, ' FWY'], + [/\sALLEY$/, ' ALY'], + [/\sCIRCLE$/, ' CIR'], + [/\sPLAZA$/, ' PLZ'], + [/\sSTAIRWAY$/, ' STWY'], +]; + +const STRIP_TRAILING_SUFFIX = /\s(ST|AVE|BLVD|RD|LN|WAY|DR|HWY|PL|TER|CT|FWY|ALY|CIR|LOOP|WALK|PATH|PLZ|STWY)$/; + +// Hand-curated SF-specific aliases that prefix-match can't catch. +// Loaded externally for testability; this is just the default. +const DEFAULT_ALIASES = { + BAYSHORE: 'BAY SHORE', +}; + +// Ordinal-word → digit-form. SF numbered streets / avenues are stored as +// "03RD ST", "16TH AVE", etc. — but AWS Transcribe outputs the English word +// "Third", "Sixteenth", which the lookup can't bridge phonetically. We +// normalize these to the digit form before prefix-matching. +const ORDINAL_WORDS = { + FIRST: '1ST', SECOND: '2ND', THIRD: '3RD', FOURTH: '4TH', + FIFTH: '5TH', SIXTH: '6TH', SEVENTH: '7TH', EIGHTH: '8TH', + NINTH: '9TH', TENTH: '10TH', ELEVENTH: '11TH', TWELFTH: '12TH', + THIRTEENTH: '13TH', FOURTEENTH: '14TH', FIFTEENTH: '15TH', + SIXTEENTH: '16TH', SEVENTEENTH: '17TH', EIGHTEENTH: '18TH', + NINETEENTH: '19TH', TWENTIETH: '20TH', + // Compound 21-30. Transcribe sometimes emits hyphenated, sometimes not. + 'TWENTY-FIRST': '21ST', 'TWENTY-SECOND': '22ND', 'TWENTY-THIRD': '23RD', + 'TWENTY-FOURTH': '24TH', 'TWENTY-FIFTH': '25TH', 'TWENTY-SIXTH': '26TH', + 'TWENTY-SEVENTH': '27TH', 'TWENTY-EIGHTH': '28TH', 'TWENTY-NINTH': '29TH', + THIRTIETH: '30TH', +}; + +/** Title-case a single word, preserving common acronyms. */ +function titleWord (w) { + if (!w) return w; + return w[0].toUpperCase() + w.slice(1).toLowerCase(); +} + +/** Title-case a phrase token-by-token. */ +export function titleCase (s) { + return String(s).split(/\s+/).map(titleWord).join(' '); +} + +/** Expand a long-form trailing suffix word to DataSF's abbreviation. */ +export function expandSuffix (raw) { + let s = String(raw).trim().toUpperCase(); + for (const [pattern, replacement] of SUFFIX_EXPANSIONS) { + if (pattern.test(s)) { + s = s.replace(pattern, replacement); + break; + } + } + return s; +} + +/** + * Replace ordinal English words ("Third", "Sixteenth", "Twenty-First") with + * their digit-suffix form ("3RD", "16TH", "21ST"). Handles both hyphenated + * ("twenty-first") and space-separated ("twenty first") compound ordinals. + */ +export function expandOrdinalWords (raw) { + let s = String(raw).toUpperCase(); + // Compound ordinals first (longest match), both hyphenated and spaced. + for (const [word, digit] of Object.entries(ORDINAL_WORDS)) { + if (word.includes('-')) { + const spaced = word.replace('-', ' '); + s = s.replace(new RegExp(`\\b${word}\\b`, 'g'), digit); + s = s.replace(new RegExp(`\\b${spaced}\\b`, 'g'), digit); + } + } + // Single-word ordinals. + for (const [word, digit] of Object.entries(ORDINAL_WORDS)) { + if (!word.includes('-')) { + s = s.replace(new RegExp(`\\b${word}\\b`, 'g'), digit); + } + } + return s; +} + +/** + * Produce candidate prefix strings for a DataSF prefix-match query. + * Returns the union of plausible variants (suffix-stripped base, "THE"-prefixed, + * alias-substituted, zero-padded numeric). + * + * @param {string} raw - user-typed street name (any case) + * @param {Record} [aliases] - alias map, e.g., { BAYSHORE: 'BAY SHORE' } + * @returns {string[]} unique prefix candidates, uppercase + */ +export function toPrefixes (raw, aliases = DEFAULT_ALIASES) { + const expanded = expandSuffix(expandOrdinalWords(raw)); + // Strip the trailing suffix word so the prefix matches any suffix variant. + let base = expanded.replace(STRIP_TRAILING_SUFFIX, ''); + // Zero-pad single-digit numbered streets: 3RD → 03RD, 7TH → 07TH. + base = base.replace(/^(\d)(ST|ND|RD|TH)\b/, '0$1$2'); + + const candidates = new Set([base]); + + // Try with leading THE and without it. + if (base.startsWith('THE ')) { + candidates.add(base.slice(4)); + } else { + candidates.add(`THE ${base}`); + } + + // Substitute known compound→split aliases (and vice versa). + for (const [from, to] of Object.entries(aliases)) { + if (base === from) candidates.add(to); + if (base === to) candidates.add(from); + } + + return Array.from(candidates).filter(Boolean); +} + +/** + * Build a display-friendly name from a DataSF row name. + * "16TH ST" → "16th St" + * "JUNIPERO SERRA BLVD" → "Junipero Serra Blvd" + * "THE EMBARCADERO" → "The Embarcadero" + * "03RD ST" → "3rd St" (zero-pad stripped) + */ +export function displayName (rawDataSfName) { + let s = String(rawDataSfName).trim(); + // Strip leading zero on numbered streets for display ("03RD" → "3RD"). + s = s.replace(/^0(\d(ST|ND|RD|TH))\b/i, '$1'); + return titleCase(s); +} + +/** + * Tokenize input on the canonical intersection connectors and return the two + * sides if it looks like a cross-street query, or null otherwise. + * + * "16th & Valencia" → { side1: "16th", side2: "Valencia" } + * "Junipero Serra and Monterey" → { side1: "Junipero Serra", side2: "Monterey" } + * "425 16th St" → null (looks like an address) + * "16th and Mission, SF, CA" → { side1: "16th", side2: "Mission" } (trailing city stripped) + */ +export function parseCrossStreetInput (raw) { + const input = String(raw).trim(); + if (!input) return null; + // If it starts with a digit followed by a non-ordinal word, treat as an address. + // Specifically: "425 Foo" is an address, but "16th & Mission" is an intersection. + if (/^\d+\s+[A-Za-z]/.test(input) && !/^\d+(st|nd|rd|th)\b/i.test(input)) { + return null; + } + // Strip trailing ", San Francisco, CA" or similar city/state tail. + const trimmed = input.replace(/,?\s*(san francisco|sf)\s*(,\s*ca\b.*)?$/i, '').trim(); + // Split on the first connector found. + const match = trimmed.match(/^\s*(.+?)\s*(?:&|\/|@|\band\b|\bat\b)\s*(.+?)\s*$/i); + if (!match) return null; + const [, side1, side2] = match; + if (!side1 || !side2) return null; + return { side1: side1.trim(), side2: side2.trim() }; +} diff --git a/server/models/incident.js b/server/models/incident.js index 986209bf..36dfd920 100644 --- a/server/models/incident.js +++ b/server/models/incident.js @@ -13,11 +13,15 @@ const { Prisma } = prismaPkg; const IncidentAttributesSchema = z.object({ cadNumber: z.string().nullable(), caseNumber: z.string().nullable(), + locationType: z.enum(['ADDRESS', 'INTERSECTION']).default('ADDRESS'), addressLine1: z.string().nullable(), addressLine2: z.string().nullable(), city: z.string().nullable(), state: z.string().nullable(), postalCode: z.string().nullable(), + street1: z.string().nullable(), + street2: z.string().nullable(), + intersectionId: z.string().nullable(), latitude: z.coerce.number().nullable(), longitude: z.coerce.number().nullable(), arrestedAt: z.coerce.date().catch(null).nullable(), diff --git a/server/package.json b/server/package.json index 311e143e..98018bfc 100644 --- a/server/package.json +++ b/server/package.json @@ -60,6 +60,7 @@ "lodash": "^4.17.21", "luxon": "^3.7.2", "mime-types": "^3.0.1", + "natural": "^8.1.1", "nodemailer": "^7.0.6", "pdf-lib": "^1.17.1", "pg-boss": "^12.14.0", diff --git a/server/prisma/ERD.md b/server/prisma/ERD.md index 1f3ba76b..39e79fa7 100644 --- a/server/prisma/ERD.md +++ b/server/prisma/ERD.md @@ -223,6 +223,13 @@ FACILITY_ADMIN FACILITY_ADMIN + IncidentLocationType { + ADDRESS ADDRESS +INTERSECTION INTERSECTION + } + + + SubjectStatusEnum { DETAINED DETAINED ONSITE_AWAITING_TRANSFER ONSITE_AWAITING_TRANSFER @@ -599,11 +606,15 @@ DEPARTURE DEPARTURE "Incident" { Int id "🗝️" String facilityId "🗝️" + IncidentLocationType locationType String addressLine1 "❓" String addressLine2 "❓" String city "❓" String state "❓" String postalCode "❓" + String street1 "❓" + String street2 "❓" + String intersectionId "❓" Decimal latitude "❓" Decimal longitude "❓" DateTime arrestedAt "❓" @@ -802,6 +813,7 @@ DEPARTURE DEPARTURE "PropertyPhoto" o|--|| "User" : "createdBy" "PropertyPhoto" o|--|| "User" : "updatedBy" "Incident" o|--|| "Facility" : "facility" + "Incident" o|--|| "IncidentLocationType" : "enum:locationType" "Incident" o|--|o "EncounteredViaEnum" : "enum:encounteredVia" "Incident" o|--|| "User" : "createdBy" "Incident" o|--|o "Organization" : "createdByOrganization" diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index 3a686c78..a11c2708 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -198,6 +198,11 @@ enum RoleEnum { FACILITY_ADMIN } +enum IncidentLocationType { + ADDRESS + INTERSECTION +} + enum SubjectStatusEnum { DETAINED ONSITE_AWAITING_TRANSFER @@ -707,11 +712,15 @@ model Incident { id Int @unique @default(autoincrement()) facilityId String @db.Uuid facility Facility @relation(fields: [facilityId], references: [id], onDelete: Cascade) + locationType IncidentLocationType @default(ADDRESS) addressLine1 String? addressLine2 String? city String? state String? postalCode String? + street1 String? + street2 String? + intersectionId String? latitude Decimal? @db.Decimal(9, 6) longitude Decimal? @db.Decimal(9, 6) arrestedAt DateTime? diff --git a/server/routes/api/ai/transcribe.js b/server/routes/api/ai/transcribe.js index edbbd00e..d2d3cafc 100644 --- a/server/routes/api/ai/transcribe.js +++ b/server/routes/api/ai/transcribe.js @@ -2,6 +2,8 @@ import { StatusCodes } from 'http-status-codes'; import { z } from 'zod'; import { TranscribeStreamingClient, StartStreamTranscriptionCommand } from '@aws-sdk/client-transcribe-streaming'; +import intersections from '#lib/intersections.js'; + let client; function getClient () { @@ -31,6 +33,9 @@ export default async function (fastify) { bodyLimit: 10 * 1024 * 1024, // 10MB — base64 audio can be large schema: { description: 'Transcribe audio to text using AWS Transcribe Streaming.', + querystring: z.object({ + mode: z.enum(['narrative', 'location']).default('narrative'), + }), body: z.object({ audio: z.string().describe('Base64-encoded 16kHz mono PCM audio'), mediaType: z.string().default('audio/pcm'), @@ -38,12 +43,42 @@ export default async function (fastify) { response: { [StatusCodes.OK]: z.object({ text: z.string(), + matches: z.array(z.object({ + cnn: z.string(), + street1: z.string(), + street2: z.string(), + street1Display: z.string(), + street2Display: z.string(), + label: z.string(), + latitude: z.number(), + longitude: z.number(), + zipCode: z.string().nullable(), + })).optional(), + parsed: z.object({ + side1: z.string(), + side2: z.string(), + side1Candidates: z.array(z.object({ + name: z.string(), + base: z.string(), + display: z.string(), + score: z.number(), + via: z.string(), + })), + side2Candidates: z.array(z.object({ + name: z.string(), + base: z.string(), + display: z.string(), + score: z.number(), + via: z.string(), + })), + }).nullable().optional(), }), }, }, }, async function (request, reply) { const { audio } = request.body; + const { mode } = request.query; const audioBuffer = Buffer.from(audio, 'base64'); request.log.info(`Transcribe: received ${audioBuffer.length} bytes`); @@ -77,6 +112,25 @@ export default async function (fastify) { const text = transcripts.join(' '); request.log.info(`Transcribe: result="${text.substring(0, 100)}${text.length > 100 ? '...' : ''}"`); + + if (mode === 'location') { + const parsed = intersections.parseOrRecover(text); + if (!parsed) { + return reply.send({ text, matches: [], parsed: null }); + } + const side1Candidates = intersections.findStreetCandidates(parsed.side1); + const side2Candidates = intersections.findStreetCandidates(parsed.side2); + const matches = intersections.searchByStreets( + side1Candidates.map(c => c.name), + side2Candidates.map(c => c.name) + ); + return reply.send({ + text, + matches, + parsed: { side1: parsed.side1, side2: parsed.side2, side1Candidates, side2Candidates }, + }); + } + reply.send({ text }); }); } diff --git a/server/routes/api/arrests.js b/server/routes/api/arrests.js index 31812bfc..2aea3720 100644 --- a/server/routes/api/arrests.js +++ b/server/routes/api/arrests.js @@ -2,7 +2,7 @@ import { StatusCodes } from 'http-status-codes'; import { DateTime } from 'luxon'; import { z } from 'zod'; -import { firstInitialLastName, streetCityState } from '#lib/forms/shared/formUtils.js'; +import { firstInitialLastName, incidentLocationText } from '#lib/forms/shared/formUtils.js'; const TIMEZONE = 'America/Los_Angeles'; const FACILITY_NAME = 'RESET'; @@ -61,9 +61,12 @@ export default async function (fastify) { }, select: { arrestedAt: true, + locationType: true, addressLine1: true, city: true, state: true, + street1: true, + street2: true, caseNumber: true, createdByBadgeNumber: true, createdBy: { select: { firstName: true, lastName: true } }, @@ -94,7 +97,7 @@ export default async function (fastify) { const subject = deflection?.subject ?? null; return { arrestedAt: i.arrestedAt.toISOString(), - address: streetCityState(i), + address: incidentLocationText(i), caseNumber: i.caseNumber ?? null, firstName: subject?.firstName ?? null, lastName: subject?.lastName ?? null, diff --git a/server/routes/api/geocode/intersections.js b/server/routes/api/geocode/intersections.js new file mode 100644 index 00000000..5022d66e --- /dev/null +++ b/server/routes/api/geocode/intersections.js @@ -0,0 +1,73 @@ +import { StatusCodes } from 'http-status-codes'; +import { z } from 'zod'; + +import intersections from '#lib/intersections.js'; + +const IntersectionResultSchema = z.object({ + cnn: z.string(), + street1: z.string(), + street2: z.string(), + street1Display: z.string(), + street2Display: z.string(), + label: z.string(), + latitude: z.number(), + longitude: z.number(), + zipCode: z.string().nullable(), +}); + +const NearestResultSchema = IntersectionResultSchema.extend({ + distanceMeters: z.number(), +}); + +export default async function (fastify, opts) { + fastify.get('/intersections', + { + onRequest: fastify.requireUser, + schema: { + description: 'Search SF intersection index by free-text cross-street query.', + querystring: z.object({ + text: z.string().min(1), + limit: z.coerce.number().int().min(1).max(20).optional(), + }), + response: { + [StatusCodes.OK]: z.array(IntersectionResultSchema), + }, + }, + }, + async function (request, reply) { + const { text, limit } = request.query; + try { + const results = intersections.search(text, { limit: limit ?? 10 }); + return reply.send(results); + } catch (error) { + fastify.log.error(error, 'Error during intersection search'); + return reply.code(StatusCodes.INTERNAL_SERVER_ERROR).send(); + } + }); + + fastify.get('/intersections/nearest', + { + onRequest: fastify.requireUser, + schema: { + description: 'Find the N intersections nearest to a given lat/lng (for "between" context).', + querystring: z.object({ + latitude: z.coerce.number(), + longitude: z.coerce.number(), + n: z.coerce.number().int().min(1).max(10).optional(), + }), + response: { + [StatusCodes.OK]: z.array(NearestResultSchema), + }, + }, + }, + async function (request, reply) { + const { latitude, longitude, n } = request.query; + try { + const results = intersections.findNearest(latitude, longitude, { n: n ?? 2 }); + return reply.send(results); + } catch (error) { + fastify.log.error(error, 'Error during nearest intersection lookup'); + return reply.code(StatusCodes.INTERNAL_SERVER_ERROR).send(); + } + }); +} diff --git a/server/scripts/bench-datasf-intersections.js b/server/scripts/bench-datasf-intersections.js new file mode 100644 index 00000000..b644bfc0 --- /dev/null +++ b/server/scripts/bench-datasf-intersections.js @@ -0,0 +1,156 @@ +// Spot-check the 30 SF intersections from bench-intersections.js against the +// authoritative DataSF dataset `jfxm-zeee` (Intersections by Each Cross Street +// Permutation). For each intersection, query both (a, b) and (b, a) orderings +// using prefix matching (suffix-agnostic), and report: +// - found / not found +// - candidate matches with lat/lng +// - the canonical street_name_1 / street_name_2 (with suffix) +// +// Run: node scripts/bench-datasf-intersections.js + +const INTERSECTIONS = [ + { a: 'Market', b: 'Castro', category: 'arterial' }, + { a: 'Mission', b: '24th', category: 'arterial' }, + { a: 'Market', b: 'Powell', category: 'arterial' }, + { a: 'Van Ness', b: 'California', category: 'arterial' }, + { a: 'Geary', b: 'Masonic', category: 'arterial' }, + + { a: '16th', b: 'Valencia', category: 'mission-soma' }, + { a: '16th', b: 'Mission', category: 'mission-soma' }, + { a: '24th', b: 'Mission', category: 'mission-soma' }, + { a: '3rd', b: 'Townsend', category: 'mission-soma' }, + { a: '7th', b: 'Folsom', category: 'mission-soma' }, + + { a: '19th Ave', b: 'Sloat', category: 'west-side' }, + { a: 'Geary', b: '20th Ave', category: 'west-side' }, + { a: 'Irving', b: '9th Ave', category: 'west-side' }, + { a: 'Taraval', b: '19th Ave', category: 'west-side' }, + { a: 'California', b: '25th Ave', category: 'west-side' }, + + { a: 'Junipero Serra', b: 'Monterey', category: 'spanish' }, + { a: 'Cesar Chavez', b: 'Mission', category: 'spanish' }, + { a: 'Embarcadero', b: 'Folsom', category: 'spanish' }, + { a: 'Bayshore', b: 'Cortland', category: 'spanish' }, + { a: 'Dolores', b: '17th', category: 'spanish' }, + + { a: 'Minna', b: '6th', category: 'alley' }, + { a: 'Natoma', b: '8th', category: 'alley' }, + { a: 'Stevenson', b: '7th', category: 'alley' }, + { a: 'Maiden Lane', b: 'Grant', category: 'alley' }, + { a: 'Hooper', b: '7th', category: 'alley' }, + + { a: 'The Embarcadero', b: 'Bay', category: 'edge' }, + { a: 'Fell', b: 'Stanyan', category: 'edge' }, + { a: 'Crestline', b: 'Twin Peaks Blvd', category: 'edge' }, + { a: 'Lombard', b: 'Hyde', category: 'edge' }, + { a: 'Kezar', b: 'Lincoln', category: 'edge' }, +]; + +// Map a user-style street name to a list of DataSF prefix candidates for query. +// DataSF uses zero-padded numbered streets ("01ST", "16TH"), uppercase, with +// suffix baked in and abbreviated (LANE → LN, STREET → ST, etc.). For some +// streets the leading "THE" is part of the canonical name ("THE EMBARCADERO"). +// And a few are spelled split where you'd expect compound ("BAY SHORE" vs +// "BAYSHORE"). We return multiple prefix candidates and query all of them. +function toPrefixes (name) { + let s = name.trim().toUpperCase(); + // Expand "LANE" → "LN", "STREET" → "ST" so abbreviated DataSF rows match. + s = s.replace(/\s+STREET$/, ' ST'); + s = s.replace(/\s+AVENUE$/, ' AVE'); + s = s.replace(/\s+BOULEVARD$/, ' BLVD'); + s = s.replace(/\s+LANE$/, ' LN'); + s = s.replace(/\s+DRIVE$/, ' DR'); + s = s.replace(/\s+ROAD$/, ' RD'); + // Strip the trailing suffix word so we can prefix-match across ST/AVE/BLVD/... + let base = s.replace(/\s+(ST|AVE|BLVD|RD|LN|WAY|DR|HWY|PL|TER|CT|FWY)$/, ''); + // Zero-pad single-digit numbered streets to match DataSF format ("3RD" → "03RD") + base = base.replace(/^(\d)(ST|ND|RD|TH)\b/, '0$1$2'); + const candidates = [base]; + // "Embarcadero" might be stored as "THE EMBARCADERO". + if (!base.startsWith('THE ')) candidates.push(`THE ${base}`); + // Stripping "THE " also yields a candidate in case the user wrote "The Embarcadero". + if (base.startsWith('THE ')) candidates.push(base.slice(4)); + // Compound→split: "BAYSHORE" might be stored as "BAY SHORE". + if (base === 'BAYSHORE') candidates.push('BAY SHORE'); + return candidates; +} + +async function findIntersection (a, b) { + const aPrefixes = toPrefixes(a); + const bPrefixes = toPrefixes(b); + const url = (s1, s2) => { + const where = `starts_with(upper(street_name_1), '${s1}') AND starts_with(upper(street_name_2), '${s2}')`; + return `https://data.sfgov.org/resource/jfxm-zeee.json?$where=${encodeURIComponent(where)}&$limit=10`; + }; + const queries = []; + for (const ap of aPrefixes) { + for (const bp of bPrefixes) { + queries.push(fetch(url(ap, bp)).then(r => r.json())); + queries.push(fetch(url(bp, ap)).then(r => r.json())); + } + } + const results = await Promise.all(queries); + // Dedupe by CNN + const byCnn = new Map(); + for (const rows of results) { + for (const row of rows) { + if (!byCnn.has(row.cnn)) byCnn.set(row.cnn, row); + } + } + return Array.from(byCnn.values()); +} + +function fmtCoord (row) { + if (!row.latitude || !row.longitude) return '(no coord)'; + return `(${(+row.latitude).toFixed(5)}, ${(+row.longitude).toFixed(5)})`; +} + +async function main () { + const results = []; + for (const x of INTERSECTIONS) { + const matches = await findIntersection(x.a, x.b); + results.push({ ...x, matches }); + } + + let lastCat = null; + for (const r of results) { + if (r.category !== lastCat) { + console.log(`\n=== ${r.category} ===`); + lastCat = r.category; + } + const label = `${r.a} & ${r.b}`.padEnd(38); + if (r.matches.length === 0) { + console.log(`❌ ${label} no match in DataSF`); + } else if (r.matches.length === 1) { + const m = r.matches[0]; + console.log(`✅ ${label} ${m.street_name_1} & ${m.street_name_2} ${fmtCoord(m)}`); + } else { + console.log(`⚠️ ${label} ${r.matches.length} candidates:`); + for (const m of r.matches) { + console.log(` · ${m.street_name_1} & ${m.street_name_2} ${fmtCoord(m)} [zip ${m.zip_code ?? '?'}]`); + } + } + } + + // Summary + console.log('\n\n=== SUMMARY ==='); + const byCat = {}; + for (const r of results) { + byCat[r.category] ??= { found: 0, multi: 0, missing: 0, total: 0 }; + byCat[r.category].total++; + if (r.matches.length === 0) byCat[r.category].missing++; + else if (r.matches.length === 1) byCat[r.category].found++; + else byCat[r.category].multi++; + } + const total = { found: 0, multi: 0, missing: 0, total: 0 }; + for (const [cat, s] of Object.entries(byCat)) { + console.log(` ${cat.padEnd(15)} found:${s.found}/${s.total} multi:${s.multi} missing:${s.missing}`); + for (const k of ['found', 'multi', 'missing', 'total']) total[k] += s[k]; + } + console.log(` ${'OVERALL'.padEnd(15)} found:${total.found}/${total.total} multi:${total.multi} missing:${total.missing}`); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/server/scripts/bench-intersections.js b/server/scripts/bench-intersections.js new file mode 100644 index 00000000..1a0725fc --- /dev/null +++ b/server/scripts/bench-intersections.js @@ -0,0 +1,246 @@ +// Bench-test AWS Location v2 forward intersection geocoding on SF intersections. +// +// Run from the server workspace: +// node scripts/bench-intersections.js +// +// Or with a single category: +// node scripts/bench-intersections.js --category=spanish +// +// Or with a single input format: +// node scripts/bench-intersections.js --format=ampersand +// +// Reports per-intersection success/fail for both AutocompleteCommand and +// GeocodeCommand, across a few input format variations. + +import '../config.js'; + +import { + GeoPlacesClient, + GeocodeCommand, + AutocompleteCommand, +} from '@aws-sdk/client-geo-places'; + +const SF_BBOX = [-122.5155, 37.7080, -122.3570, 37.8120]; +const SF_BIAS = [-122.4194, 37.7749]; + +const client = new GeoPlacesClient({ + credentials: { + accessKeyId: process.env.AWS_LOCATION_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_LOCATION_SECRET_ACCESS_KEY, + }, + region: process.env.AWS_LOCATION_REGION ?? 'us-west-2', +}); + +// Curated test set: ~30 SF intersections across 6 categories. +// Each entry: { a, b, category, note? }. `a` and `b` are the bare street names +// without suffixes (the CAD convention) — the script generates suffix variants +// itself. +const INTERSECTIONS = [ + // Major arterials (English) — should be easy + { a: 'Market', b: 'Castro', category: 'arterial' }, + { a: 'Mission', b: '24th', category: 'arterial' }, + { a: 'Market', b: 'Powell', category: 'arterial' }, + { a: 'Van Ness', b: 'California', category: 'arterial' }, + { a: 'Geary', b: 'Masonic', category: 'arterial' }, + + // Numbered grid — Mission/SoMa + { a: '16th', b: 'Valencia', category: 'mission-soma' }, + { a: '16th', b: 'Mission', category: 'mission-soma' }, + { a: '24th', b: 'Mission', category: 'mission-soma' }, + { a: '3rd', b: 'Townsend', category: 'mission-soma' }, + { a: '7th', b: 'Folsom', category: 'mission-soma' }, + + // Numbered grid — Sunset/Richmond (avenues) + { a: '19th Ave', b: 'Sloat', category: 'west-side' }, + { a: 'Geary', b: '20th Ave', category: 'west-side' }, + { a: 'Irving', b: '9th Ave', category: 'west-side' }, + { a: 'Taraval', b: '19th Ave', category: 'west-side' }, + { a: 'California', b: '25th Ave', category: 'west-side' }, + + // Spanish-origin / phonetically tricky + { a: 'Junipero Serra', b: 'Monterey', category: 'spanish', note: 'prompt example' }, + { a: 'Cesar Chavez', b: 'Mission', category: 'spanish' }, + { a: 'Embarcadero', b: 'Folsom', category: 'spanish' }, + { a: 'Bayshore', b: 'Cortland', category: 'spanish' }, + { a: 'Dolores', b: '17th', category: 'spanish' }, + + // Alley / minor street intersections + { a: 'Minna', b: '6th', category: 'alley' }, + { a: 'Natoma', b: '8th', category: 'alley' }, + { a: 'Stevenson', b: '7th', category: 'alley' }, + { a: 'Maiden Lane', b: 'Grant', category: 'alley' }, + { a: 'Hooper', b: '7th', category: 'alley' }, + + // Weird / historic / edge cases + { a: 'The Embarcadero', b: 'Bay', category: 'edge' }, + { a: 'Fell', b: 'Stanyan', category: 'edge', note: 'Panhandle corner' }, + { a: 'Crestline', b: 'Twin Peaks Blvd', category: 'edge' }, + { a: 'Lombard', b: 'Hyde', category: 'edge', note: 'curvy block' }, + { a: 'Kezar', b: 'Lincoln', category: 'edge' }, +]; + +const FORMATS = { + ampersand: ({ a, b }) => `${a} & ${b}, San Francisco, CA`, + and: ({ a, b }) => `${a} and ${b}, San Francisco, CA`, + withSuffix: ({ a, b }) => `${withSt(a)} & ${withSt(b)}, San Francisco, CA`, + bare: ({ a, b }) => `${a} & ${b}`, +}; + +function withSt (s) { + // Don't double-suffix already-suffixed names like "19th Ave" or "The Embarcadero". + if (/\b(St|Ave|Blvd|Rd|Lane|Way)\b/i.test(s)) return s; + if (/^the /i.test(s)) return s; + return `${s} St`; +} + +async function tryGeocode (text) { + try { + const response = await client.send(new GeocodeCommand({ + QueryText: text, + MaxResults: 1, + BiasPosition: SF_BIAS, + Filter: { BoundingBox: SF_BBOX, IncludeCountries: ['USA'] }, + Language: 'en', + IntendedUse: 'SingleUse', + })); + const item = response.ResultItems?.[0]; + if (!item) return { ok: false, reason: 'no-result' }; + return { + ok: true, + placeType: item.PlaceType, + label: item.Address?.Label ?? item.Title ?? null, + intersection: item.Address?.Intersection ?? null, + position: item.Position ?? null, + }; + } catch (err) { + return { ok: false, reason: 'error', error: err.name + ': ' + err.message }; + } +} + +async function tryAutocomplete (text) { + try { + const response = await client.send(new AutocompleteCommand({ + QueryText: text, + MaxResults: 5, + BiasPosition: SF_BIAS, + Filter: { BoundingBox: SF_BBOX, IncludeCountries: ['USA'] }, + Language: 'en', + IntendedUse: 'SingleUse', + })); + const items = response.ResultItems ?? []; + if (items.length === 0) return { ok: false, reason: 'no-result' }; + return { + ok: true, + count: items.length, + first: { + placeType: items[0].PlaceType, + title: items[0].Title, + label: items[0].Address?.Label ?? null, + intersection: items[0].Address?.Intersection ?? null, + }, + anyIntersection: items.some(it => it.PlaceType === 'Intersection'), + }; + } catch (err) { + return { ok: false, reason: 'error', error: err.name + ': ' + err.message }; + } +} + +function fmtResult (r) { + if (!r.ok) return `❌ ${r.reason}${r.error ? ' (' + r.error + ')' : ''}`; + if (r.placeType) { + const tag = r.placeType === 'Intersection' ? '✅ INT' : '⚠️ ' + r.placeType; + return `${tag} · ${r.label ?? '(no label)'}`; + } + const tag = r.anyIntersection ? '✅ INT' : '⚠️ ' + (r.first?.placeType ?? '?'); + return `${tag} · ${r.first?.label ?? r.first?.title ?? '(no label)'} (${r.count})`; +} + +async function main () { + const args = Object.fromEntries( + process.argv.slice(2).map(a => a.replace(/^--/, '').split('=')) + ); + const onlyCategory = args.category; + const onlyFormat = args.format; + + const rows = onlyCategory + ? INTERSECTIONS.filter(x => x.category === onlyCategory) + : INTERSECTIONS; + const formats = onlyFormat ? [onlyFormat] : Object.keys(FORMATS); + + const results = []; + for (const intersection of rows) { + for (const formatName of formats) { + const text = FORMATS[formatName](intersection); + const [geocode, autocomplete] = await Promise.all([ + tryGeocode(text), + tryAutocomplete(text), + ]); + results.push({ intersection, formatName, text, geocode, autocomplete }); + } + } + + // Pretty-print grouped by intersection + let lastKey = null; + for (const r of results) { + const key = `${r.intersection.a} & ${r.intersection.b} [${r.intersection.category}]`; + if (key !== lastKey) { + console.log(''); + console.log(`▶ ${key}${r.intersection.note ? ' — ' + r.intersection.note : ''}`); + lastKey = key; + } + console.log(` [${r.formatName.padEnd(10)}] "${r.text}"`); + console.log(` Geocode : ${fmtResult(r.geocode)}`); + console.log(` Autocomplete : ${fmtResult(r.autocomplete)}`); + } + + // Summary + console.log('\n\n=== SUMMARY ==='); + const byCategoryFormat = {}; + for (const r of results) { + const cat = r.intersection.category; + byCategoryFormat[cat] ??= {}; + byCategoryFormat[cat][r.formatName] ??= { geocodeInt: 0, geocodeAny: 0, autoInt: 0, autoAny: 0, total: 0 }; + const s = byCategoryFormat[cat][r.formatName]; + s.total++; + if (r.geocode.ok) { + s.geocodeAny++; + if (r.geocode.placeType === 'Intersection') s.geocodeInt++; + } + if (r.autocomplete.ok) { + s.autoAny++; + if (r.autocomplete.anyIntersection) s.autoInt++; + } + } + for (const [cat, byFmt] of Object.entries(byCategoryFormat)) { + console.log(`\n${cat}:`); + for (const [fmt, s] of Object.entries(byFmt)) { + console.log(` ${fmt.padEnd(11)} Geocode: ${s.geocodeInt}/${s.total} intersection, ${s.geocodeAny}/${s.total} any | Autocomplete: ${s.autoInt}/${s.total} intersection, ${s.autoAny}/${s.total} any`); + } + } + + // Overall + const overall = { geocodeInt: 0, geocodeAny: 0, autoInt: 0, autoAny: 0, total: 0 }; + for (const r of results) { + overall.total++; + if (r.geocode.ok) { + overall.geocodeAny++; + if (r.geocode.placeType === 'Intersection') overall.geocodeInt++; + } + if (r.autocomplete.ok) { + overall.autoAny++; + if (r.autocomplete.anyIntersection) overall.autoInt++; + } + } + console.log(`\nOVERALL (${overall.total} queries across ${rows.length} intersections × ${formats.length} formats):`); + console.log(` Geocode → Intersection: ${overall.geocodeInt}/${overall.total} (${pct(overall.geocodeInt, overall.total)}) · Any result: ${overall.geocodeAny}/${overall.total} (${pct(overall.geocodeAny, overall.total)})`); + console.log(` Autocomplete → Intersection: ${overall.autoInt}/${overall.total} (${pct(overall.autoInt, overall.total)}) · Any result: ${overall.autoAny}/${overall.total} (${pct(overall.autoAny, overall.total)})`); +} + +function pct (n, d) { + return d === 0 ? '–' : `${Math.round((n / d) * 100)}%`; +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/server/scripts/build-intersection-data.js b/server/scripts/build-intersection-data.js new file mode 100644 index 00000000..ed3930e1 --- /dev/null +++ b/server/scripts/build-intersection-data.js @@ -0,0 +1,159 @@ +// Fetch SF intersection + street data from DataSF and write vendored JSON. +// +// Run from the server workspace: +// node scripts/build-intersection-data.js +// +// Output: +// server/data/sf-intersections.json — one row per intersection (deduped by CNN) +// server/data/sf-streets.json — one row per canonical street name +// +// Data source: DataSF dataset `jfxm-zeee` ("Intersections by Each Cross Street +// Permutation"). ~21k rows; we dedupe to ~10–11k unique intersections. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const DATA_DIR = path.resolve(__dirname, '..', 'data'); + +const SOCRATA_RESOURCE = 'https://data.sfgov.org/resource/jfxm-zeee.json'; +const PAGE_SIZE = 5000; + +async function fetchAllRows () { + const allRows = []; + let offset = 0; + while (true) { + const url = `${SOCRATA_RESOURCE}?$limit=${PAGE_SIZE}&$offset=${offset}&$order=cnn`; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`DataSF fetch failed: ${response.status} ${response.statusText}`); + } + const rows = await response.json(); + console.log(` fetched ${rows.length} rows (offset ${offset})`); + if (rows.length === 0) break; + allRows.push(...rows); + if (rows.length < PAGE_SIZE) break; + offset += PAGE_SIZE; + } + return allRows; +} + +function loadAliases () { + try { + const raw = fs.readFileSync(path.join(DATA_DIR, 'streetAliases.json'), 'utf8'); + return JSON.parse(raw); + } catch { + return {}; + } +} + +// Strip a known DataSF suffix from the end of a street name. +function splitSuffix (name) { + const m = name.match(/^(.+?)\s(ST|AVE|BLVD|RD|LN|WAY|DR|HWY|PL|TER|CT|FWY|ALY|CIR|LOOP|WALK|PATH|PLZ|STWY)$/); + if (m) return { base: m[1], suffix: m[2] }; + return { base: name, suffix: null }; +} + +function buildStreets (intersectionRows, aliases) { + // Collect unique full names from both columns. + const namesSeen = new Set(); + for (const row of intersectionRows) { + if (row.street_name_1) namesSeen.add(row.street_name_1); + if (row.street_name_2) namesSeen.add(row.street_name_2); + } + // Build alias reverse-map: { "BAY SHORE BLVD": ["BAYSHORE"], ... } — when a + // canonical street's BASE matches an alias value, list the alias key as an + // alternate. + const aliasReverse = {}; + for (const [aliasKey, aliasValue] of Object.entries(aliases)) { + // The alias maps user-form → DataSF-form base. We attach the user-form to + // every canonical name whose base matches. + aliasReverse[aliasValue] = aliasReverse[aliasValue] ?? []; + aliasReverse[aliasValue].push(aliasKey); + } + + const streets = []; + for (const name of Array.from(namesSeen).sort()) { + const { base, suffix } = splitSuffix(name); + const street = { + name, + base, + suffix, + display: toDisplay(name), + }; + if (aliasReverse[base]) street.aliases = aliasReverse[base]; + streets.push(street); + } + return streets; +} + +function toDisplay (rawName) { + // Strip zero-pad on numbered streets ("03RD" → "3RD"); title-case the rest. + const stripped = rawName.replace(/\b0(\d(ST|ND|RD|TH))\b/g, '$1'); + return stripped + .split(/\s+/) + .map(w => (w ? w[0].toUpperCase() + w.slice(1).toLowerCase() : w)) + .join(' '); +} + +function buildIntersections (rows) { + // Dedupe by CNN. Each CNN appears twice (A→B, B→A); we keep the alphabetically + // earlier (street1, street2) ordering for determinism. + const byCnn = new Map(); + for (const r of rows) { + if (!r.cnn || !r.street_name_1 || !r.street_name_2 || !r.latitude || !r.longitude) continue; + const lat = Number(r.latitude); + const lng = Number(r.longitude); + if (!Number.isFinite(lat) || !Number.isFinite(lng)) continue; + const [s1, s2] = [r.street_name_1, r.street_name_2].sort(); + const entry = { + cnn: r.cnn, + street1: s1, + street2: s2, + lat, + lng, + zip: r.zip_code || null, + }; + const existing = byCnn.get(r.cnn); + if (!existing) { + byCnn.set(r.cnn, entry); + } + } + // Sort by street1 then street2 for stable diffs. + return Array.from(byCnn.values()).sort((a, b) => { + if (a.street1 !== b.street1) return a.street1 < b.street1 ? -1 : 1; + return a.street2 < b.street2 ? -1 : 1; + }); +} + +async function main () { + console.log('Fetching DataSF jfxm-zeee...'); + const rows = await fetchAllRows(); + console.log(`Total rows: ${rows.length}`); + + const aliases = loadAliases(); + console.log(`Aliases: ${Object.keys(aliases).length}`); + + const intersections = buildIntersections(rows); + console.log(`Unique intersections: ${intersections.length}`); + + const streets = buildStreets(rows, aliases); + console.log(`Unique street names: ${streets.length}`); + + fs.mkdirSync(DATA_DIR, { recursive: true }); + fs.writeFileSync( + path.join(DATA_DIR, 'sf-intersections.json'), + JSON.stringify(intersections, null, 2) + '\n' + ); + fs.writeFileSync( + path.join(DATA_DIR, 'sf-streets.json'), + JSON.stringify(streets, null, 2) + '\n' + ); + console.log('Wrote sf-intersections.json + sf-streets.json'); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/server/test/lib/intersections.test.js b/server/test/lib/intersections.test.js new file mode 100644 index 00000000..91285e0e --- /dev/null +++ b/server/test/lib/intersections.test.js @@ -0,0 +1,121 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import intersections from '../../lib/intersections.js'; + +// Integration tests against the vendored sf-intersections.json + sf-streets.json. +// These rely on the JSON being present (built via scripts/build-intersection-data.js). + +test('search — exact known intersection', () => { + const results = intersections.search('16th & Valencia'); + assert.ok(results.length >= 1); + const hit = results.find(r => r.street1 === '16TH ST' && r.street2 === 'VALENCIA ST'); + assert.ok(hit, 'expected to find 16TH ST × VALENCIA ST'); + assert.equal(hit.label, '16th St & Valencia St'); + assert.ok(Math.abs(hit.latitude - 37.7649) < 0.001); + assert.ok(Math.abs(hit.longitude - (-122.4219)) < 0.001); +}); + +test('search — handles "and" connector', () => { + const results = intersections.search('16th and Mission'); + assert.ok(results.some(r => r.label.includes('16th St') && r.label.includes('Mission St'))); +}); + +test('search — handles partial name (Junipero matches Junipero Serra)', () => { + const results = intersections.search('Junipero & Monterey'); + const hit = results.find(r => r.label.includes('Junipero Serra')); + assert.ok(hit, 'expected Junipero Serra Blvd & Monterey Blvd'); +}); + +test('search — handles Bayshore alias (DataSF stores as "BAY SHORE")', () => { + const results = intersections.search('Bayshore & Cortland'); + assert.ok(results.some(r => r.street1.includes('BAY SHORE') || r.street2.includes('BAY SHORE'))); +}); + +test('search — handles "The Embarcadero" with and without "The"', () => { + const withThe = intersections.search('The Embarcadero & Bay'); + const withoutThe = intersections.search('Embarcadero and Bay'); + assert.ok(withThe.length >= 1); + assert.ok(withoutThe.length >= 1); +}); + +test('search — non-cross-street query returns empty', () => { + assert.deepEqual(intersections.search('425 16th St'), []); + assert.deepEqual(intersections.search('Valencia'), []); + assert.deepEqual(intersections.search(''), []); +}); + +test('search — unknown street returns empty', () => { + assert.deepEqual(intersections.search('Nonexistent & Mission'), []); +}); + +test('search — multi-candidate (Fell & Stanyan returns multiple nodes)', () => { + const results = intersections.search('Fell & Stanyan'); + // Fell St & Stanyan St plus Fell Access Rd & Stanyan St — both real + assert.ok(results.length >= 1); + assert.ok(results.every(r => r.street1.startsWith('FELL') || r.street2.startsWith('FELL'))); +}); + +test('search — west-side avenues (numbered Ave intersections)', () => { + const results = intersections.search('Geary & 20th Ave'); + const hit = results.find(r => + (r.street1 === 'GEARY BLVD' && r.street2 === '20TH AVE') || + (r.street2 === 'GEARY BLVD' && r.street1 === '20TH AVE')); + assert.ok(hit, 'expected GEARY BLVD × 20TH AVE'); +}); + +test('search — single-digit numbered streets (3rd & Townsend)', () => { + const results = intersections.search('3rd & Townsend'); + const hit = results.find(r => + (r.street1 === '03RD ST' && r.street2 === 'TOWNSEND ST') || + (r.street2 === '03RD ST' && r.street1 === 'TOWNSEND ST')); + assert.ok(hit, 'expected 03RD ST × TOWNSEND ST'); + // Display should strip the zero-pad + assert.ok(hit.label.startsWith('3rd St') || hit.label.endsWith('3rd St')); +}); + +test('findNearest — returns N nodes ordered by distance', () => { + // 16th & Valencia coordinates + const results = intersections.findNearest(37.76492, -122.42189, { n: 3 }); + assert.equal(results.length, 3); + // First result should be 16TH ST × VALENCIA ST itself + assert.equal(results[0].cnn, '24183000'); + // Distances should be monotonically increasing + assert.ok(results[0].distanceMeters <= results[1].distanceMeters); + assert.ok(results[1].distanceMeters <= results[2].distanceMeters); +}); + +test('findByCnn — known CNN returns row', () => { + const result = intersections.findByCnn('24183000'); + assert.ok(result); + assert.equal(result.street1, '16TH ST'); + assert.equal(result.street2, 'VALENCIA ST'); +}); + +test('findByCnn — unknown CNN returns null', () => { + assert.equal(intersections.findByCnn('99999999'), null); +}); + +test('parseOrRecover — recovers from "and Mission" → "admission" fusion', () => { + // The STT collapse case from real-world voice transcription. + const parsed = intersections.parseOrRecover('14th admission'); + assert.ok(parsed); + assert.equal(parsed.side1, '14th'); + assert.equal(parsed.side2.toUpperCase(), 'MISSION'); +}); + +test('parseOrRecover — direct parse still works when connector is present', () => { + assert.deepEqual( + intersections.parseOrRecover('16th & Valencia'), + { side1: '16th', side2: 'Valencia' }, + ); +}); + +test('parseOrRecover — rejects unrelated tokens (no fusion match)', () => { + assert.equal(intersections.parseOrRecover('quickbrown fox jumped'), null); +}); + +test('search — recovers end-to-end from "X admission" voice fusion', () => { + const results = intersections.search('14th admission'); + assert.ok(results.some(r => r.label === '14th St & Mission St')); +}); diff --git a/server/test/lib/phoneticMatch.test.js b/server/test/lib/phoneticMatch.test.js new file mode 100644 index 00000000..17458f37 --- /dev/null +++ b/server/test/lib/phoneticMatch.test.js @@ -0,0 +1,55 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { buildMatcher } from '../../lib/phoneticMatch.js'; + +const SF_STREETS_SUBSET = [ + { name: 'JUNIPERO SERRA BLVD', base: 'JUNIPERO SERRA', display: 'Junipero Serra Blvd', suffix: 'BLVD' }, + { name: 'CESAR CHAVEZ ST', base: 'CESAR CHAVEZ', display: 'Cesar Chavez St', suffix: 'ST' }, + { name: 'VALENCIA ST', base: 'VALENCIA', display: 'Valencia St', suffix: 'ST' }, + { name: 'MISSION ST', base: 'MISSION', display: 'Mission St', suffix: 'ST' }, + { name: 'MONTEREY BLVD', base: 'MONTEREY', display: 'Monterey Blvd', suffix: 'BLVD' }, + { name: 'EMBARCADERO', base: 'EMBARCADERO', display: 'The Embarcadero', suffix: null }, + { name: 'THE EMBARCADERO', base: 'THE EMBARCADERO', display: 'The Embarcadero', suffix: null }, + { name: 'GEARY BLVD', base: 'GEARY', display: 'Geary Blvd', suffix: 'BLVD' }, + { name: 'POLK ST', base: 'POLK', display: 'Polk St', suffix: 'ST' }, + { name: 'SARAH ST', base: 'SARAH', display: 'Sarah St', suffix: 'ST' }, +]; + +test('match — Junipero Sarah → Junipero Serra (the headline case)', () => { + const matcher = buildMatcher(SF_STREETS_SUBSET); + const hits = matcher.match('Junipero Sarah'); + assert.ok(hits.length >= 1); + assert.equal(hits[0].name, 'JUNIPERO SERRA BLVD'); + assert.ok(hits[0].score > 0.7, `expected high score, got ${hits[0].score}`); +}); + +test('match — single-word phonetic equivalents', () => { + const matcher = buildMatcher(SF_STREETS_SUBSET); + // Both should phonetically equal Sarah's SR code → finds both Serra and Sarah + const hits = matcher.match('Sarah'); + assert.ok(hits.length >= 1); + // Top hit could be the literal SARAH ST or JUNIPERO SERRA depending on + // Levenshtein; both should appear. + const names = hits.map(h => h.name); + assert.ok(names.includes('SARAH ST')); +}); + +test('match — partial multi-word input (Junipero alone matches JUNIPERO SERRA)', () => { + const matcher = buildMatcher(SF_STREETS_SUBSET); + const hits = matcher.match('Junipero'); + // "Junipero" alone shouldn't phonetically match "JUNIPERO SERRA" exactly, + // but the prefix-fallback path should catch it. + assert.ok(hits.some(h => h.name === 'JUNIPERO SERRA BLVD')); +}); + +test('match — empty input returns empty', () => { + const matcher = buildMatcher(SF_STREETS_SUBSET); + assert.deepEqual(matcher.match(''), []); +}); + +test('match — limit caps the result count', () => { + const matcher = buildMatcher(SF_STREETS_SUBSET); + const hits = matcher.match('Mission', { limit: 1 }); + assert.equal(hits.length, 1); +}); diff --git a/server/test/lib/streetNormalization.test.js b/server/test/lib/streetNormalization.test.js new file mode 100644 index 00000000..2530dbf3 --- /dev/null +++ b/server/test/lib/streetNormalization.test.js @@ -0,0 +1,157 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + expandSuffix, + expandOrdinalWords, + toPrefixes, + displayName, + titleCase, + parseCrossStreetInput, +} from '../../lib/streetNormalization.js'; + +test('expandSuffix — long-form suffix → DataSF abbreviation', () => { + assert.equal(expandSuffix('Valencia Street'), 'VALENCIA ST'); + assert.equal(expandSuffix('19th Avenue'), '19TH AVE'); + assert.equal(expandSuffix('Junipero Serra Boulevard'), 'JUNIPERO SERRA BLVD'); + assert.equal(expandSuffix('Maiden Lane'), 'MAIDEN LN'); + assert.equal(expandSuffix('Twin Peaks Drive'), 'TWIN PEAKS DR'); + assert.equal(expandSuffix('Sacramento Road'), 'SACRAMENTO RD'); + assert.equal(expandSuffix('Acme Alley'), 'ACME ALY'); +}); + +test('expandSuffix — already-abbreviated suffix passes through', () => { + assert.equal(expandSuffix('Valencia St'), 'VALENCIA ST'); + assert.equal(expandSuffix('19th Ave'), '19TH AVE'); + assert.equal(expandSuffix('BAY SHORE BLVD'), 'BAY SHORE BLVD'); +}); + +test('expandSuffix — no suffix passes through uppercase', () => { + assert.equal(expandSuffix('Embarcadero'), 'EMBARCADERO'); + assert.equal(expandSuffix('the embarcadero'), 'THE EMBARCADERO'); +}); + +test('toPrefixes — strips trailing suffix for prefix-match', () => { + assert.ok(toPrefixes('Valencia St').includes('VALENCIA')); + assert.ok(toPrefixes('Junipero Serra Blvd').includes('JUNIPERO SERRA')); +}); + +test('toPrefixes — zero-pads single-digit numbered streets', () => { + assert.ok(toPrefixes('3rd St').includes('03RD')); + assert.ok(toPrefixes('7th').includes('07TH')); + // 10th and above should not be padded + assert.ok(toPrefixes('16th St').includes('16TH')); + assert.ok(!toPrefixes('16th St').some(p => p.startsWith('0'))); +}); + +test('toPrefixes — handles leading THE bidirectionally', () => { + assert.ok(toPrefixes('Embarcadero').includes('THE EMBARCADERO')); + assert.ok(toPrefixes('Embarcadero').includes('EMBARCADERO')); + assert.ok(toPrefixes('The Embarcadero').includes('EMBARCADERO')); + assert.ok(toPrefixes('The Embarcadero').includes('THE EMBARCADERO')); +}); + +test('toPrefixes — applies BAYSHORE → BAY SHORE alias', () => { + const prefixes = toPrefixes('Bayshore'); + assert.ok(prefixes.includes('BAYSHORE')); + assert.ok(prefixes.includes('BAY SHORE')); +}); + +test('toPrefixes — does not over-pad multi-digit ordinals', () => { + assert.ok(!toPrefixes('25th Ave').some(p => p.startsWith('025'))); +}); + +test('toPrefixes — long-form suffix expanded before stripping', () => { + assert.ok(toPrefixes('Maiden Lane').includes('MAIDEN')); +}); + +test('toPrefixes — accepts uppercase input', () => { + assert.ok(toPrefixes('VALENCIA ST').includes('VALENCIA')); +}); + +test('displayName — title-cases with zero-pad stripped', () => { + assert.equal(displayName('16TH ST'), '16th St'); + assert.equal(displayName('03RD ST'), '3rd St'); + assert.equal(displayName('JUNIPERO SERRA BLVD'), 'Junipero Serra Blvd'); + assert.equal(displayName('THE EMBARCADERO'), 'The Embarcadero'); + assert.equal(displayName('BAY SHORE BLVD'), 'Bay Shore Blvd'); +}); + +test('titleCase — basic', () => { + assert.equal(titleCase('FOO BAR'), 'Foo Bar'); + assert.equal(titleCase('foo'), 'Foo'); + assert.equal(titleCase(''), ''); +}); + +test('parseCrossStreetInput — recognizes &, /, @, and, at', () => { + assert.deepEqual(parseCrossStreetInput('16th & Valencia'), { side1: '16th', side2: 'Valencia' }); + assert.deepEqual(parseCrossStreetInput('16th / Valencia'), { side1: '16th', side2: 'Valencia' }); + assert.deepEqual(parseCrossStreetInput('16th @ Valencia'), { side1: '16th', side2: 'Valencia' }); + assert.deepEqual(parseCrossStreetInput('16th and Valencia'), { side1: '16th', side2: 'Valencia' }); + assert.deepEqual(parseCrossStreetInput('16th at Valencia'), { side1: '16th', side2: 'Valencia' }); +}); + +test('parseCrossStreetInput — strips trailing city/state tail', () => { + assert.deepEqual( + parseCrossStreetInput('16th & Valencia, San Francisco, CA'), + { side1: '16th', side2: 'Valencia' } + ); + assert.deepEqual( + parseCrossStreetInput('Junipero Serra and Monterey, SF'), + { side1: 'Junipero Serra', side2: 'Monterey' } + ); +}); + +test('parseCrossStreetInput — rejects addresses (digit + non-ordinal word)', () => { + assert.equal(parseCrossStreetInput('425 16th St'), null); + assert.equal(parseCrossStreetInput('100 Mission St'), null); +}); + +test('parseCrossStreetInput — accepts ordinal-starting intersections', () => { + // "16th & Mission" starts with a digit but is an ordinal — still a cross-street + assert.deepEqual(parseCrossStreetInput('16th & Mission'), { side1: '16th', side2: 'Mission' }); +}); + +test('parseCrossStreetInput — rejects single-side input', () => { + assert.equal(parseCrossStreetInput('Valencia St'), null); + assert.equal(parseCrossStreetInput('16th'), null); +}); + +test('parseCrossStreetInput — handles multi-word street names', () => { + assert.deepEqual( + parseCrossStreetInput('Junipero Serra & Monterey'), + { side1: 'Junipero Serra', side2: 'Monterey' } + ); + assert.deepEqual( + parseCrossStreetInput('Cesar Chavez and Mission'), + { side1: 'Cesar Chavez', side2: 'Mission' } + ); +}); + +test('parseCrossStreetInput — empty/null returns null', () => { + assert.equal(parseCrossStreetInput(''), null); + assert.equal(parseCrossStreetInput(' '), null); +}); + +test('expandOrdinalWords — single-word ordinals', () => { + assert.equal(expandOrdinalWords('Third'), '3RD'); + assert.equal(expandOrdinalWords('Sixteenth'), '16TH'); + assert.equal(expandOrdinalWords('Nineteenth Avenue'), '19TH AVENUE'); +}); + +test('expandOrdinalWords — compound ordinals (hyphenated and spaced)', () => { + assert.equal(expandOrdinalWords('Twenty-Fourth'), '24TH'); + assert.equal(expandOrdinalWords('twenty fourth'), '24TH'); + assert.equal(expandOrdinalWords('Twenty-First Avenue'), '21ST AVENUE'); +}); + +test('expandOrdinalWords — leaves non-ordinal words alone', () => { + assert.equal(expandOrdinalWords('Mission Street'), 'MISSION STREET'); + assert.equal(expandOrdinalWords('Junipero Serra'), 'JUNIPERO SERRA'); +}); + +test('toPrefixes — handles ordinal-word input ("Third" → "03RD")', () => { + assert.ok(toPrefixes('Third').includes('03RD')); + assert.ok(toPrefixes('Sixteenth').includes('16TH')); + assert.ok(toPrefixes('Twenty-First').includes('21ST')); +}); diff --git a/server/test/routes/api/geocode-intersections.test.js b/server/test/routes/api/geocode-intersections.test.js new file mode 100644 index 00000000..c9e4ccb7 --- /dev/null +++ b/server/test/routes/api/geocode-intersections.test.js @@ -0,0 +1,103 @@ +import { test } from 'node:test'; +import * as assert from 'node:assert'; +import { StatusCodes } from 'http-status-codes'; + +import { authenticate, build } from '#test/helper.js'; + +test('/api/geocode/intersections', async (t) => { + const app = await build(t); + + await t.test('returns 401 without authentication', async () => { + const response = await app.inject() + .get('/api/geocode/intersections?text=16th+%26+Valencia'); + assert.deepStrictEqual(response.statusCode, StatusCodes.UNAUTHORIZED); + }); + + await t.test('returns 400 when text param is missing', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections') + .headers(userHeaders); + assert.deepStrictEqual(response.statusCode, StatusCodes.UNPROCESSABLE_ENTITY); + }); + + await t.test('returns intersection matches for a valid cross-street query', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections?text=16th+%26+Valencia') + .headers(userHeaders); + + assert.deepStrictEqual(response.statusCode, StatusCodes.OK); + const data = JSON.parse(response.body); + assert.ok(Array.isArray(data)); + assert.ok(data.length >= 1, 'expected at least one match'); + const hit = data.find(r => r.street1 === '16TH ST' && r.street2 === 'VALENCIA ST'); + assert.ok(hit, '16TH ST × VALENCIA ST should be in results'); + assert.deepStrictEqual(hit.label, '16th St & Valencia St'); + assert.ok(typeof hit.latitude === 'number'); + assert.ok(typeof hit.longitude === 'number'); + }); + + await t.test('returns empty array for a non-cross-street query', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections?text=425+16th+St') + .headers(userHeaders); + assert.deepStrictEqual(response.statusCode, StatusCodes.OK); + assert.deepStrictEqual(JSON.parse(response.body), []); + }); + + await t.test('handles "and" connector', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections?text=Junipero+Serra+and+Monterey') + .headers(userHeaders); + assert.deepStrictEqual(response.statusCode, StatusCodes.OK); + const data = JSON.parse(response.body); + assert.ok(data.some(r => r.label.includes('Junipero Serra')), 'expected Junipero Serra match'); + }); + + await t.test('respects limit param', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections?text=Fell+%26+Stanyan&limit=1') + .headers(userHeaders); + assert.deepStrictEqual(response.statusCode, StatusCodes.OK); + const data = JSON.parse(response.body); + assert.ok(data.length <= 1); + }); +}); + +test('/api/geocode/intersections/nearest', async (t) => { + const app = await build(t); + + await t.test('returns 401 without authentication', async () => { + const response = await app.inject() + .get('/api/geocode/intersections/nearest?latitude=37.76492&longitude=-122.42189'); + assert.deepStrictEqual(response.statusCode, StatusCodes.UNAUTHORIZED); + }); + + await t.test('returns N nearest intersections ordered by distance', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections/nearest?latitude=37.76492&longitude=-122.42189&n=3') + .headers(userHeaders); + assert.deepStrictEqual(response.statusCode, StatusCodes.OK); + const data = JSON.parse(response.body); + assert.deepStrictEqual(data.length, 3); + // First result should be the exact 16th & Valencia intersection. + assert.deepStrictEqual(data[0].cnn, '24183000'); + // Distances should be monotonically increasing. + assert.ok(data[0].distanceMeters <= data[1].distanceMeters); + assert.ok(data[1].distanceMeters <= data[2].distanceMeters); + }); + + await t.test('defaults to n=2 when not specified', async () => { + const userHeaders = await authenticate(app, 'regular.user@test.com', 'test'); + const response = await app.inject() + .get('/api/geocode/intersections/nearest?latitude=37.76492&longitude=-122.42189') + .headers(userHeaders); + assert.deepStrictEqual(response.statusCode, StatusCodes.OK); + assert.deepStrictEqual(JSON.parse(response.body).length, 2); + }); +}); From 40e7fc014f4e6d5c3f3433e27effa7a0a262d6c7 Mon Sep 17 00:00:00 2001 From: Saad Abdali Date: Wed, 20 May 2026 21:58:23 -0400 Subject: [PATCH 2/3] Remove tap-to-confirm, add neighborhood label --- .../src/components/LocationAutocomplete.jsx | 26 +++++- client/src/lesc/components/IncidentForm.jsx | 92 ++++++++++++++----- docs/cross-streets-spot-fixes.md | 28 +++++- server/lib/intersections.js | 1 + server/lib/location.js | 1 + server/routes/api/ai/transcribe.js | 1 + server/routes/api/geocode/intersections.js | 1 + server/routes/api/geocode/reverse.js | 1 + server/scripts/build-intersection-data.js | 70 ++++++++++++++ 9 files changed, 195 insertions(+), 26 deletions(-) diff --git a/client/src/components/LocationAutocomplete.jsx b/client/src/components/LocationAutocomplete.jsx index 0360d65a..3cb99d73 100644 --- a/client/src/components/LocationAutocomplete.jsx +++ b/client/src/components/LocationAutocomplete.jsx @@ -39,6 +39,12 @@ const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, r const [loading, setLoading] = useState(false); const resultsRef = useRef({ mode: null, items: [] }); const abortControllerRef = useRef(null); + // Set true at the top of handleOptionSubmit and cleared on the next tick. + // Any onChange that fires during this window is from the option-submit + // itself (Mantine echoes the picked value through onChange after the pick), + // so we skip the invalidation logic — the option-submit already set the + // values it wants. + const skipInvalidationRef = useRef(false); const [debounced] = useDebouncedValue(value, 300); useEffect(() => { @@ -68,7 +74,10 @@ const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, r if (intersectionMode) { setData(items .filter((r) => r.cnn) - .map((r) => ({ value: r.cnn, label: `⌗ ${r.label}` }))); + .map((r) => ({ + value: r.cnn, + label: r.neighborhood ? `⌗ ${r.label} (${r.neighborhood})` : `⌗ ${r.label}`, + }))); } else { setData(items .filter((r) => r.placeId && r.addressLine1) @@ -93,6 +102,16 @@ const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, r function handleChange (val) { setValue(val); + if (skipInvalidationRef.current) { + return; + } + // Real user-typed change. Invalidate the cached neighborhood hint and + // intersectionId — both were tied to a prior selection that no longer + // matches what the user is typing. + const fv = form.getValues(); + if (fv.neighborhood || fv.intersectionId) { + form.setValues({ neighborhood: '', intersectionId: '' }); + } } function handleFocus (e) { @@ -110,6 +129,9 @@ const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, r const match = items.find((r) => (mode === 'INTERSECTION' ? r.cnn : r.placeId) === key); if (!match) return; + skipInvalidationRef.current = true; + setTimeout(() => { skipInvalidationRef.current = false; }, 0); + if (mode === 'INTERSECTION') { const label = `${match.street1Display} & ${match.street2Display}`; setValue(label); @@ -118,6 +140,7 @@ const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, r street1: match.street1Display, street2: match.street2Display, intersectionId: match.cnn, + neighborhood: match.neighborhood ?? '', city: 'San Francisco', state: 'CA', latitude: match.latitude, @@ -134,6 +157,7 @@ const LocationAutocomplete = forwardRef(function LocationAutocomplete ({ form, r city: match.city ?? '', state: match.state ?? '', postalCode: match.postalCode?.split('-')[0] ?? '', + neighborhood: match.neighborhood ?? '', latitude: match.latitude ?? '', longitude: match.longitude ?? '', street1: null, diff --git a/client/src/lesc/components/IncidentForm.jsx b/client/src/lesc/components/IncidentForm.jsx index f5e8b6aa..0494c861 100644 --- a/client/src/lesc/components/IncidentForm.jsx +++ b/client/src/lesc/components/IncidentForm.jsx @@ -44,6 +44,11 @@ const initialValues = { street1: '', street2: '', intersectionId: '', + // `neighborhood` is a transient client-only hint sourced from autocomplete, + // voice match, or geolocation. It's never sent to the server (see + // buildIncidentPayload) — it's just used to render the "in X" confidence + // line under the field. + neighborhood: '', latitude: '', longitude: '', arrestedAt: '', @@ -66,6 +71,7 @@ function normalizeIncidentFormValues (values) { street1: values?.street1 ?? '', street2: values?.street2 ?? '', intersectionId: values?.intersectionId ?? '', + neighborhood: values?.neighborhood ?? '', latitude: values?.latitude ?? '', longitude: values?.longitude ?? '', arrestedAt: values?.arrestedAt ?? '', @@ -82,24 +88,28 @@ function buildIncidentPayload (values) { zone: 'local', }); + // `neighborhood` is a transient client-only hint; strip it out so we don't + // send a non-schema field to the server. + const { neighborhood: _neighborhood, ...rest } = values; + return { - ...values, - cadNumber: emptyStringToNull(values.cadNumber), - caseNumber: emptyStringToNull(values.caseNumber), - encounteredVia: emptyStringToNull(values.encounteredVia), - locationType: values.locationType ?? 'ADDRESS', - addressLine1: emptyStringToNull(values.addressLine1), - addressLine2: emptyStringToNull(values.addressLine2), - city: emptyStringToNull(values.city), - state: emptyStringToNull(values.state), - postalCode: emptyStringToNull(values.postalCode), - street1: emptyStringToNull(values.street1), - street2: emptyStringToNull(values.street2), - intersectionId: emptyStringToNull(values.intersectionId), - latitude: emptyStringToNull(values.latitude), - longitude: emptyStringToNull(values.longitude), + ...rest, + cadNumber: emptyStringToNull(rest.cadNumber), + caseNumber: emptyStringToNull(rest.caseNumber), + encounteredVia: emptyStringToNull(rest.encounteredVia), + locationType: rest.locationType ?? 'ADDRESS', + addressLine1: emptyStringToNull(rest.addressLine1), + addressLine2: emptyStringToNull(rest.addressLine2), + city: emptyStringToNull(rest.city), + state: emptyStringToNull(rest.state), + postalCode: emptyStringToNull(rest.postalCode), + street1: emptyStringToNull(rest.street1), + street2: emptyStringToNull(rest.street2), + intersectionId: emptyStringToNull(rest.intersectionId), + latitude: emptyStringToNull(rest.latitude), + longitude: emptyStringToNull(rest.longitude), arrestedAt: arrestedAt.isValid ? arrestedAt.toISO() : null, - supervisorBadgeNumber: emptyStringToNull(values.supervisorBadgeNumber), + supervisorBadgeNumber: emptyStringToNull(rest.supervisorBadgeNumber), }; } @@ -209,6 +219,7 @@ function IncidentForm () { form.setValues({ locationType: 'ADDRESS', ...address, + neighborhood: address?.neighborhood ?? '', street1: null, street2: null, intersectionId: null, @@ -217,9 +228,15 @@ function IncidentForm () { }; function handleVoiceResult (data) { - // Always show the confirmation chip — even a single high-confidence match - // gets a tap-to-confirm step. Officer-entered evidence shouldn't silently - // commit; the chip is the place where voice → field becomes deliberate. + // Single confident match → auto-apply and stay silent (the field itself + // updates, which is feedback enough). Multi-match → show the picker chip. + // No match → keep voiceResult so the chip can show the "couldn't match" + // hint and surface the raw transcript. + if (data?.matches?.length === 1) { + applyIntersectionMatch(data.matches[0]); + setVoiceResult(null); + return; + } setVoiceResult(data); } @@ -229,6 +246,7 @@ function IncidentForm () { street1: match.street1Display, street2: match.street2Display, intersectionId: match.cnn, + neighborhood: match.neighborhood ?? '', city: 'San Francisco', state: 'CA', latitude: match.latitude, @@ -239,6 +257,28 @@ function IncidentForm () { }); } + // Any manual edit of a location-defining field invalidates derived hints. + // We clear `neighborhood` (the confidence indicator) and `intersectionId` + // (which would otherwise point at a stale CNN). The setValues call also + // triggers a re-render so the hint disappears. + function invalidateDerivedLocationState () { + const v = form.getValues(); + if (v.neighborhood || v.intersectionId) { + form.setValues({ neighborhood: '', intersectionId: '' }); + } + } + + function wrapInputForLocationField (field) { + const props = form.getInputProps(field); + return { + ...props, + onChange: (event) => { + props.onChange(event); + invalidateDerivedLocationState(); + }, + }; + } + const onSubmitMutation = useMutation({ mutationFn: (formData) => isEditing @@ -328,6 +368,11 @@ function IncidentForm () { setTimeout(() => addressRef.current?.focus(), 100); }} /> + {form.getValues().neighborhood && ( + + in {form.getValues().neighborhood} + + )} { applyIntersectionMatch(m); setVoiceResult(null); }} @@ -360,6 +405,11 @@ function IncidentForm () { } rightSectionWidth={84} /> + {form.getValues().neighborhood && ( + + in {form.getValues().neighborhood} + + )} { applyIntersectionMatch(m); setVoiceResult(null); }} @@ -372,13 +422,13 @@ function IncidentForm () { Street 1*} /> Street 2*} /> diff --git a/docs/cross-streets-spot-fixes.md b/docs/cross-streets-spot-fixes.md index 65157b89..b792546a 100644 --- a/docs/cross-streets-spot-fixes.md +++ b/docs/cross-streets-spot-fixes.md @@ -69,17 +69,17 @@ Pattern tags emerging so far: --- -## #4 — Confirmation chip never appeared after a successful voice match +## #4 — Confirmation chip never appeared after a successful voice match *(later reverted in #9)* **Observed:** User reported: "I don't see a chip anywhere after I successfully enter an intersection." -**Diagnosis:** Mid-implementation I'd added a "if exactly 1 match, auto-fill and skip the chip" optimization in `IncidentForm.jsx:handleVoiceResult`. For the *common* case (e.g. "16th and Valencia" → 1 clean match), this meant zero feedback that voice was processed — the field silently updated. This directly contradicted what the implementation plan called for ("show a confirmation chip before commit. CAD systems do this. It's cheap, prevents silent misroutes."). +**Diagnosis:** Mid-implementation I'd added a "if exactly 1 match, auto-fill and skip the chip" optimization in `IncidentForm.jsx:handleVoiceResult`. For the *common* case (e.g. "16th and Valencia" → 1 clean match), this meant zero feedback that voice was processed — the field silently updated. **Fix:** removed the shortcut. Chip always shows when there's at least one match. Tightened the wording: single match reads "Tap to confirm:", multi reads "Did you mean:" (`client/src/components/LocationVoiceButton.jsx:LocationConfirmationChip`). **Pattern tag:** `ux-feedback` -**Notes:** This was a self-inflicted optimization that I added without thinking through the implications for evidence integrity (officer-entered data). Worth a reminder to myself: don't optimize away verification steps in officer/evidence flows. +**Notes:** **This change was reversed in #9 once the user tried it in practice.** See #9 for the final state. The lesson here is in the spread between *theory* (CAD-style verification is the cautious default) and *use* (one extra tap on the happy path felt like friction, and the field updating *is* feedback). --- @@ -160,12 +160,32 @@ Tests added for ordinal expansion (`expandOrdinalWords`) and ordinal-aware `toPr --- +## #9 — Reverted #4: single-match voice should auto-apply silently + +**Observed:** After using the always-show-chip behavior from #4 in practice, the user said: +> "I don't like the 'show a confirmation, then user taps to confirm'. I don't mind the other panel (when we can't match the voice to something, we at least display what we heard, and encourage the user to type it in). But I find that when the system is able to transcribe my audio to a real intersection, I don't want a separate tap, I just want it to populate the field." + +**Diagnosis:** The chip's value is genuine on multi-match (real ambiguity that needs a choice) and no-match (failure mode that needs explanation + the raw transcript). On a clean single match, the field itself updating is sufficient feedback — adding a confirmation tap on the *happy path* is friction without value. + +**Fix:** restored the single-match auto-apply shortcut in `IncidentForm.jsx:handleVoiceResult`. New three-way logic: +- **1 match** → `applyIntersectionMatch(matches[0])`, clear `voiceResult` (no chip) +- **2+ matches** → `setVoiceResult(data)` so the chip shows the picker +- **0 matches / unparseable** → `setVoiceResult(data)` so the chip shows the "couldn't match" hint with the raw transcript + +**Pattern tag:** `ux-feedback` + +**Notes:** This is a meta-lesson about my earlier reasoning in #4. I framed the chip as protecting "evidence integrity," but in practice the verification CAD systems do is for *dispatchers* coordinating with field officers over radio — different context. For a field officer self-entering, the field's own value display is the verification. Default to the lighter UX; add friction only where ambiguity or failure genuinely requires it. + +The previous chip wording change ("Tap to confirm:" vs "Did you mean:") stays — it's still relevant for the multi-match case the chip *does* render in. Single match just doesn't render the chip at all anymore. + +--- + ## Pattern summary so far - **`data-shape-assumption`** (3 occurrences): #1, #2, #3. Each was a code path that assumed address-only and didn't know about the locationType discriminator. **If another shows up, lean on a typed read-side wrapper** (e.g. `getIncidentLocation(incident) → { type: 'address'|'intersection', text, lat, lng }`) and migrate call sites to it rather than continuing inline branching. - **`ranking-bug`** (1 occurrence): #6. Test coverage gap — popular base names with compound siblings need integration tests. - **`stt-fusion`** (2 occurrences): #7, #8. Both handled with focused fixes in the normalization/recovery pipeline. **If 3+ more such fixes pile up, escalate the STT strategy** to Deepgram Nova-3 per the implementation plan's Part 7 gate. #8 in particular suggests a more general principle: anywhere a canonical name has a non-phonetic transformation from speech (digits ↔ words, abbreviations, leading "THE"), we need explicit normalization — phonetic match alone won't bridge it. -- **`ux-feedback`** (2 occurrences): #4, #5. Both were my misses, not systemic. Manual UX review would have caught both before user found them. +- **`ux-feedback`** (3 occurrences): #4, #5, #9. #5 was a genuine miss. #4 + #9 together form one back-and-forth: I built the cautious-by-default UX, the user tried it and preferred the lighter one. Lesson: when defaulting cautious vs. light on a UX call where I'm guessing, prefer light + escalate friction only where ambiguity/failure requires it — and surface the choice early instead of letting it ship and bounce. --- diff --git a/server/lib/intersections.js b/server/lib/intersections.js index 5caecdfe..c514ba5c 100644 --- a/server/lib/intersections.js +++ b/server/lib/intersections.js @@ -245,6 +245,7 @@ function toApiResult (row) { latitude: row.lat, longitude: row.lng, zipCode: row.zip ?? null, + neighborhood: row.neighborhood ?? null, }; } diff --git a/server/lib/location.js b/server/lib/location.js index 1a3a5e8f..82e21be6 100644 --- a/server/lib/location.js +++ b/server/lib/location.js @@ -58,6 +58,7 @@ async function reverseGeocode (latitude, longitude) { city: address?.Locality || null, state: address?.Region?.Code || null, postalCode: address?.PostalCode || null, + neighborhood: address?.District || null, }; } diff --git a/server/routes/api/ai/transcribe.js b/server/routes/api/ai/transcribe.js index d2d3cafc..7aada706 100644 --- a/server/routes/api/ai/transcribe.js +++ b/server/routes/api/ai/transcribe.js @@ -53,6 +53,7 @@ export default async function (fastify) { latitude: z.number(), longitude: z.number(), zipCode: z.string().nullable(), + neighborhood: z.string().nullable(), })).optional(), parsed: z.object({ side1: z.string(), diff --git a/server/routes/api/geocode/intersections.js b/server/routes/api/geocode/intersections.js index 5022d66e..5889eb84 100644 --- a/server/routes/api/geocode/intersections.js +++ b/server/routes/api/geocode/intersections.js @@ -13,6 +13,7 @@ const IntersectionResultSchema = z.object({ latitude: z.number(), longitude: z.number(), zipCode: z.string().nullable(), + neighborhood: z.string().nullable(), }); const NearestResultSchema = IntersectionResultSchema.extend({ diff --git a/server/routes/api/geocode/reverse.js b/server/routes/api/geocode/reverse.js index 07fe305c..3e1e2424 100644 --- a/server/routes/api/geocode/reverse.js +++ b/server/routes/api/geocode/reverse.js @@ -19,6 +19,7 @@ export default async function (fastify, opts) { city: z.string().nullable(), state: z.string().nullable(), postalCode: z.string().nullable(), + neighborhood: z.string().nullable(), }), }, }, diff --git a/server/scripts/build-intersection-data.js b/server/scripts/build-intersection-data.js index ed3930e1..598a96e4 100644 --- a/server/scripts/build-intersection-data.js +++ b/server/scripts/build-intersection-data.js @@ -18,6 +18,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const DATA_DIR = path.resolve(__dirname, '..', 'data'); const SOCRATA_RESOURCE = 'https://data.sfgov.org/resource/jfxm-zeee.json'; +const NEIGHBORHOODS_GEOJSON = 'https://data.sfgov.org/resource/j2bu-swwd.geojson?$limit=200'; const PAGE_SIZE = 5000; async function fetchAllRows () { @@ -127,6 +128,69 @@ function buildIntersections (rows) { }); } +async function fetchNeighborhoodPolygons () { + const response = await fetch(NEIGHBORHOODS_GEOJSON); + if (!response.ok) { + throw new Error(`Neighborhood fetch failed: ${response.status}`); + } + const geojson = await response.json(); + // Each feature has properties.nhood and a (Multi)Polygon geometry. Flatten + // MultiPolygon into one polygon ring list per feature; we only need the + // outer ring of each polygon for point-in-poly. SF neighborhoods have no + // holes in practice. + const polygons = []; + for (const feature of geojson.features ?? []) { + const name = feature?.properties?.nhood; + const geom = feature?.geometry; + if (!name || !geom) continue; + if (geom.type === 'Polygon') { + polygons.push({ name, rings: [geom.coordinates[0]] }); + } else if (geom.type === 'MultiPolygon') { + const rings = geom.coordinates.map(poly => poly[0]); + polygons.push({ name, rings }); + } + } + return polygons; +} + +// Ray-casting point-in-polygon (lng, lat against ring of [lng, lat] points). +function pointInRing (lng, lat, ring) { + let inside = false; + for (let i = 0, j = ring.length - 1; i < ring.length; j = i++) { + const xi = ring[i][0]; + const yi = ring[i][1]; + const xj = ring[j][0]; + const yj = ring[j][1]; + const intersects = ((yi > lat) !== (yj > lat)) && + (lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi); + if (intersects) inside = !inside; + } + return inside; +} + +function neighborhoodFor (lat, lng, polygons) { + for (const poly of polygons) { + for (const ring of poly.rings) { + if (pointInRing(lng, lat, ring)) return poly.name; + } + } + return null; +} + +function annotateNeighborhoods (intersections, polygons) { + let matched = 0; + for (const row of intersections) { + const hood = neighborhoodFor(row.lat, row.lng, polygons); + if (hood) { + row.neighborhood = hood; + matched++; + } else { + row.neighborhood = null; + } + } + return matched; +} + async function main () { console.log('Fetching DataSF jfxm-zeee...'); const rows = await fetchAllRows(); @@ -138,6 +202,12 @@ async function main () { const intersections = buildIntersections(rows); console.log(`Unique intersections: ${intersections.length}`); + console.log('Fetching SF Analysis Neighborhoods...'); + const polygons = await fetchNeighborhoodPolygons(); + console.log(`Neighborhood polygons: ${polygons.length}`); + const matched = annotateNeighborhoods(intersections, polygons); + console.log(`Intersections matched to a neighborhood: ${matched} / ${intersections.length}`); + const streets = buildStreets(rows, aliases); console.log(`Unique street names: ${streets.length}`); From c5fdf0802bc8b92ef189ae2ece46f36144fcade5 Mon Sep 17 00:00:00 2001 From: Saad Abdali Date: Wed, 10 Jun 2026 19:52:50 -0400 Subject: [PATCH 3/3] Spot fixes to intersection data --- docs/cross-streets-spot-fixes.md | 54 ++++++++++++++++++++++- server/scripts/build-intersection-data.js | 21 ++++----- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/docs/cross-streets-spot-fixes.md b/docs/cross-streets-spot-fixes.md index b792546a..b353a0fb 100644 --- a/docs/cross-streets-spot-fixes.md +++ b/docs/cross-streets-spot-fixes.md @@ -5,6 +5,36 @@ feature, with diagnosis and fix. Kept so we can spot recurring patterns and decide if/when to revise the underlying strategy rather than continue whack-a-mole. +## Where we left off — session resume notes + +**Branch:** `sa/cross-streets`. Not yet committed at session close — every change above is in the working tree. + +**Phases of `docs/cross-streets-implementation-plan.md` completed:** +- ✅ P1 Data pipeline — vendored DataSF index (now 11,006 intersections with neighborhood), normalization library, 45 unit tests +- ✅ P2 Schema — `locationType` + `street1`/`street2`/`intersectionId` on `Incident` (applied via `prisma db push`) +- ✅ P3 Server endpoints — `/api/geocode/intersections` (search + nearest), voice transcribe with `mode=location` and phonetic-fallback pipeline +- ✅ P4 Client UI — `LocationAutocomplete`, voice mic on collapsed view, confirmation chip (multi-match or no-match only — single match auto-applies per #9), neighborhood hint with manual-edit invalidation +- ✅ P5 Display sites — `Incident.jsx`, `CustodyDetailContent.jsx`, `facilityAddressLink` map URLs, "between context" component for the detail view + +**Phases NOT done:** +- ❌ P6 Tests — only the lib tests are written (`server/test/lib/{streetNormalization,intersections,phoneticMatch}.test.js`). Route tests and client component tests not written. The integration-test fixture proposed in #6 and #10 (popular base names, 3-way intersections) has not been added. +- ❌ P7 Rollout — no PostHog instrumentation, no measurement, no decision gate work. + +**The session was largely about real-world testing.** The user manually tested voice and dropdown picks and reported issues. We worked through #1–#10 in the spot-fix log over the course of the session. + +**Where to start next time, in priority order:** +1. **Validate that the latest fix (#10) holds up under more 3-way intersection testing.** Spot-test a few more: Market & Castro & 17th, Market & Octavia & 9th, Fell & Stanyan, Embarcadero & Bay & Battery, anywhere with a triangular intersection. +2. **Commit and PR.** A lot of in-flight work to a single branch with no commits — risk of losing it. See `docs/cross-streets-implementation-plan.md` for the suggested 3-PR split (Data+Schema / Server+Display / Client+Voice). +3. **Address P6 — at minimum, a route test for `/api/geocode/intersections` and an integration test fixture for known-tricky lookups.** The known patterns of failures we've seen (#6 ranking, #10 dedup) are easy to cover. +4. **P7 instrumentation** before any FIELD rollout — we need the measurement to enforce the decision rules (Deepgram escalation, alias map growth). + +**Things to think about next session:** +- Should we proactively populate `neighborhood` for all *historical* incidents at load time? Currently only fresh autocomplete picks set it; older records show no hint. Could do a one-time client-side computation on incident load (using lat/lng + nearest-intersection lookup) if it's important. +- Build-script automation: currently the rebuild is manual. Worth a `npm run build:intersections` script and maybe a quarterly cron on CI. +- The `LocationAutocomplete` JSX inside `IncidentForm.jsx` is getting busy. Consider extracting the collapsed/expanded location chrome into its own component. + +--- + Each entry: what was observed → what the root cause was → what we changed → optional pattern tag. @@ -160,6 +190,28 @@ Tests added for ordinal expansion (`expandOrdinalWords`) and ordinal-aware `toPr --- +## #10 — "Baker and Geary" silently lost from the data — 3-way intersection dedup bug + +**Observed:** Voice "Baker and Geary" came back as "Couldn't match those streets." Both streets resolve via prefix match individually (Baker → `BAKER ST`, Geary → `GEARY BLVD`), yet the intersection lookup returned nothing. + +**Diagnosis:** This is a real intersection in SF and DataSF carries it (cnn `26811000`), but it's also a **3-way node** — Baker St, Geary Blvd, and Saint Josephs Ave all meet there. DataSF's `jfxm-zeee` emits one row per *pair* at the node, so this cnn appears as `BAKER × GEARY`, `BAKER × SAINT JOSEPHS`, *and* `GEARY × SAINT JOSEPHS`. My build script's dedupe logic kept only **one** of these per cnn — and which one survived was order-dependent. For Baker/Geary/St Josephs, we'd kept `GEARY × SAINT JOSEPHS` and dropped the other two pairs. So Baker × Geary was silently missing from our vendored data. + +Same bug affects any 3+ way intersection in SF (~hundreds of nodes). + +**Fix:** changed the dedupe key in `server/scripts/build-intersection-data.js:buildIntersections` from `cnn` to `(cnn + sorted pair)`. Now each distinct pair at a node survives; only the reverse-ordered duplicates collapse (the cheap form of dedup that's safe). Same cnn and lat/lng across the multiple rows — they all point at the same node. + +Rebuilt `sf-intersections.json`: **9,433 → 11,006 intersections** (+1,573 — that's the previously-lost 3+ way pairs). Neighborhood match rate ticked up slightly too (98.8% → 98.9%) because more of the points were inside polygons. + +Verified end-to-end: `search('Baker and Geary')` → `Baker St & Geary Blvd (Japantown)`. + +**Pattern tag:** `data-shape-assumption` (specifically: I assumed cnn was a unique key for "an intersection" when actually it's a unique key for "a centerline node" — a node may participate in multiple pairs) + +**Notes:** This bug was invisible because all the streets *individually* resolve. It only manifests when a user queries the specific lost pair. Hard to catch without exhaustive testing. The Part 9 bench from `cross-streets.md` did 30 intersections, and Fell & Stanyan was the only 3-way in that set — and by luck the surviving pair *was* `FELL ST × STANYAN ST`. So the bench missed this. + +**Strategy note (parked):** when validating data correctness in the future, sample test queries should include known 3-way intersections (Market & Castro & 17th; Market & Octavia & 9th; Fell & Stanyan; Baker & Geary & St Josephs; etc.). Worth adding to the integration test fixture proposed in #6. + +--- + ## #9 — Reverted #4: single-match voice should auto-apply silently **Observed:** After using the always-show-chip behavior from #4 in practice, the user said: @@ -182,7 +234,7 @@ The previous chip wording change ("Tap to confirm:" vs "Did you mean:") stays ## Pattern summary so far -- **`data-shape-assumption`** (3 occurrences): #1, #2, #3. Each was a code path that assumed address-only and didn't know about the locationType discriminator. **If another shows up, lean on a typed read-side wrapper** (e.g. `getIncidentLocation(incident) → { type: 'address'|'intersection', text, lat, lng }`) and migrate call sites to it rather than continuing inline branching. +- **`data-shape-assumption`** (4 occurrences): #1, #2, #3, #10. #1–#3 were the same address-vs-intersection discriminator pattern. #10 is a different flavor — assuming `cnn` was a unique key for "an intersection" when it's actually a unique key for "a centerline node" (which may participate in multiple street pairs). The first three pointed at a strategy revision (typed read-side wrapper) that we've still parked — #10 doesn't quite fit that solution because it's a build-time data-modeling assumption, not a downstream-consumer one. - **`ranking-bug`** (1 occurrence): #6. Test coverage gap — popular base names with compound siblings need integration tests. - **`stt-fusion`** (2 occurrences): #7, #8. Both handled with focused fixes in the normalization/recovery pipeline. **If 3+ more such fixes pile up, escalate the STT strategy** to Deepgram Nova-3 per the implementation plan's Part 7 gate. #8 in particular suggests a more general principle: anywhere a canonical name has a non-phonetic transformation from speech (digits ↔ words, abbreviations, leading "THE"), we need explicit normalization — phonetic match alone won't bridge it. - **`ux-feedback`** (3 occurrences): #4, #5, #9. #5 was a genuine miss. #4 + #9 together form one back-and-forth: I built the cautious-by-default UX, the user tried it and preferred the lighter one. Lesson: when defaulting cautious vs. light on a UX call where I'm guessing, prefer light + escalate friction only where ambiguity/failure requires it — and surface the choice early instead of letting it ship and bounce. diff --git a/server/scripts/build-intersection-data.js b/server/scripts/build-intersection-data.js index 598a96e4..7ea434af 100644 --- a/server/scripts/build-intersection-data.js +++ b/server/scripts/build-intersection-data.js @@ -99,30 +99,31 @@ function toDisplay (rawName) { } function buildIntersections (rows) { - // Dedupe by CNN. Each CNN appears twice (A→B, B→A); we keep the alphabetically - // earlier (street1, street2) ordering for determinism. - const byCnn = new Map(); + // Each CNN can appear many times: once per (street A, street B) pair at the + // node, in both orderings. For a 3-way intersection of streets X/Y/Z, we + // get rows for X-Y, X-Z, Y-Z (plus reverses). We dedupe by (cnn + sorted + // pair) so all distinct pairs at a node survive, while reverse-ordered + // duplicates collapse. + const byCnnAndPair = new Map(); for (const r of rows) { if (!r.cnn || !r.street_name_1 || !r.street_name_2 || !r.latitude || !r.longitude) continue; const lat = Number(r.latitude); const lng = Number(r.longitude); if (!Number.isFinite(lat) || !Number.isFinite(lng)) continue; const [s1, s2] = [r.street_name_1, r.street_name_2].sort(); - const entry = { + const key = `${r.cnn}|${s1}|${s2}`; + if (byCnnAndPair.has(key)) continue; + byCnnAndPair.set(key, { cnn: r.cnn, street1: s1, street2: s2, lat, lng, zip: r.zip_code || null, - }; - const existing = byCnn.get(r.cnn); - if (!existing) { - byCnn.set(r.cnn, entry); - } + }); } // Sort by street1 then street2 for stable diffs. - return Array.from(byCnn.values()).sort((a, b) => { + return Array.from(byCnnAndPair.values()).sort((a, b) => { if (a.street1 !== b.street1) return a.street1 < b.street1 ? -1 : 1; return a.street2 < b.street2 ? -1 : 1; });