Fix: sort strings in UTF-8 encoded byte order with lazy encoding

.changeset/large-pants-hide.md

@@ -0,0 +1,6 @@
+---
+'@firebase/firestore': patch
+'firebase': patch
+---
+
+Use lazy encoding in UTF-8 encoded byte comparison for strings.

packages/firestore/src/local/indexeddb_remote_document_cache.ts

@@ -655,5 +655,9 @@ export function dbKeyComparator(l: DocumentKey, r: DocumentKey): number {
     return cmp;
   }
 
+  // TODO(b/329441702): Document IDs should be sorted by UTF-8 encoded byte
+  // order, but IndexedDB sorts strings lexicographically. Document ID
+  // comparison here still relies on primitive comparison to avoid mismatches
+  // observed in snapshot listeners with Unicode characters in documentIds
   return primitiveComparator(left[left.length - 1], right[right.length - 1]);
 }

packages/firestore/src/model/path.ts

@@ -19,6 +19,7 @@ import { Integer } from '@firebase/webchannel-wrapper/bloom-blob';
 
 import { debugAssert, fail } from '../util/assert';
 import { Code, FirestoreError } from '../util/error';
+import { compareUtf8Strings, primitiveComparator } from '../util/misc';
 
 export const DOCUMENT_KEY_NAME = '__name__';
 
@@ -181,7 +182,7 @@ abstract class BasePath<B extends BasePath<B>> {
         return comparison;
       }
     }
-    return Math.sign(p1.length - p2.length);
+    return primitiveComparator(p1.length, p2.length);
   }
 
   private static compareSegments(lhs: string, rhs: string): number {
@@ -201,13 +202,7 @@ abstract class BasePath<B extends BasePath<B>> {
       );
     } else {
       // both non-numeric
-      if (lhs < rhs) {
-        return -1;
-      }
-      if (lhs > rhs) {
-        return 1;
-      }
-      return 0;
+      return compareUtf8Strings(lhs, rhs);
     }
   }
 

packages/firestore/src/model/values.ts

@@ -25,7 +25,11 @@ import {
   Value
 } from '../protos/firestore_proto_api';
 import { fail } from '../util/assert';
-import { arrayEquals, primitiveComparator } from '../util/misc';
+import {
+  arrayEquals,
+  compareUtf8Strings,
+  primitiveComparator
+} from '../util/misc';
 import { forEach, objectSize } from '../util/obj';
 import { isNegativeZero } from '../util/types';
 
@@ -251,7 +255,7 @@ export function valueCompare(left: Value, right: Value): number {
         getLocalWriteTime(right)
       );
     case TypeOrder.StringValue:
-      return primitiveComparator(left.stringValue!, right.stringValue!);
+      return compareUtf8Strings(left.stringValue!, right.stringValue!);
     case TypeOrder.BlobValue:
       return compareBlobs(left.bytesValue!, right.bytesValue!);
     case TypeOrder.RefValue:
@@ -400,7 +404,7 @@ function compareMaps(left: MapValue, right: MapValue): number {
   rightKeys.sort();
 
   for (let i = 0; i < leftKeys.length && i < rightKeys.length; ++i) {
-    const keyCompare = primitiveComparator(leftKeys[i], rightKeys[i]);
+    const keyCompare = compareUtf8Strings(leftKeys[i], rightKeys[i]);
     if (keyCompare !== 0) {
       return keyCompare;
     }

packages/firestore/src/util/misc.ts

@@ -16,6 +16,7 @@
  */
 
 import { randomBytes } from '../platform/random_bytes';
+import { newTextEncoder } from '../platform/text_serializer';
 
 import { debugAssert } from './assert';
 
@@ -74,6 +75,67 @@ export interface Equatable<T> {
   isEqual(other: T): boolean;
 }
 
+/** Compare strings in UTF-8 encoded byte order */
+export function compareUtf8Strings(left: string, right: string): number {
+  let i = 0;
+  while (i < left.length && i < right.length) {
+    const leftCodePoint = left.codePointAt(i)!;
+    const rightCodePoint = right.codePointAt(i)!;
+
+    if (leftCodePoint !== rightCodePoint) {
+      if (leftCodePoint < 128 && rightCodePoint < 128) {
+        // ASCII comparison
+        return primitiveComparator(leftCodePoint, rightCodePoint);
+      } else {
+        // Lazy instantiate TextEncoder
+        const encoder = newTextEncoder();
+
+        // UTF-8 encode the character at index i for byte comparison.
+        const leftBytes = encoder.encode(getUtf8SafeSubstring(left, i));
+        const rightBytes = encoder.encode(getUtf8SafeSubstring(right, i));
+
+        const comp = compareByteArrays(leftBytes, rightBytes);
+        if (comp !== 0) {
+          return comp;
+        } else {
+          // EXTREMELY RARE CASE: Code points differ, but their UTF-8 byte
+          // representations are identical. This can happen with malformed input
+          // (invalid surrogate pairs). The backend also actively prevents invalid
+          // surrogates as INVALID_ARGUMENT errors, so we almost never receive
+          // invalid strings from backend.
+          // Fallback to code point comparison for graceful handling.
+          return primitiveComparator(leftCodePoint, rightCodePoint);
+        }
+      }
+    }
+    // Increment by 2 for surrogate pairs, 1 otherwise
+    i += leftCodePoint > 0xffff ? 2 : 1;
+  }
+
+  // Compare lengths if all characters are equal
+  return primitiveComparator(left.length, right.length);
+}
+
+function getUtf8SafeSubstring(str: string, index: number): string {
+  const firstCodePoint = str.codePointAt(index)!;
+  if (firstCodePoint > 0xffff) {
+    // It's a surrogate pair, return the whole pair
+    return str.substring(index, index + 2);
+  } else {
+    // It's a single code point, return it
+    return str.substring(index, index + 1);
+  }
+}
+
+function compareByteArrays(left: Uint8Array, right: Uint8Array): number {
+  for (let i = 0; i < left.length && i < right.length; ++i) {
+    if (left[i] !== right[i]) {
+      return primitiveComparator(left[i], right[i]);
+    }
+  }
+  return primitiveComparator(left.length, right.length);
+}
+
 export interface Iterable<V> {
   forEach: (cb: (v: V) => void) => void;
 }

packages/firestore/test/integration/api/database.test.ts

@@ -2424,4 +2424,243 @@ apiDescribe('Database', persistence => {
       });
     });
   });
+
+  describe('Sort unicode strings', () => {
+    const expectedDocs = [
+      'b',
+      'a',
+      'h',
+      'i',
+      'c',
+      'f',
+      'e',
+      'd',
+      'g',
+      'k',
+      'j'
+    ];
+    it('snapshot listener sorts unicode strings the same as server', async () => {
+      const testDocs = {
+        'a': { value: 'Łukasiewicz' },
+        'b': { value: 'Sierpiński' },
+        'c': { value: '岩澤' },
+        'd': { value: '🄟' },
+        'e': { value: 'Ｐ' },
+        'f': { value: '︒' },
+        'g': { value: '🐵' },
+        'h': { value: '你好' },
+        'i': { value: '你顥' },
+        'j': { value: '😁' },
+        'k': { value: '😀' }
+      };
+
+      return withTestCollection(persistence, testDocs, async collectionRef => {
+        const orderedQuery = query(collectionRef, orderBy('value'));
+
+        const getSnapshot = await getDocsFromServer(orderedQuery);
+        expect(toIds(getSnapshot)).to.deep.equal(expectedDocs);
+
+        const storeEvent = new EventsAccumulator<QuerySnapshot>();
+        const unsubscribe = onSnapshot(orderedQuery, storeEvent.storeEvent);
+        const watchSnapshot = await storeEvent.awaitEvent();
+        expect(toIds(watchSnapshot)).to.deep.equal(toIds(getSnapshot));
+
+        unsubscribe();
+
+        await checkOnlineAndOfflineResultsMatch(orderedQuery, ...expectedDocs);
+      });
+    });
+
+    it('snapshot listener sorts unicode strings in array the same  as server', async () => {
+      const testDocs = {
+        'a': { value: ['Łukasiewicz'] },
+        'b': { value: ['Sierpiński'] },
+        'c': { value: ['岩澤'] },
+        'd': { value: ['🄟'] },
+        'e': { value: ['Ｐ'] },
+        'f': { value: ['︒'] },
+        'g': { value: ['🐵'] },
+        'h': { value: ['你好'] },
+        'i': { value: ['你顥'] },
+        'j': { value: ['😁'] },
+        'k': { value: ['😀'] }
+      };
+
+      return withTestCollection(persistence, testDocs, async collectionRef => {
+        const orderedQuery = query(collectionRef, orderBy('value'));
+
+        const getSnapshot = await getDocsFromServer(orderedQuery);
+        expect(toIds(getSnapshot)).to.deep.equal(expectedDocs);
+
+        const storeEvent = new EventsAccumulator<QuerySnapshot>();
+        const unsubscribe = onSnapshot(orderedQuery, storeEvent.storeEvent);
+        const watchSnapshot = await storeEvent.awaitEvent();
+        expect(toIds(watchSnapshot)).to.deep.equal(toIds(getSnapshot));
+
+        unsubscribe();
+
+        await checkOnlineAndOfflineResultsMatch(orderedQuery, ...expectedDocs);
+      });
+    });
+
+    it('snapshot listener sorts unicode strings in map the same as server', async () => {
+      const testDocs = {
+        'a': { value: { foo: 'Łukasiewicz' } },
+        'b': { value: { foo: 'Sierpiński' } },
+        'c': { value: { foo: '岩澤' } },
+        'd': { value: { foo: '🄟' } },
+        'e': { value: { foo: 'Ｐ' } },
+        'f': { value: { foo: '︒' } },
+        'g': { value: { foo: '🐵' } },
+        'h': { value: { foo: '你好' } },
+        'i': { value: { foo: '你顥' } },
+        'j': { value: { foo: '😁' } },
+        'k': { value: { foo: '😀' } }
+      };
+
+      return withTestCollection(persistence, testDocs, async collectionRef => {
+        const orderedQuery = query(collectionRef, orderBy('value'));
+
+        const getSnapshot = await getDocsFromServer(orderedQuery);
+        expect(toIds(getSnapshot)).to.deep.equal(expectedDocs);
+
+        const storeEvent = new EventsAccumulator<QuerySnapshot>();
+        const unsubscribe = onSnapshot(orderedQuery, storeEvent.storeEvent);
+        const watchSnapshot = await storeEvent.awaitEvent();
+        expect(toIds(watchSnapshot)).to.deep.equal(toIds(getSnapshot));
+
+        unsubscribe();
+
+        await checkOnlineAndOfflineResultsMatch(orderedQuery, ...expectedDocs);
+      });
+    });
+
+    it('snapshot listener sorts unicode strings in map key the same as server', async () => {
+      const testDocs = {
+        'a': { value: { 'Łukasiewicz': true } },
+        'b': { value: { 'Sierpiński': true } },
+        'c': { value: { '岩澤': true } },
+        'd': { value: { '🄟': true } },
+        'e': { value: { 'Ｐ': true } },
+        'f': { value: { '︒': true } },
+        'g': { value: { '🐵': true } },
+        'h': { value: { '你好': true } },
+        'i': { value: { '你顥': true } },
+        'j': { value: { '😁': true } },
+        'k': { value: { '😀': true } }
+      };
+
+      return withTestCollection(persistence, testDocs, async collectionRef => {
+        const orderedQuery = query(collectionRef, orderBy('value'));
+
+        const getSnapshot = await getDocsFromServer(orderedQuery);
+        expect(toIds(getSnapshot)).to.deep.equal(expectedDocs);
+
+        const storeEvent = new EventsAccumulator<QuerySnapshot>();
+        const unsubscribe = onSnapshot(orderedQuery, storeEvent.storeEvent);
+        const watchSnapshot = await storeEvent.awaitEvent();
+        expect(toIds(watchSnapshot)).to.deep.equal(toIds(getSnapshot));
+
+        unsubscribe();
+
+        await checkOnlineAndOfflineResultsMatch(orderedQuery, ...expectedDocs);
+      });
+    });
+
+    it('snapshot listener sorts unicode strings in document key the same as server', async () => {
+      const testDocs = {
+        'Łukasiewicz': { value: true },
+        'Sierpiński': { value: true },
+        '岩澤': { value: true },
+        '🄟': { value: true },
+        'Ｐ': { value: true },
+        '︒': { value: true },
+        '🐵': { value: true },
+        '你好': { value: true },
+        '你顥': { value: true },
+        '😁': { value: true },
+        '😀': { value: true }
+      };
+
+      return withTestCollection(persistence, testDocs, async collectionRef => {
+        const orderedQuery = query(collectionRef, orderBy(documentId()));
+
+        const getSnapshot = await getDocsFromServer(orderedQuery);
+        const expectedDocs = [
+          'Sierpiński',
+          'Łukasiewicz',
+          '你好',
+          '你顥',
+          '岩澤',
+          '︒',
+          'Ｐ',
+          '🄟',
+          '🐵',
+          '😀',
+          '😁'
+        ];
+        expect(toIds(getSnapshot)).to.deep.equal(expectedDocs);
+
+        const storeEvent = new EventsAccumulator<QuerySnapshot>();
+        const unsubscribe = onSnapshot(orderedQuery, storeEvent.storeEvent);
+        const watchSnapshot = await storeEvent.awaitEvent();
+        expect(toIds(watchSnapshot)).to.deep.equal(toIds(getSnapshot));
+
+        unsubscribe();
+
+        await checkOnlineAndOfflineResultsMatch(orderedQuery, ...expectedDocs);
+      });
+    });
+
+    // eslint-disable-next-line no-restricted-properties
+    (persistence.storage === 'indexeddb' ? it.skip : it)(
+      'snapshot listener sorts unicode strings in document key the same as server with persistence',
+      async () => {
+        const testDocs = {
+          'Łukasiewicz': { value: true },
+          'Sierpiński': { value: true },
+          '岩澤': { value: true },
+          '🄟': { value: true },
+          'Ｐ': { value: true },
+          '︒': { value: true },
+          '🐵': { value: true },
+          '你好': { value: true },
+          '你顥': { value: true },
+          '😁': { value: true },
+          '😀': { value: true }
+        };
+
+        return withTestCollection(
+          persistence,
+          testDocs,
+          async collectionRef => {
+            const orderedQuery = query(collectionRef, orderBy('value'));
+
+            const getSnapshot = await getDocsFromServer(orderedQuery);
+            expect(toIds(getSnapshot)).to.deep.equal([
+              'Sierpiński',
+              'Łukasiewicz',
+              '你好',
+              '你顥',
+              '岩澤',
+              '︒',
+              'Ｐ',
+              '🄟',
+              '🐵',
+              '😀',
+              '😁'
+            ]);
+
+            const storeEvent = new EventsAccumulator<QuerySnapshot>();
+            const unsubscribe = onSnapshot(orderedQuery, storeEvent.storeEvent);
+            const watchSnapshot = await storeEvent.awaitEvent();
+            // TODO: IndexedDB sorts string lexicographically, and misses the document with ID '🄟','🐵'
+            expect(toIds(watchSnapshot)).to.deep.equal(toIds(getSnapshot));
+
+            unsubscribe();
+          }
+        );
+      }
+    );
+  });
 });