Merge pull request #10909 from DestinyItemManager/stately-sync

Use the sync token from the DIM API to load incremental changes
DestinyItemManager · Jan 27, 2025 · 0b753ee · 0b753ee
2 parents 2a1c182 + 7b71d74
commit 0b753ee
Show file tree

Hide file tree

Showing 11 changed files with 193 additions and 78 deletions.
diff --git a/config/feature-flags.ts b/config/feature-flags.ts
@@ -53,6 +53,8 @@ export function makeFeatureFlags(env: {
     runLoInBackground: true,
     // Whether to allow setting in-game loadout identifiers on DIM loadouts.
     editInGameLoadoutIdentifiers: false,
+    // Whether to sync DIM API data instead of loading everything
+    dimApiSync: !env.release,
   };
 }
 

diff --git a/config/i18n.json b/config/i18n.json
@@ -1352,6 +1352,7 @@
     "MenuTitle": "Sync & Backups",
     "ProfileErrorTitle": "DIM Sync Download Error",
     "ProfileErrorBody": "We had a problem communicating with DIM Sync. Your latest settings, tags, loadouts, and searches may not be shown. Your data is still on our servers, and any updates you make locally will be saved when we can reconnect. We'll keep retrying while DIM is open.",
+    "RefreshDimSync": "Reload remote data from DIM Sync",
     "UpdateInvalid": "Failed to save data to DIM Sync",
     "UpdateInvalidBody": "Data sent to DIM Sync was invalid and will not be saved.",
     "UpdateInvalidBodyLoadout": "The loadout \"{{name}}\" is invalid and will not be saved. If you imported it from another site, please let them know that they are exporting invalid loadouts.",

diff --git a/package.json b/package.json
@@ -192,7 +192,8 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.26.0",
-    "@destinyitemmanager/dim-api-types": "^1.33.0",
+    "@beyond-js/md5": "^0.0.1",
+    "@destinyitemmanager/dim-api-types": "^1.34.0",
     "@fortawesome/fontawesome-free": "^5.15.4",
     "@fortawesome/fontawesome-svg-core": "^1.2.36",
     "@fortawesome/react-fontawesome": "^0.2.2",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/app/dim-api/actions.ts b/src/app/dim-api/actions.ts
@@ -171,8 +171,18 @@ let waitingForApiPermission = false;
  * for whether the user has opted in to Sync, and if they haven't, we prompt.
  * Usually they already made their choice at login, though.
  */
-export function loadDimApiData(forceLoad = false): ThunkResult {
+export function loadDimApiData(
+  options: {
+    /**
+     * forceLoad will load from the server even if the minimum refresh
+     * interval has not passed. Keep in mind the server caches full-profile data for
+     * up to 60 seconds. This will also skip using a sync token to load incremental changes.
+     */
+    forceLoad?: boolean;
+  } = {},
+): ThunkResult {
   return async (dispatch, getState) => {
+    const { forceLoad = false } = options;
     installApiPermissionObserver(dispatch);
 
     // Load from indexedDB if needed
@@ -241,7 +251,11 @@ export function loadDimApiData(forceLoad = false): ThunkResult {
 
     if (forceLoad || profileOutOfDateOrMissing) {
       try {
-        const profileResponse = await getDimApiProfile(currentAccount);
+        const syncToken =
+          currentAccount && $featureFlags.dimApiSync && !forceLoad
+            ? getState().dimApi.profiles?.[makeProfileKeyFromAccount(currentAccount)]?.syncToken
+            : undefined;
+        const profileResponse = await getDimApiProfile(currentAccount, syncToken);
         dispatch(profileLoaded({ profileResponse, account: currentAccount }));
         infoLog(TAG, 'Loaded profile from DIM API', profileResponse);
 
@@ -273,7 +287,7 @@ export function loadDimApiData(forceLoad = false): ThunkResult {
         infoLog(TAG, 'Waiting', waitTime, 'ms before re-attempting profile fetch');
 
         // Wait, then retry. We don't await this here so we don't stop the finally block from running
-        delay(waitTime).then(() => dispatch(loadDimApiData(forceLoad)));
+        delay(waitTime).then(() => dispatch(loadDimApiData(options)));
       } finally {
         // Release the app to load with whatever language was saved or the
         // default. Better to have the wrong language (that fixes itself on

diff --git a/src/app/dim-api/dim-api.ts b/src/app/dim-api/dim-api.ts
@@ -29,19 +29,23 @@ export async function getGlobalSettings() {
   return response.settings;
 }
 
-export async function getDimApiProfile(account?: DestinyAccount) {
+export async function getDimApiProfile(account?: DestinyAccount, syncToken?: string) {
+  const params: Record<string, string> = account
+    ? {
+        platformMembershipId: account.membershipId,
+        destinyVersion: account.destinyVersion.toString(),
+        components: 'settings,loadouts,tags,hashtags,searches,triumphs',
+      }
+    : {
+        components: 'settings',
+      };
+  if (syncToken) {
+    params.sync = syncToken;
+  }
   return authenticatedApi<ProfileResponse>({
     url: '/profile',
     method: 'GET',
-    params: account
-      ? {
-          platformMembershipId: account.membershipId,
-          destinyVersion: account.destinyVersion.toString(),
-          components: 'settings,loadouts,tags,hashtags,searches,triumphs',
-        }
-      : {
-          components: 'settings',
-        },
+    params,
   });
 }
 

diff --git a/src/app/dim-api/import.ts b/src/app/dim-api/import.ts
@@ -49,7 +49,7 @@ export function importDataBackup(data: ExportResponse, silent = false): ThunkRes
         // dim-api can cache the data for up to 60 seconds. Reload from the
         // server after that so we don't use our faked import data too long. We
         // won't wait for this.
-        delay(60_000).then(() => dispatch(loadDimApiData(true)));
+        delay(60_000).then(() => dispatch(loadDimApiData({ forceLoad: true })));
         infoLog(TAG, 'Successfully imported data into DIM API', result);
         showImportSuccessNotification(result, true);
         return;

diff --git a/src/app/dim-api/reducer.ts b/src/app/dim-api/reducer.ts
@@ -1,10 +1,12 @@
+import md5 from '@beyond-js/md5';
 import {
   CustomStatWeights,
   DestinyVersion,
   GlobalSettings,
   ItemAnnotation,
   ItemHashTag,
   Loadout,
+  ProfileResponse,
   ProfileUpdateResult,
   Search,
   SearchType,
@@ -85,6 +87,9 @@ export interface DimApiState {
       };
       /** Tracked triumphs */
       triumphs: number[];
+
+      /** This allows us to get just the items that changed from the DIM API instead of the whole deal. */
+      syncToken?: string;
     };
   };
 
@@ -142,6 +147,7 @@ export const initialState: DimApiState = {
 
   itemHashTags: {},
   profiles: {},
+  // TODO: move searches into profiles
   searches: {
     1: [],
     2: [],
@@ -225,60 +231,7 @@ export const dimApi = (
 
     case getType(actions.profileLoaded): {
       const { profileResponse, account } = action.payload;
-
-      const profileKey = account ? makeProfileKeyFromAccount(account) : '';
-      const existingProfile = account ? state.profiles[profileKey] : undefined;
-
-      // TODO: clean out invalid/simple searches on first load?
-      const newState: DimApiState = migrateSettings({
-        ...state,
-        profileLoaded: true,
-        profileLoadedError: undefined,
-        profileLastLoaded: Date.now(),
-        settings: {
-          ...state.settings,
-          ...(profileResponse.settings as Settings),
-        },
-        itemHashTags: profileResponse.itemHashTags
-          ? keyBy(profileResponse.itemHashTags, (t) => t.hash)
-          : state.itemHashTags,
-        profiles: account
-          ? {
-              ...state.profiles,
-              // Overwrite just this account's profile. If a specific key is missing from the response, don't overwrite it.
-              [profileKey]: {
-                profileLastLoaded: Date.now(),
-                loadouts: profileResponse.loadouts
-                  ? keyBy(profileResponse.loadouts, (l) => l.id)
-                  : (existingProfile?.loadouts ?? {}),
-                tags: profileResponse.tags
-                  ? keyBy(profileResponse.tags, (t) => t.id)
-                  : (existingProfile?.tags ?? {}),
-                triumphs: profileResponse.triumphs
-                  ? profileResponse.triumphs.map((t) => parseInt(t.toString(), 10))
-                  : (existingProfile?.triumphs ?? []),
-              },
-            }
-          : state.profiles,
-        searches:
-          account && profileResponse.searches
-            ? {
-                ...state.searches,
-                [account.destinyVersion]: profileResponse.searches || [],
-              }
-            : state.searches,
-      });
-
-      // If this is the first load, cleanup searches
-      if (
-        account &&
-        profileResponse.searches?.length &&
-        !state.searches[account.destinyVersion].length
-      ) {
-        return produce(newState, (state) => cleanupInvalidSearches(state, account));
-      }
-
-      return newState;
+      return profileLoaded(state, profileResponse, account);
     }
 
     case getType(actions.profileLoadError): {
@@ -434,6 +387,118 @@ export const dimApi = (
   }
 };
 
+function profileLoaded(
+  state: DimApiState,
+  profileResponse: ProfileResponse,
+  account?: DestinyAccount,
+) {
+  const profileKey = account ? makeProfileKeyFromAccount(account) : '';
+
+  // If the response is a sync, we'll start with the existing items and merge in
+  // changed items. Otherwise we don't keep anything from the existing profile.
+
+  let itemHashTags = state.itemHashTags;
+  if (profileResponse.itemHashTags || profileResponse.deletedItemHashTagHashes?.length) {
+    const existingItemHashTags = profileResponse.sync ? state.itemHashTags : undefined;
+    itemHashTags = {
+      ...existingItemHashTags,
+      ...keyBy(profileResponse.itemHashTags ?? [], (t) => t.hash),
+    };
+    for (const t of profileResponse.deletedItemHashTagHashes ?? []) {
+      delete itemHashTags[t.toString()];
+    }
+  }
+
+  let searches = state.searches ?? {};
+  if (account && (profileResponse.searches || profileResponse.deletedSearchHashes?.length)) {
+    const existingSearches = profileResponse.sync
+      ? state.searches[account.destinyVersion]
+      : undefined;
+    const newSearches = [...(existingSearches ?? [])];
+    for (const search of profileResponse.searches ?? []) {
+      const foundSearchIndex = newSearches.findIndex(
+        (s) => s.query === search.query && search.type === s.type,
+      );
+      if (foundSearchIndex >= 0) {
+        newSearches[foundSearchIndex] = search;
+      }
+    }
+    for (const searchHash of profileResponse.deletedSearchHashes ?? []) {
+      const foundSearchIndex = newSearches.findIndex((s) => md5(s.query) === searchHash);
+      if (foundSearchIndex >= 0) {
+        newSearches.splice(foundSearchIndex, 1);
+      }
+    }
+    searches = { ...searches, [account.destinyVersion]: newSearches };
+  }
+
+  let profiles = state.profiles;
+  if (account) {
+    const existingProfile = state.profiles[profileKey];
+    const existingLoadouts = profileResponse.sync ? existingProfile?.loadouts : undefined;
+    const newLoadouts = {
+      ...existingLoadouts,
+      ...keyBy(profileResponse.loadouts ?? [], (l) => l.id),
+    };
+    for (const l of profileResponse.deletedLoadoutIds ?? []) {
+      delete newLoadouts[l];
+    }
+
+    const existingTags = profileResponse.sync ? existingProfile?.tags : undefined;
+    const newTags = { ...existingTags, ...keyBy(profileResponse.tags ?? [], (t) => t.id) };
+    for (const t of profileResponse.deletedTagsIds ?? []) {
+      delete newTags[t];
+    }
+
+    const existingTriumphs = profileResponse.sync ? existingProfile?.triumphs : undefined;
+    const newTriumphs = new Set([
+      ...(existingTriumphs ?? []),
+      ...(profileResponse.triumphs ?? []).map((t) => parseInt(t.toString(), 10)),
+    ]);
+    for (const t of profileResponse.deletedTriumphs ?? []) {
+      newTriumphs.delete(t);
+    }
+
+    profiles = {
+      ...state.profiles,
+      // Overwrite just this account's profile. If a specific key is missing from the response, don't overwrite it.
+      [profileKey]: {
+        profileLastLoaded: Date.now(),
+        loadouts: newLoadouts,
+        tags: newTags,
+        triumphs: [...newTriumphs],
+        syncToken: profileResponse.syncToken,
+      },
+    };
+  }
+
+  // TODO: clean out invalid/simple searches on first load?
+  const newState: DimApiState = migrateSettings({
+    ...state,
+    profileLoaded: true,
+    profileLoadedError: undefined,
+    profileLastLoaded: Date.now(),
+    settings: {
+      ...state.settings,
+      ...(profileResponse.settings as Settings),
+    },
+    itemHashTags,
+    profiles,
+    searches,
+  });
+
+  // If this is the first load, cleanup searches
+  if (
+    account &&
+    profileResponse.searches?.length &&
+    !state.searches[account.destinyVersion].length
+  ) {
+    return produce(newState, (state) => cleanupInvalidSearches(state, account));
+  }
+
+  return newState;
+}
+
 /**
  * Migrates deprecated settings to their new equivalent, and erroneous settings values to their correct value.
  * This updates the settings state and adds their updates to the update queue