From 33eb914e0c3a9f25eaad0220bc056bd2b04edc1a Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 25 Jan 2025 23:29:21 -0800 Subject: [PATCH 1/3] Implement sync --- package.json | 3 +- pnpm-lock.yaml | 24 +++-- src/app/dim-api/actions.ts | 5 +- src/app/dim-api/dim-api.ts | 24 ++--- src/app/dim-api/reducer.ts | 173 +++++++++++++++++++++++++------------ src/global.d.ts | 4 + 6 files changed, 162 insertions(+), 71 deletions(-) diff --git a/package.json b/package.json index 616bacea5b..9e5dd6d535 100644 --- a/package.json +++ b/package.json @@ -192,7 +192,8 @@ }, "dependencies": { "@babel/runtime": "^7.26.0", - "@destinyitemmanager/dim-api-types": "^1.33.0", + "@beyond-js/md5": "^0.0.1", + "@destinyitemmanager/dim-api-types": "^1.34.0", "@fortawesome/fontawesome-free": "^5.15.4", "@fortawesome/fontawesome-svg-core": "^1.2.36", "@fortawesome/react-fontawesome": "^0.2.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1faa540aea..df6d16af6f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,9 +14,12 @@ dependencies: '@babel/runtime': specifier: ^7.26.0 version: 7.26.0 + '@beyond-js/md5': + specifier: ^0.0.1 + version: 0.0.1 '@destinyitemmanager/dim-api-types': - specifier: ^1.33.0 - version: 1.33.0 + specifier: ^1.34.0 + version: 1.34.0 '@fortawesome/fontawesome-free': specifier: ^5.15.4 version: 5.15.4 @@ -1867,6 +1870,13 @@ packages: dependencies: regenerator-runtime: 0.14.1 + /@babel/runtime@7.26.7: + resolution: {integrity: sha512-AOPI3D+a8dXnja+iwsUqGRjr1BbZIe771sXdapOtYI531gSqpi92vXivKcq2asu/DFpdl1ceFAKZyRzK2PCVcQ==} + engines: {node: '>=6.9.0'} + dependencies: + regenerator-runtime: 0.14.1 + dev: true + /@babel/template@7.25.9: resolution: {integrity: sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==} engines: {node: '>=6.9.0'} @@ -1926,6 +1936,10 @@ packages: resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} dev: true + /@beyond-js/md5@0.0.1: + resolution: {integrity: sha512-pW3l7xy9xxL9rC2fuojXTuzpWtijdpdwCmopn58YxgGcfAkmps0ZMYW+mQwkFbh5Y6ELXCLPXO/+scMqIE4JrQ==} + dev: false + /@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3): resolution: {integrity: sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==} engines: {node: '>=18'} @@ -1960,8 +1974,8 @@ packages: postcss-selector-parser: 7.0.0 dev: true - /@destinyitemmanager/dim-api-types@1.33.0: - resolution: {integrity: sha512-9UyC+Wmk8Us9FUeuJ7YvVurfa5Xwmf6V99N+3baoZpCuH8iHJxDpa7IFQ1ex96532rZQqsPDyKBr32LJTSq9Sg==} + /@destinyitemmanager/dim-api-types@1.34.0: + resolution: {integrity: sha512-E43g4Xl48ckPKSqIbuSvDTIsEtcyslu9D1+psT/qc7rj9ujXRic/kig4onbZRUiLAE5bUi+KdtmkmclfDATnwA==} dev: false /@discoveryjs/json-ext@0.6.3: @@ -4467,7 +4481,7 @@ packages: engines: {node: '>=18'} dependencies: '@babel/code-frame': 7.26.2 - '@babel/runtime': 7.26.0 + '@babel/runtime': 7.26.7 '@types/aria-query': 5.0.4 aria-query: 5.3.0 chalk: 4.1.2 diff --git a/src/app/dim-api/actions.ts b/src/app/dim-api/actions.ts index 3f4d991027..69c705be0b 100644 --- a/src/app/dim-api/actions.ts +++ b/src/app/dim-api/actions.ts @@ -241,7 +241,10 @@ export function loadDimApiData(forceLoad = false): ThunkResult { if (forceLoad || profileOutOfDateOrMissing) { try { - const profileResponse = await getDimApiProfile(currentAccount); + const syncToken = currentAccount + ? getState().dimApi.profiles[makeProfileKeyFromAccount(currentAccount)].syncToken + : undefined; + const profileResponse = await getDimApiProfile(currentAccount, syncToken); dispatch(profileLoaded({ profileResponse, account: currentAccount })); infoLog(TAG, 'Loaded profile from DIM API', profileResponse); diff --git a/src/app/dim-api/dim-api.ts b/src/app/dim-api/dim-api.ts index 9870f3d7cd..d34664191d 100644 --- a/src/app/dim-api/dim-api.ts +++ b/src/app/dim-api/dim-api.ts @@ -29,19 +29,23 @@ export async function getGlobalSettings() { return response.settings; } -export async function getDimApiProfile(account?: DestinyAccount) { +export async function getDimApiProfile(account?: DestinyAccount, syncToken?: string) { + const params: Record = account + ? { + platformMembershipId: account.membershipId, + destinyVersion: account.destinyVersion.toString(), + components: 'settings,loadouts,tags,hashtags,searches,triumphs', + } + : { + components: 'settings', + }; + if (syncToken) { + params.sync = syncToken; + } return authenticatedApi({ url: '/profile', method: 'GET', - params: account - ? { - platformMembershipId: account.membershipId, - destinyVersion: account.destinyVersion.toString(), - components: 'settings,loadouts,tags,hashtags,searches,triumphs', - } - : { - components: 'settings', - }, + params, }); } diff --git a/src/app/dim-api/reducer.ts b/src/app/dim-api/reducer.ts index cdc2b321bb..21f45560cb 100644 --- a/src/app/dim-api/reducer.ts +++ b/src/app/dim-api/reducer.ts @@ -1,3 +1,4 @@ +import md5 from '@beyond-js/md5'; import { CustomStatWeights, DestinyVersion, @@ -5,6 +6,7 @@ import { ItemAnnotation, ItemHashTag, Loadout, + ProfileResponse, ProfileUpdateResult, Search, SearchType, @@ -85,6 +87,9 @@ export interface DimApiState { }; /** Tracked triumphs */ triumphs: number[]; + + /** This allows us to get just the items that changed from the DIM API instead of the whole deal. */ + syncToken?: string; }; }; @@ -142,6 +147,7 @@ export const initialState: DimApiState = { itemHashTags: {}, profiles: {}, + // TODO: move searches into profiles searches: { 1: [], 2: [], @@ -225,60 +231,7 @@ export const dimApi = ( case getType(actions.profileLoaded): { const { profileResponse, account } = action.payload; - - const profileKey = account ? makeProfileKeyFromAccount(account) : ''; - const existingProfile = account ? state.profiles[profileKey] : undefined; - - // TODO: clean out invalid/simple searches on first load? - const newState: DimApiState = migrateSettings({ - ...state, - profileLoaded: true, - profileLoadedError: undefined, - profileLastLoaded: Date.now(), - settings: { - ...state.settings, - ...(profileResponse.settings as Settings), - }, - itemHashTags: profileResponse.itemHashTags - ? keyBy(profileResponse.itemHashTags, (t) => t.hash) - : state.itemHashTags, - profiles: account - ? { - ...state.profiles, - // Overwrite just this account's profile. If a specific key is missing from the response, don't overwrite it. - [profileKey]: { - profileLastLoaded: Date.now(), - loadouts: profileResponse.loadouts - ? keyBy(profileResponse.loadouts, (l) => l.id) - : (existingProfile?.loadouts ?? {}), - tags: profileResponse.tags - ? keyBy(profileResponse.tags, (t) => t.id) - : (existingProfile?.tags ?? {}), - triumphs: profileResponse.triumphs - ? profileResponse.triumphs.map((t) => parseInt(t.toString(), 10)) - : (existingProfile?.triumphs ?? []), - }, - } - : state.profiles, - searches: - account && profileResponse.searches - ? { - ...state.searches, - [account.destinyVersion]: profileResponse.searches || [], - } - : state.searches, - }); - - // If this is the first load, cleanup searches - if ( - account && - profileResponse.searches?.length && - !state.searches[account.destinyVersion].length - ) { - return produce(newState, (state) => cleanupInvalidSearches(state, account)); - } - - return newState; + return profileLoaded(state, profileResponse, account); } case getType(actions.profileLoadError): { @@ -434,6 +387,118 @@ export const dimApi = ( } }; +function profileLoaded( + state: DimApiState, + profileResponse: ProfileResponse, + account?: DestinyAccount, +) { + const profileKey = account ? makeProfileKeyFromAccount(account) : ''; + + // If the response is a sync, we'll start with the existing items and merge in + // changed items. Otherwise we don't keep anything from the existing profile. + + let itemHashTags = state.itemHashTags; + if (profileResponse.itemHashTags || profileResponse.deletedItemHashTagHashes?.length) { + const existingItemHashTags = profileResponse.sync ? state.itemHashTags : undefined; + itemHashTags = { + ...existingItemHashTags, + ...keyBy(profileResponse.itemHashTags ?? [], (t) => t.hash), + }; + for (const t of profileResponse.deletedItemHashTagHashes ?? []) { + delete itemHashTags[t.toString()]; + } + } + + let searches = state.searches ?? {}; + if (account && (profileResponse.searches || profileResponse.deletedSearchHashes?.length)) { + const existingSearches = profileResponse.sync + ? state.searches[account.destinyVersion] + : undefined; + const newSearches = [...(existingSearches ?? [])]; + for (const search of profileResponse.searches ?? []) { + const foundSearchIndex = newSearches.findIndex( + (s) => s.query === search.query && search.type === s.type, + ); + if (foundSearchIndex >= 0) { + newSearches[foundSearchIndex] = search; + } + } + for (const searchHash of profileResponse.deletedSearchHashes ?? []) { + const foundSearchIndex = newSearches.findIndex((s) => md5(s.query) === searchHash); + if (foundSearchIndex >= 0) { + newSearches.splice(foundSearchIndex, 1); + } + } + searches = { ...searches, [account.destinyVersion]: newSearches }; + } + + let profiles = state.profiles; + if (account) { + const existingProfile = state.profiles[profileKey]; + const existingLoadouts = profileResponse.sync ? existingProfile?.loadouts : undefined; + const newLoadouts = { + ...existingLoadouts, + ...keyBy(profileResponse.loadouts ?? [], (l) => l.id), + }; + for (const l of profileResponse.deletedLoadoutIds ?? []) { + delete newLoadouts[l]; + } + + const existingTags = profileResponse.sync ? existingProfile?.tags : undefined; + const newTags = { ...existingTags, ...keyBy(profileResponse.tags ?? [], (t) => t.id) }; + for (const t of profileResponse.deletedTagsIds ?? []) { + delete newTags[t]; + } + + const existingTriumphs = profileResponse.sync ? existingProfile?.triumphs : undefined; + const newTriumphs = new Set([ + ...(existingTriumphs ?? []), + ...(profileResponse.triumphs ?? []).map((t) => parseInt(t.toString(), 10)), + ]); + for (const t of profileResponse.deletedTriumphs ?? []) { + newTriumphs.delete(t); + } + + profiles = { + ...state.profiles, + // Overwrite just this account's profile. If a specific key is missing from the response, don't overwrite it. + [profileKey]: { + profileLastLoaded: Date.now(), + loadouts: newLoadouts, + tags: newTags, + triumphs: [...newTriumphs], + syncToken: profileResponse.syncToken, + }, + }; + } + + // TODO: clean out invalid/simple searches on first load? + const newState: DimApiState = migrateSettings({ + ...state, + profileLoaded: true, + profileLoadedError: undefined, + profileLastLoaded: Date.now(), + settings: { + ...state.settings, + ...(profileResponse.settings as Settings), + }, + itemHashTags, + profiles, + searches, + }); + + // If this is the first load, cleanup searches + if ( + account && + profileResponse.searches?.length && + !state.searches[account.destinyVersion].length + ) { + return produce(newState, (state) => cleanupInvalidSearches(state, account)); + } + + return newState; +} + /** * Migrates deprecated settings to their new equivalent, and erroneous settings values to their correct value. * This updates the settings state and adds their updates to the update queue diff --git a/src/global.d.ts b/src/global.d.ts index 7a325b501b..8a48d28ef4 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -117,3 +117,7 @@ declare module 'locale/*.json' { const value: string; export default value; } + +declare module '@beyond-js/md5' { + export default function md5(str: string): string; +} From 70dc84f47700d16e67fadedce1abe13bab4cdef1 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 26 Jan 2025 19:09:37 -0800 Subject: [PATCH 2/3] Feature flag it --- config/feature-flags.ts | 2 ++ src/app/dim-api/actions.ts | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/feature-flags.ts b/config/feature-flags.ts index 72f9c22518..0100409319 100644 --- a/config/feature-flags.ts +++ b/config/feature-flags.ts @@ -53,6 +53,8 @@ export function makeFeatureFlags(env: { runLoInBackground: true, // Whether to allow setting in-game loadout identifiers on DIM loadouts. editInGameLoadoutIdentifiers: false, + // Whether to sync DIM API data instead of loading everything + dimApiSync: !env.release, }; } diff --git a/src/app/dim-api/actions.ts b/src/app/dim-api/actions.ts index 69c705be0b..2461a2845e 100644 --- a/src/app/dim-api/actions.ts +++ b/src/app/dim-api/actions.ts @@ -241,9 +241,10 @@ export function loadDimApiData(forceLoad = false): ThunkResult { if (forceLoad || profileOutOfDateOrMissing) { try { - const syncToken = currentAccount - ? getState().dimApi.profiles[makeProfileKeyFromAccount(currentAccount)].syncToken - : undefined; + const syncToken = + currentAccount && $featureFlags.dimApiSync + ? getState().dimApi.profiles[makeProfileKeyFromAccount(currentAccount)].syncToken + : undefined; const profileResponse = await getDimApiProfile(currentAccount, syncToken); dispatch(profileLoaded({ profileResponse, account: currentAccount })); infoLog(TAG, 'Loaded profile from DIM API', profileResponse); From 7b71d7469cdf4e6f91577610c6134e1da4b4945d Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sun, 26 Jan 2025 20:08:22 -0800 Subject: [PATCH 3/3] Add a settings button to force a full sync --- config/i18n.json | 1 + src/app/dim-api/actions.ts | 18 ++++++++++++++---- src/app/dim-api/import.ts | 2 +- src/app/storage/DimApiSettings.tsx | 17 +++++++++++++---- src/locale/en.json | 1 + 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/config/i18n.json b/config/i18n.json index acce19d412..4732561ef2 100644 --- a/config/i18n.json +++ b/config/i18n.json @@ -1352,6 +1352,7 @@ "MenuTitle": "Sync & Backups", "ProfileErrorTitle": "DIM Sync Download Error", "ProfileErrorBody": "We had a problem communicating with DIM Sync. Your latest settings, tags, loadouts, and searches may not be shown. Your data is still on our servers, and any updates you make locally will be saved when we can reconnect. We'll keep retrying while DIM is open.", + "RefreshDimSync": "Reload remote data from DIM Sync", "UpdateInvalid": "Failed to save data to DIM Sync", "UpdateInvalidBody": "Data sent to DIM Sync was invalid and will not be saved.", "UpdateInvalidBodyLoadout": "The loadout \"{{name}}\" is invalid and will not be saved. If you imported it from another site, please let them know that they are exporting invalid loadouts.", diff --git a/src/app/dim-api/actions.ts b/src/app/dim-api/actions.ts index 2461a2845e..cfbe8296c9 100644 --- a/src/app/dim-api/actions.ts +++ b/src/app/dim-api/actions.ts @@ -171,8 +171,18 @@ let waitingForApiPermission = false; * for whether the user has opted in to Sync, and if they haven't, we prompt. * Usually they already made their choice at login, though. */ -export function loadDimApiData(forceLoad = false): ThunkResult { +export function loadDimApiData( + options: { + /** + * forceLoad will load from the server even if the minimum refresh + * interval has not passed. Keep in mind the server caches full-profile data for + * up to 60 seconds. This will also skip using a sync token to load incremental changes. + */ + forceLoad?: boolean; + } = {}, +): ThunkResult { return async (dispatch, getState) => { + const { forceLoad = false } = options; installApiPermissionObserver(dispatch); // Load from indexedDB if needed @@ -242,8 +252,8 @@ export function loadDimApiData(forceLoad = false): ThunkResult { if (forceLoad || profileOutOfDateOrMissing) { try { const syncToken = - currentAccount && $featureFlags.dimApiSync - ? getState().dimApi.profiles[makeProfileKeyFromAccount(currentAccount)].syncToken + currentAccount && $featureFlags.dimApiSync && !forceLoad + ? getState().dimApi.profiles?.[makeProfileKeyFromAccount(currentAccount)]?.syncToken : undefined; const profileResponse = await getDimApiProfile(currentAccount, syncToken); dispatch(profileLoaded({ profileResponse, account: currentAccount })); @@ -277,7 +287,7 @@ export function loadDimApiData(forceLoad = false): ThunkResult { infoLog(TAG, 'Waiting', waitTime, 'ms before re-attempting profile fetch'); // Wait, then retry. We don't await this here so we don't stop the finally block from running - delay(waitTime).then(() => dispatch(loadDimApiData(forceLoad))); + delay(waitTime).then(() => dispatch(loadDimApiData(options))); } finally { // Release the app to load with whatever language was saved or the // default. Better to have the wrong language (that fixes itself on diff --git a/src/app/dim-api/import.ts b/src/app/dim-api/import.ts index 40fe1612c7..0e2dfd4194 100644 --- a/src/app/dim-api/import.ts +++ b/src/app/dim-api/import.ts @@ -49,7 +49,7 @@ export function importDataBackup(data: ExportResponse, silent = false): ThunkRes // dim-api can cache the data for up to 60 seconds. Reload from the // server after that so we don't use our faked import data too long. We // won't wait for this. - delay(60_000).then(() => dispatch(loadDimApiData(true))); + delay(60_000).then(() => dispatch(loadDimApiData({ forceLoad: true }))); infoLog(TAG, 'Successfully imported data into DIM API', result); showImportSuccessNotification(result, true); return; diff --git a/src/app/storage/DimApiSettings.tsx b/src/app/storage/DimApiSettings.tsx index d74500f348..82154d3353 100644 --- a/src/app/storage/DimApiSettings.tsx +++ b/src/app/storage/DimApiSettings.tsx @@ -13,7 +13,7 @@ import Checkbox from 'app/settings/Checkbox'; import { fineprintClass, horizontalClass, settingClass } from 'app/settings/SettingsPage'; import { Settings } from 'app/settings/initial-settings'; import ErrorPanel from 'app/shell/ErrorPanel'; -import { AppIcon, deleteIcon } from 'app/shell/icons'; +import { AppIcon, deleteIcon, refreshIcon } from 'app/shell/icons'; import { useThunkDispatch } from 'app/store/thunk-dispatch'; import { errorMessage } from 'app/utils/errors'; import React, { useState } from 'react'; @@ -84,6 +84,10 @@ export default function DimApiSettings() { } }; + const refreshDimSync = async () => { + await dispatch(loadDimApiData({ forceLoad: true })); + }; + return (
{confirmDialog} @@ -102,9 +106,14 @@ export default function DimApiSettings() { />
{t('Storage.DimApiFinePrint')}
{apiPermissionGranted && ( - + <> + + + )} {profileLoadedError && ( diff --git a/src/locale/en.json b/src/locale/en.json index bbd2b782b9..df4970d8ad 100644 --- a/src/locale/en.json +++ b/src/locale/en.json @@ -1336,6 +1336,7 @@ "MenuTitle": "Sync & Backups", "ProfileErrorBody": "We had a problem communicating with DIM Sync. Your latest settings, tags, loadouts, and searches may not be shown. Your data is still on our servers, and any updates you make locally will be saved when we can reconnect. We'll keep retrying while DIM is open.", "ProfileErrorTitle": "DIM Sync Download Error", + "RefreshDimSync": "Reload remote data from DIM Sync", "UpdateErrorBody": "We had a problem saving your data to DIM Sync. We'll keep retrying while DIM is open.", "UpdateErrorTitle": "DIM Sync Save Error", "UpdateInvalid": "Failed to save data to DIM Sync",