Skip to content

Commit 08ffe9e

Browse files
Apollon77AlCalzone
Apollon77
and
AlCalzone
authored
Implement enum caching to speed tup object deletes (#1530)
* * (Apollon77) Add enum caching to adapter class to improve performance * (Apollon77) convert removeIdFromAllEnums to async and add option to hand ver list of enums * * fix * * optimizations * Update packages/common/lib/common/tools.js Co-authored-by: AlCalzone <[email protected]> * Update packages/common/lib/common/tools.js Co-authored-by: AlCalzone <[email protected]> * * fix used reserved word Co-authored-by: AlCalzone <[email protected]>
1 parent 882ced8 commit 08ffe9e

File tree

3 files changed

+95
-93
lines changed

3 files changed

+95
-93
lines changed

‎packages/adapter/lib/adapter/adapter.js‎

Lines changed: 59 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,6 @@ function Adapter(options) {
181181

182182
config = options.config || config;
183183
this.startedInCompactMode = options.compact;
184-
const regUser = /^system\.user\./;
185-
const regGroup = /^system\.group\./;
186184
let firstConnection = true;
187185
let systemSecret = null;
188186

@@ -191,6 +189,7 @@ function Adapter(options) {
191189
this.logList = [];
192190
this.aliases = {};
193191
this.aliasPatterns = [];
192+
this.enums = {};
194193

195194
this.eventLoopLags = [];
196195
this.overwriteLogLevel = false;
@@ -644,7 +643,7 @@ function Adapter(options) {
644643
throw new Error('checkPassword: no callback');
645644
}
646645

647-
if (user && !regUser.test(user)) {
646+
if (user && !user.startsWith('system.user.')) {
648647
// its not yet a `system.user.xy` id, thus we assume it's a username
649648
if (!this.usernames[user]) {
650649
// we did not find the id of the username in our cache -> update cache
@@ -724,7 +723,7 @@ function Adapter(options) {
724723
options = null;
725724
}
726725

727-
if (user && !regUser.test(user)) {
726+
if (user && !user.startsWith('system.user.')) {
728727
// its not yet a `system.user.xy` id, thus we assume it's a username
729728
if (!this.usernames[user]) {
730729
// we did not find the id of the username in our cache -> update cache
@@ -806,7 +805,7 @@ function Adapter(options) {
806805
options = null;
807806
}
808807

809-
if (user && !regUser.test(user)) {
808+
if (user && !user.startsWith('system.user.')) {
810809
// its not yet a `system.user.xy` id, thus we assume it's a username
811810
if (!this.usernames[user]) {
812811
// we did not find the id of the username in our cache -> update cache
@@ -827,7 +826,7 @@ function Adapter(options) {
827826
}
828827
}
829828

830-
if (group && !regGroup.test(group)) {
829+
if (group && !group.startsWith('system.group.')) {
831830
group = 'system.group.' + group;
832831
}
833832
this.getForeignObject(user, options, (err, obj) => {
@@ -955,7 +954,7 @@ function Adapter(options) {
955954
options = null;
956955
}
957956

958-
if (user && !regUser.test(user)) {
957+
if (user && !user.startsWith('system.user.')) {
959958
// its not yet a `system.user.xy` id, thus we assume it's a username
960959
if (!this.usernames[user]) {
961960
// we did not find the id of the username in our cache -> update cache
@@ -1522,7 +1521,7 @@ function Adapter(options) {
15221521
namespace: this.namespaceLog,
15231522
connection: config.objects,
15241523
logger: logger,
1525-
connected: () => {
1524+
connected: async () => {
15261525
this.connected = true;
15271526
if (initializeTimeout) {
15281527
clearTimeout(initializeTimeout);
@@ -1532,6 +1531,10 @@ function Adapter(options) {
15321531
// subscribe to user changes
15331532
adapterObjects.subscribe('system.user.*');
15341533

1534+
// get all enums and register for enum changes
1535+
this.enums = await this.getEnumsAsync();
1536+
adapterObjects.subscribe('enum.*');
1537+
15351538
// Read dateformat if using of formatDate is announced
15361539
if (options.useFormatDate) {
15371540
this.getForeignObject('system.config', (err, data) => {
@@ -1682,11 +1685,19 @@ function Adapter(options) {
16821685
}
16831686

16841687
// Clear cache if got the message about change (Will work for admin and javascript - TODO: maybe always subscribe?)
1685-
if (id.match(/^system\.user\./) || id.match(/^system\.group\./)) {
1688+
if (id.startsWith('system.user.') || id.startsWith('system.group.')) {
16861689
this.users = {};
16871690
this.groups = {};
16881691
this.usernames = {};
16891692
}
1693+
1694+
if (id.startsWith('enum.')) {
1695+
if (!obj) {
1696+
delete this.enums[id];
1697+
} else if (obj.type === 'enum') {
1698+
this.enums[id] = obj;
1699+
}
1700+
}
16901701
},
16911702
changeUser: (id, obj) => { // User level object changes
16921703
if (obj === 'null' || obj === '') {
@@ -3131,26 +3142,23 @@ function Adapter(options) {
31313142
return tools.maybeCallback(cb);
31323143
} else {
31333144
const task = tasks.shift();
3134-
adapterObjects.delObject(task.id, options, err => {
3145+
adapterObjects.delObject(task.id, options, async err => {
31353146
if (err) {
31363147
return tools.maybeCallbackWithError(cb, err);
3137-
} else if (!task.state) {
3138-
// if not a state, we dont have to delete state
3139-
tools.removeIdFromAllEnums(adapterObjects, task.id).then(() => setImmediate(_deleteObjects, tasks, options, cb))
3140-
.catch(e => {
3141-
this.log.warn(`Could not remove ${task.id} from enums: ${e.message}`);
3142-
setImmediate(_deleteObjects, tasks, options, cb);
3143-
});
3144-
} else {
3145-
this.delForeignState(task.id, options, err => {
3146-
err && this.log.warn(`Could not remove state of ${task.id}`);
3147-
tools.removeIdFromAllEnums(adapterObjects, task.id).then(() => setImmediate(_deleteObjects, tasks, options, cb))
3148-
.catch(e => {
3149-
this.log.warn(`Could not remove ${task.id} from enums: ${e.message}`);
3150-
setImmediate(_deleteObjects, tasks, options, cb);
3151-
});
3152-
});
31533148
}
3149+
if (task.state) {
3150+
try {
3151+
await this.delForeignStateAsync(task.id, options);
3152+
} catch (e) {
3153+
this.log.warn(`Could not remove state of ${task.id}: ${e.message}`);
3154+
}
3155+
}
3156+
try {
3157+
await tools.removeIdFromAllEnums(adapterObjects, task.id, this.enums);
3158+
} catch (e) {
3159+
this.log.warn(`Could not remove ${task.id} from enums: ${e.message}`);
3160+
}
3161+
setImmediate(_deleteObjects, tasks, options, cb);
31543162
});
31553163
}
31563164
};
@@ -3205,52 +3213,38 @@ function Adapter(options) {
32053213
});
32063214
});
32073215
} else {
3208-
adapterObjects.getObject(id, options, (err, obj) => {
3216+
adapterObjects.getObject(id, options, async (err, obj) => {
32093217
if (err) {
32103218
return tools.maybeCallbackWithError(callback, err);
3211-
} else if (!obj) {
3212-
// obj non existing we can return right now
3213-
return tools.maybeCallback(callback);
3214-
} else {
3219+
} else if (obj) {
32153220
// do not allow deletion of objects with dontDelete flag
3216-
if (!obj.common || !obj.common.dontDelete) {
3217-
adapterObjects.delObject(obj._id, options, err => {
3218-
if (err) {
3219-
return tools.maybeCallbackWithError(callback, err);
3220-
} else if (obj.type !== 'state') {
3221-
tools.removeIdFromAllEnums(adapterObjects, id)
3222-
.then(() => {
3223-
return tools.maybeCallback(callback);
3224-
})
3225-
.catch(e => {
3226-
return tools.maybeCallbackWithError(callback, e);
3227-
});
3221+
if (obj.common && obj.common.dontDelete) {
3222+
return tools.maybeCallbackWithError(callback, new Error('not deletable'));
3223+
}
3224+
3225+
try {
3226+
await adapterObjects.delObject(obj._id, options);
3227+
} catch (err) {
3228+
return tools.maybeCallbackWithError(callback, err);
3229+
}
3230+
if (obj.type === 'state') {
3231+
try {
3232+
if (obj.binary) {
3233+
await this.delBinaryStateAsync(id, options);
32283234
} else {
3229-
if (obj.binary) {
3230-
this.delBinaryState(id, options, () =>
3231-
tools.removeIdFromAllEnums(adapterObjects, id)
3232-
.then(() => {
3233-
return tools.maybeCallback(callback);
3234-
})
3235-
.catch(e => {
3236-
return tools.maybeCallbackWithError(callback, e);
3237-
}));
3238-
} else {
3239-
this.delForeignState(id, options, () =>
3240-
tools.removeIdFromAllEnums(adapterObjects, id)
3241-
.then(() => {
3242-
return tools.maybeCallback(callback);
3243-
})
3244-
.catch(e => {
3245-
return tools.maybeCallbackWithError(callback, e);
3246-
}));
3247-
}
3235+
await this.delForeignStateAsync(id, options);
32483236
}
3249-
});
3250-
} else {
3251-
return tools.maybeCallbackWithError(callback, 'not deletable');
3237+
} catch {
3238+
// Ignore
3239+
}
3240+
}
3241+
try {
3242+
await tools.removeIdFromAllEnums(adapterObjects, id, this.enums);
3243+
} catch (e) {
3244+
return tools.maybeCallbackWithError(callback, e);
32523245
}
32533246
}
3247+
return tools.maybeCallback(callback);
32543248
});
32553249
}
32563250
};

‎packages/common/lib/common/tools.js‎

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2262,7 +2262,7 @@ function formatAliasValue(sourceObj, targetObj, state, logger, logNamespace) {
22622262
const func = new Function('val', 'type', 'min', 'max', 'sType', 'sMin', 'sMax', 'return ' + targetObj.alias.read);
22632263
state.val = func(state.val, targetObj.type, targetObj.min, targetObj.max, sourceObj.type, sourceObj.min, sourceObj.max);
22642264
} catch (e) {
2265-
logger.error(`${logNamespace}Invalid read function for ${targetObj._id}: ${targetObj.alias.read} => ${e.message}`);
2265+
logger.error(`${logNamespace} Invalid read function for ${targetObj._id}: ${targetObj.alias.read} => ${e.message}`);
22662266
return null;
22672267
}
22682268
}
@@ -2273,7 +2273,7 @@ function formatAliasValue(sourceObj, targetObj, state, logger, logNamespace) {
22732273
const func = new Function('val', 'type', 'min', 'max', 'tType', 'tMin', 'tMax', 'return ' + sourceObj.alias.write);
22742274
state.val = func(state.val, sourceObj.type, sourceObj.min, sourceObj.max, targetObj.type, targetObj.min, targetObj.max);
22752275
} catch (e) {
2276-
logger.error(`${logNamespace}Invalid write function for ${sourceObj._id}: ${sourceObj.alias.write} => ${e.message}`);
2276+
logger.error(`${logNamespace} Invalid write function for ${sourceObj._id}: ${sourceObj.alias.write} => ${e.message}`);
22772277
return null;
22782278
}
22792279
}
@@ -2317,35 +2317,43 @@ function formatAliasValue(sourceObj, targetObj, state, logger, logNamespace) {
23172317
* @memberof tools
23182318
* @param {object} objects object to access objects db
23192319
* @param {string} id the object id which will be deleted from enums
2320-
* @returns {Promise}
2320+
* @param {object} [allEnums] objects with all enums to use - if not provided all enums will be queried
2321+
* @returns {Promise} All objects are tried to be updated - reject will happen as soon as one fails with the error of the first fail
23212322
*
23222323
*/
2323-
function removeIdFromAllEnums(objects, id) {
2324-
return new Promise((resolve, reject) => {
2325-
objects.getObjectView('system', 'enum', {
2324+
async function removeIdFromAllEnums(objects, id, allEnums) {
2325+
2326+
if (!allEnums) {
2327+
allEnums = {};
2328+
const res = await objects.getObjectViewAsync('system', 'enum', {
23262329
startkey: 'enum.',
23272330
endkey: 'enum.\u9999'
2328-
}, (err, res) => {
2329-
if (err) {
2330-
reject(err);
2331-
} else {
2332-
const promises = [];
2333-
for (const obj of res.rows) {
2334-
const idx = obj.value && obj.value.common && obj.value.common.members ? obj.value.common.members.indexOf(id) : -1;
2335-
if (idx !== -1) {
2336-
// the id is in the enum now we have to remove it
2337-
obj.value.common.members.splice(idx, 1);
2338-
promises.push(new Promise(resolve => {
2339-
objects.setObject(obj.value._id, obj.value, err => {
2340-
err ? reject(err) : resolve();
2341-
});
2342-
}));
2343-
} // endIf
2344-
} // endFor
2345-
Promise.all(promises).then(resolve);
2346-
} // endElse
23472331
});
2348-
});
2332+
if (res && res.rows) {
2333+
for (const row of res.rows) {
2334+
allEnums[row.id] = row.value;
2335+
}
2336+
}
2337+
}
2338+
2339+
let error = null;
2340+
for (const [enumId, enumObj] of Object.entries(allEnums)) {
2341+
const idx = enumObj.common.members ? enumObj.common.members.indexOf(id) : -1;
2342+
if (idx !== -1) {
2343+
// the id is in the enum now we have to remove it
2344+
enumObj.common.members.splice(idx, 1);
2345+
try {
2346+
await objects.setObjectAsync(enumId, enumObj);
2347+
} catch (err) {
2348+
if (!error) {
2349+
error = err;
2350+
}
2351+
}
2352+
}
2353+
}
2354+
if (error) {
2355+
throw error;
2356+
}
23492357
}
23502358

23512359
/**

‎packages/controller/test/lib/testObjectsACL.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ function register(it, expect, context) {
113113
}));
114114
}).timeout(1000);
115115

116-
it(textName + 'invalid user name must be checked', () => {
116+
it(textName + 'invalid user name must be checked #1', () => {
117117
const objects = context.objects;
118118
return objects.getObject(secretId, {user: 'admin'}).then(_obj => {
119119
expect(1).to.be.equal('Never happens');
@@ -123,7 +123,7 @@ function register(it, expect, context) {
123123
});
124124
}).timeout(1000);
125125

126-
it(textName + 'invalid user name must be checked', () => {
126+
it(textName + 'invalid user name must be checked #2', () => {
127127
const objects = context.objects;
128128
return objects.getObject(secretId, {user: 'system.user.admin1'}).then(_obj => {
129129
expect(1).to.be.equal('Never happens');

0 commit comments

Comments
 (0)