diff --git a/docs/data-types.md b/docs/data-types.md index b1cfb53be738..2e3f12f53601 100644 --- a/docs/data-types.md +++ b/docs/data-types.md @@ -229,13 +229,14 @@ modules.exports = function sequelizeAdditions(Sequelize) { } } + DataTypes.NEWTYPE = NEWTYPE; + // Mandatory, set key DataTypes.NEWTYPE.prototype.key = DataTypes.NEWTYPE.key = 'NEWTYPE' - // Optional, disable escaping after stringifier. Not recommended. // Warning: disables Sequelize protection against SQL injections - //DataTypes.NEWTYPE.escape = false + // DataTypes.NEWTYPE.escape = false // For convenience // `classToInvokable` allows you to use the datatype without `new` diff --git a/docs/hooks.md b/docs/hooks.md index 34db02068136..8699315df11d 100644 --- a/docs/hooks.md +++ b/docs/hooks.md @@ -186,11 +186,13 @@ new Sequelize(..., { ### Connection Hooks -Sequelize provides two hooks that are executed immediately before and after a database connection is obtained: +Sequelize provides four hooks that are executed immediately before and after a database connection is obtained or released: ```text beforeConnect(config) afterConnect(connection, config) +beforeDisconnect(connection) +afterDisconnect(connection) ``` These hooks can be useful if you need to asynchronously obtain database credentials, or need to directly access the low-level database connection after it has been created. diff --git a/docs/migrations.md b/docs/migrations.md index 7710c613d225..966dc75b3bb2 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -139,7 +139,9 @@ module.exports = { return queryInterface.bulkInsert('Users', [{ firstName: 'John', lastName: 'Doe', - email: 'demo@demo.com' + email: 'demo@demo.com', + createdAt: Date.now(), + updatedAt: Date.now() }], {}); }, diff --git a/docs/models-definition.md b/docs/models-definition.md index 8881b0216a51..1c7bac0f7d17 100644 --- a/docs/models-definition.md +++ b/docs/models-definition.md @@ -71,7 +71,7 @@ Foo.init({ } }, - // It is possible to add coments on columns for MySQL, PostgreSQL and MSSQL only + // It is possible to add comments on columns for MySQL, PostgreSQL and MSSQL only commentMe: { type: Sequelize.INTEGER, @@ -710,7 +710,7 @@ User.init({}, { } }, - // A BTREE index with a ordered field + // A BTREE index with an ordered field { name: 'title_index', using: 'BTREE', diff --git a/docs/models-usage.md b/docs/models-usage.md index 818c4dcff4c4..2ca6904cff58 100644 --- a/docs/models-usage.md +++ b/docs/models-usage.md @@ -36,6 +36,8 @@ The method `findOrCreate` can be used to check if a certain element already exis Let's assume we have an empty database with a `User` model which has a `username` and a `job`. +`where` option will be appended to `defaults` for create case. + ```js User .findOrCreate({where: {username: 'sdepold'}, defaults: {job: 'Technical Lead JavaScript'}}) diff --git a/lib/dialects/abstract/connection-manager.js b/lib/dialects/abstract/connection-manager.js index a38a14584f24..b37d412e0a2c 100644 --- a/lib/dialects/abstract/connection-manager.js +++ b/lib/dialects/abstract/connection-manager.js @@ -320,7 +320,9 @@ class ConnectionManager { * @returns {Promise} */ _disconnect(connection) { - return this.dialect.connectionManager.disconnect(connection); + return this.sequelize.runHooks('beforeDisconnect', connection) + .then(() => this.dialect.connectionManager.disconnect(connection)) + .then(() => this.sequelize.runHooks('afterDisconnect', connection)); } /** diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 45058507f036..e0800c2d29e0 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -36,9 +36,6 @@ class QueryGenerator { // dialect name this.dialect = options._dialect.name; this._dialect = options._dialect; - - // template config - this._templateSettings = require('lodash').runInContext().templateSettings; } extractTableDetails(tableName, options) { @@ -111,9 +108,9 @@ class QueryGenerator { const quotedTable = this.quoteTable(table); const bindParam = options.bindParam === undefined ? this.bindParam(bind) : options.bindParam; let query; - let valueQuery = `<%= tmpTable %>INSERT<%= ignoreDuplicates %> INTO ${quotedTable} (<%= attributes %>)<%= output %> VALUES (<%= values %>)<%= onConflictDoNothing %>`; - let emptyQuery = `<%= tmpTable %>INSERT<%= ignoreDuplicates %> INTO ${quotedTable}<%= output %><%= onConflictDoNothing %>`; - let outputFragment; + let valueQuery = ''; + let emptyQuery = ''; + let outputFragment = ''; let identityWrapperRequired = false; let tmpTable = ''; //tmpTable declaration for trigger @@ -180,25 +177,6 @@ class QueryGenerator { if (this._dialect.supports.EXCEPTION && options.exception) { // Not currently supported with bind parameters (requires output of multiple queries) options.bindParam = false; - // Mostly for internal use, so we expect the user to know what he's doing! - // pg_temp functions are private per connection, so we never risk this function interfering with another one. - if (semver.gte(this.sequelize.options.databaseVersion, '9.2.0')) { - // >= 9.2 - Use a UUID but prefix with 'func_' (numbers first not allowed) - const delimiter = `$func_${uuidv4().replace(/-/g, '')}$`; - - options.exception = 'WHEN unique_violation THEN GET STACKED DIAGNOSTICS sequelize_caught_exception = PG_EXCEPTION_DETAIL;'; - valueQuery = `${`CREATE OR REPLACE FUNCTION pg_temp.testfunc(OUT response ${quotedTable}, OUT sequelize_caught_exception text) RETURNS RECORD AS ${delimiter}` + - ' BEGIN '}${valueQuery} INTO response; EXCEPTION ${options.exception} END ${delimiter - } LANGUAGE plpgsql; SELECT (testfunc.response).*, testfunc.sequelize_caught_exception FROM pg_temp.testfunc(); DROP FUNCTION IF EXISTS pg_temp.testfunc()`; - } else { - options.exception = 'WHEN unique_violation THEN NULL;'; - valueQuery = `CREATE OR REPLACE FUNCTION pg_temp.testfunc() RETURNS SETOF ${quotedTable} AS $body$ BEGIN RETURN QUERY ${valueQuery}; EXCEPTION ${options.exception} END; $body$ LANGUAGE plpgsql; SELECT * FROM pg_temp.testfunc(); DROP FUNCTION IF EXISTS pg_temp.testfunc();`; - } - } - - if (this._dialect.supports['ON DUPLICATE KEY'] && options.onDuplicate) { - valueQuery += ` ON DUPLICATE KEY ${options.onDuplicate}`; - emptyQuery += ` ON DUPLICATE KEY ${options.onDuplicate}`; } valueHash = Utils.removeNullValuesFromHash(valueHash, this.options.omitNull); @@ -239,6 +217,31 @@ class QueryGenerator { tmpTable }; + valueQuery = `${tmpTable}INSERT${replacements.ignoreDuplicates} INTO ${quotedTable} (${replacements.attributes})${replacements.output} VALUES (${replacements.values})${replacements.onConflictDoNothing}${valueQuery}`; + emptyQuery = `${tmpTable}INSERT${replacements.ignoreDuplicates} INTO ${quotedTable}${replacements.output}${replacements.onConflictDoNothing}${emptyQuery}`; + + if (this._dialect.supports.EXCEPTION && options.exception) { + // Mostly for internal use, so we expect the user to know what he's doing! + // pg_temp functions are private per connection, so we never risk this function interfering with another one. + if (semver.gte(this.sequelize.options.databaseVersion, '9.2.0')) { + // >= 9.2 - Use a UUID but prefix with 'func_' (numbers first not allowed) + const delimiter = `$func_${uuidv4().replace(/-/g, '')}$`; + + options.exception = 'WHEN unique_violation THEN GET STACKED DIAGNOSTICS sequelize_caught_exception = PG_EXCEPTION_DETAIL;'; + valueQuery = `${`CREATE OR REPLACE FUNCTION pg_temp.testfunc(OUT response ${quotedTable}, OUT sequelize_caught_exception text) RETURNS RECORD AS ${delimiter}` + + ' BEGIN '}${valueQuery} INTO response; EXCEPTION ${options.exception} END ${delimiter + } LANGUAGE plpgsql; SELECT (testfunc.response).*, testfunc.sequelize_caught_exception FROM pg_temp.testfunc(); DROP FUNCTION IF EXISTS pg_temp.testfunc()`; + } else { + options.exception = 'WHEN unique_violation THEN NULL;'; + valueQuery = `CREATE OR REPLACE FUNCTION pg_temp.testfunc() RETURNS SETOF ${quotedTable} AS $body$ BEGIN RETURN QUERY ${valueQuery}; EXCEPTION ${options.exception} END; $body$ LANGUAGE plpgsql; SELECT * FROM pg_temp.testfunc(); DROP FUNCTION IF EXISTS pg_temp.testfunc();`; + } + } + + if (this._dialect.supports['ON DUPLICATE KEY'] && options.onDuplicate) { + valueQuery += ` ON DUPLICATE KEY ${options.onDuplicate}`; + emptyQuery += ` ON DUPLICATE KEY ${options.onDuplicate}`; + } + query = `${replacements.attributes.length ? valueQuery : emptyQuery};`; if (this._dialect.supports.finalTable) { query = `SELECT * FROM FINAL TABLE(${ replacements.attributes.length ? valueQuery : emptyQuery });`; @@ -247,7 +250,6 @@ class QueryGenerator { query = `SET IDENTITY_INSERT ${quotedTable} ON; ${query} SET IDENTITY_INSERT ${quotedTable} OFF;`; } - query = _.template(query, this._templateSettings)(replacements); // Used by Postgres upsertQuery and calls to here with options.exception set to true const result = { query }; if (options.bindParam !== false) { @@ -2122,7 +2124,10 @@ class QueryGenerator { } } - if (key === Op.or || key === Op.and || key === Op.not) { + if (key === Op.or || key === Op.and || key === Op.not) { + if (key === Op.or) { + options.hasOr = true; + } return this._whereGroupBind(key, value, options); } @@ -2531,6 +2536,7 @@ class QueryGenerator { } } +// Object.assign(QueryGenerator.prototype, require('../db2/query-generator/operators')); Object.assign(QueryGenerator.prototype, require('./query-generator/operators')); Object.assign(QueryGenerator.prototype, require('./query-generator/transaction')); diff --git a/lib/dialects/db2/data-types.js b/lib/dialects/db2/data-types.js index 118c030c6d2d..2035a73c0971 100644 --- a/lib/dialects/db2/data-types.js +++ b/lib/dialects/db2/data-types.js @@ -48,6 +48,7 @@ module.exports = BaseTypes => { BaseTypes.ENUM.types.db2 = ['VARCHAR']; BaseTypes.REAL.types.db2 = ['REAL']; BaseTypes.DOUBLE.types.db2 = ['DOUBLE']; + BaseTypes.JSON.types.db2 = ['JSON', 'JSONB']; BaseTypes.GEOMETRY.types.db2 = false; class BLOB extends BaseTypes.BLOB { @@ -86,6 +87,29 @@ module.exports = BaseTypes => { } } + class JSONTYPE extends BaseTypes.JSON { + toSql() { + return 'CLOB(32672)'; + } + _stringify(value) { + return JSON.stringify(value); + } + + defaultType() { + return 'JSON'; + } + + static parse(value) { + return JSON.parse(value); + } + } + + class JSONB extends BaseTypes.JSON { + defaultType() { + return 'JSONB'; + } + } + class STRING extends BaseTypes.STRING { toSql() { if (!this._binary) { @@ -134,7 +158,7 @@ module.exports = BaseTypes => { } if (len > 0 ) { this._length = len; } } else { this._length = 32672; } - if ( this._length > 32672 ) + if ( this._length > 4000 ) { len = `CLOB(${ this._length })`; } @@ -149,7 +173,7 @@ module.exports = BaseTypes => { class BOOLEAN extends BaseTypes.BOOLEAN { toSql() { - return 'BOOLEAN'; + return 'INTEGER'; } _sanitize(value) { if (value !== null && value !== undefined) { @@ -171,6 +195,10 @@ module.exports = BaseTypes => { return value; } + + defaultType() { + return 'BOOLEAN'; + } } BOOLEAN.parse = BOOLEAN.prototype._sanitize; @@ -213,6 +241,10 @@ module.exports = BaseTypes => { value = new Date(moment.utc(value)); return value; } + + // defaultType () { + // return 'DATE' + // } } class DATEONLY extends BaseTypes.DATEONLY { @@ -330,6 +362,8 @@ module.exports = BaseTypes => { BIGINT, REAL, FLOAT, - TEXT + TEXT, + JSON: JSONTYPE, + JSONB }; }; diff --git a/lib/dialects/db2/query-generator.js b/lib/dialects/db2/query-generator.js index 38f744ba457f..50f6c7d15acc 100644 --- a/lib/dialects/db2/query-generator.js +++ b/lib/dialects/db2/query-generator.js @@ -6,6 +6,7 @@ const DataTypes = require('../../data-types'); const AbstractQueryGenerator = require('../abstract/query-generator'); const randomBytes = require('crypto').randomBytes; const Op = require('../../operators'); +const Db2QueryInterface = require('./query-interface'); /* istanbul ignore next */ const throwMethodUndefined = function(methodName) { @@ -62,7 +63,7 @@ class Db2QueryGenerator extends AbstractQueryGenerator { let commentStr = ''; for (const attr in attributes) { - if (attributes.hasOwnProperty(attr)) { + if (Object.prototype.hasOwnProperty.call(attributes, attr)) { let dataType = attributes[attr]; let match; @@ -135,7 +136,7 @@ class Db2QueryGenerator extends AbstractQueryGenerator { } for (const fkey in foreignKeys) { - if (foreignKeys.hasOwnProperty(fkey)) { + if (Object.prototype.hasOwnProperty.call(foreignKeys, fkey)) { values.attributes += `, FOREIGN KEY (${ this.quoteIdentifier(fkey) }) ${ foreignKeys[fkey] }`; } } @@ -331,7 +332,7 @@ class Db2QueryGenerator extends AbstractQueryGenerator { _.forEach(attrValueHashes, attrValueHash => { tuples.push(`(${ allAttributes.map(key => - this.escape(attrValueHash[key]), undefined, { context: 'INSERT' }).join(',')})`); + this.escape(attrValueHash[key], attributes[key], { context: 'INSERT' })).join(',')})`); }); allQueries.push(query); @@ -387,7 +388,7 @@ class Db2QueryGenerator extends AbstractQueryGenerator { let query; const whereOptions = _.defaults({ bindParam }, options); - query = `UPDATE (SELECT * FROM ${this.quoteTable(tableName)} ${this.whereQuery(where, whereOptions)} FETCH NEXT ${this.escape(options.limit)} ROWS ONLY) SET ${values.join(',')}`; + query = `UPDATE (SELECT * FROM ${this.quoteTable(tableName)} ${this.whereQuery(where, whereOptions)} LIMIT ${this.escape(options.limit)} ) SET ${values.join(',')}`; query = `SELECT * FROM FINAL TABLE (${ query })`; return { query, bind }; } @@ -505,11 +506,12 @@ class Db2QueryGenerator extends AbstractQueryGenerator { let limit = ''; - if (options.offset > 0) { - limit = ` OFFSET ${ this.escape(options.offset) } ROWS`; - } if (options.limit) { - limit += ` FETCH NEXT ${ this.escape(options.limit) } ROWS ONLY`; + limit += ` LIMIT ${ this.escape(options.limit) } `; + } + + if (options.offset > 0) { + limit = ` OFFSET ${ this.escape(options.offset) } `; } const replacements = { @@ -608,8 +610,12 @@ class Db2QueryGenerator extends AbstractQueryGenerator { } // Blobs/texts cannot have a defaultValue - if (attribute.type !== 'TEXT' && attribute.type._binary !== true && - Utils.defaultValueSchemable(attribute.defaultValue)) { + if (_.get(attribute, 'type.constructor.name', '') === 'JSONTYPE' ) { + if (attribute.defaultValue) { + template += ` DEFAULT '${JSON.stringify(attribute.defaultValue)}'`; + } + } else if (attribute.type !== 'TEXT' && attribute.type._binary !== true && + Utils.defaultValueSchemable(attribute.defaultValue)) { template += ` DEFAULT ${this.escape(attribute.defaultValue)}`; } @@ -837,12 +843,12 @@ class Db2QueryGenerator extends AbstractQueryGenerator { const offset = options.offset || 0; let fragment = ''; - if (offset > 0) { - fragment += ` OFFSET ${ this.escape(offset) } ROWS`; + if (options.limit) { + fragment += ` LIMIT ${ this.escape(options.limit) } `; } - if (options.limit) { - fragment += ` FETCH NEXT ${ this.escape(options.limit) } ROWS ONLY`; + if (offset > 0) { + fragment += ` OFFSET ${ this.escape(offset) } `; } return fragment; @@ -869,6 +875,147 @@ class Db2QueryGenerator extends AbstractQueryGenerator { } return uniqno; } + + // OR/AND/N_whereGroupBindOT grouping logic + _whereGroupBind(key, value, options) { + const binding = key === Op.or ? this.OperatorMap[Op.or] : this.OperatorMap[Op.and]; + const outerBinding = key === Op.not ? 'NOT ': ''; + + let nextValue = value; + if (Array.isArray(value)) { + nextValue = value.map(item => { + let itemQuery = this.whereItemsQuery(item, options, this.OperatorMap[Op.and]); + if (itemQuery && itemQuery.length && (Array.isArray(item) || _.isPlainObject(item)) && Utils.getComplexSize(item) > 1) { + itemQuery = `(${itemQuery})`; + } + return itemQuery; + }).filter(item => item && item.length); + + nextValue = nextValue.length && nextValue.join(binding); + } else { + nextValue = this.whereItemsQuery(value, options, binding); + } + // Op.or: [] should return no data. + // Op.not of no restriction should also return no data + if ((key === Op.or || key === Op.not) && !nextValue) { + const isArray = Array.isArray(value); + // 如果内部是 json 条件,则跳过 + if (_.every(value, (item, k) => Db2QueryInterface.isJsonFilter(options.model, isArray ? item : { [k]: item }))) { + return undefined; + } + return '0 = 1'; + } + + return nextValue ? `${outerBinding}(${nextValue})` : undefined; + } + + // db2 不支持 json 筛选,忽略 json 条件 + _whereJSON(key, value, options) { + if (options.hasOr) { + return ' 1 = 1'; + } + return ''; + } + + _whereParseSingleValueObject(key, field, prop, value, options) { + if (prop === Op.not) { + if (Array.isArray(value)) { + prop = Op.notIn; + } else if (value !== null && value !== true && value !== false) { + prop = Op.ne; + } + } + + let comparator = this.OperatorMap[prop] || this.OperatorMap[Op.eq]; + switch (prop) { + case Op.in: + case Op.notIn: + if (value instanceof Utils.Literal) { + return this._joinKeyValue(key, value.val, comparator, options.prefix); + } + + if (value.length) { + return this._joinKeyValue(key, `(${value.map(item => this.escape(item, field)).join(', ')})`, comparator, options.prefix); + } + + if (comparator === this.OperatorMap[Op.in]) { + return this._joinKeyValue(key, '(NULL)', comparator, options.prefix); + } + + return ''; + case Op.any: + case Op.all: + comparator = `${this.OperatorMap[Op.eq]} ${comparator}`; + if (value[Op.values]) { + return this._joinKeyValue(key, `(VALUES ${value[Op.values].map(item => `(${this.escape(item)})`).join(', ')})`, comparator, options.prefix); + } + + return this._joinKeyValue(key, `(${this.escape(value, field)})`, comparator, options.prefix); + case Op.between: + case Op.notBetween: + return this._joinKeyValue(key, `${this.escape(value[0], field)} AND ${this.escape(value[1], field)}`, comparator, options.prefix); + case Op.raw: + throw new Error('The `$raw` where property is no longer supported. Use `sequelize.literal` instead.'); + case Op.col: + comparator = this.OperatorMap[Op.eq]; + value = value.split('.'); + + if (value.length > 2) { + value = [ + // join the tables by -> to match out internal namings + value.slice(0, -1).join('->'), + value[value.length - 1] + ]; + } + + return this._joinKeyValue(key, value.map(identifier => this.quoteIdentifier(identifier)).join('.'), comparator, options.prefix); + case Op.startsWith: + comparator = this.OperatorMap[Op.like]; + return `upper(${ options.prefix}."${key}") ${comparator} ${this.escape(`${_.toUpper(value)}%`)}`; + case Op.endsWith: + comparator = this.OperatorMap[Op.like]; + return `upper(${ options.prefix}."${key}") ${comparator} ${this.escape(`$${_.toUpper(value)}`)}`; + case Op.substring: + comparator = this.OperatorMap[Op.like]; + return `upper(${ options.prefix}."${key}") ${comparator} ${this.escape(`%${_.toUpper(value)}%`)}`; + case Op.like: + case Op.iLike: + comparator = this.OperatorMap[Op.like]; + return `upper(${ options.prefix}."${key}") ${comparator} ${this.escape(`${_.toUpper(value)}`)}`; + case Op.notLike: + case Op.notILike: + comparator = this.OperatorMap[Op.notLike]; + return `upper(${ options.prefix}."${key}") ${comparator} ${this.escape(`${_.toUpper(value)}`)}`; + } + + const escapeOptions = { + acceptStrings: comparator.includes(this.OperatorMap[Op.like]) + }; + + if (_.isPlainObject(value)) { + if (value[Op.col]) { + return this._joinKeyValue(key, this.whereItemQuery(null, value), comparator, options.prefix); + } + if (value[Op.any]) { + escapeOptions.isList = true; + return this._joinKeyValue(key, `(${this.escape(value[Op.any], field, escapeOptions)})`, `${comparator} ${this.OperatorMap[Op.any]}`, options.prefix); + } + if (value[Op.all]) { + escapeOptions.isList = true; + return this._joinKeyValue(key, `(${this.escape(value[Op.all], field, escapeOptions)})`, `${comparator} ${this.OperatorMap[Op.all]}`, options.prefix); + } + } + + if (value === null && comparator === this.OperatorMap[Op.eq]) { + return this._joinKeyValue(key, this.escape(value, field, escapeOptions), this.OperatorMap[Op.is], options.prefix); + } + if (value === null && comparator === this.OperatorMap[Op.ne]) { + return this._joinKeyValue(key, this.escape(value, field, escapeOptions), this.OperatorMap[Op.not], options.prefix); + } + + return this._joinKeyValue(key, this.escape(value, field, escapeOptions), comparator, options.prefix); + } + } // private methods diff --git a/lib/dialects/db2/query-generator/operators.js b/lib/dialects/db2/query-generator/operators.js new file mode 100644 index 000000000000..2b6733c97f8e --- /dev/null +++ b/lib/dialects/db2/query-generator/operators.js @@ -0,0 +1,84 @@ +'use strict'; + +const _ = require('lodash'); +const Op = require('../../../operators'); +const Utils = require('../../../utils'); + +const OperatorHelpers = { + OperatorMap: { + [Op.eq]: '=', + [Op.ne]: '!=', + [Op.gte]: '>=', + [Op.gt]: '>', + [Op.lte]: '<=', + [Op.lt]: '<', + [Op.not]: 'IS NOT', + [Op.is]: 'IS', + [Op.in]: 'IN', + [Op.notIn]: 'NOT IN', + [Op.like]: 'LIKE', + [Op.notLike]: 'NOT LIKE', + [Op.iLike]: 'LIKE', + [Op.notILike]: 'NOT LIKE', + [Op.startsWith]: 'LIKE', + [Op.endsWith]: 'LIKE', + [Op.substring]: 'LIKE', + [Op.regexp]: '~', + [Op.notRegexp]: '!~', + [Op.iRegexp]: '~*', + [Op.notIRegexp]: '!~*', + [Op.between]: 'BETWEEN', + [Op.notBetween]: 'NOT BETWEEN', + [Op.overlap]: '&&', + [Op.contains]: '@>', + [Op.contained]: '<@', + [Op.adjacent]: '-|-', + [Op.strictLeft]: '<<', + [Op.strictRight]: '>>', + [Op.noExtendRight]: '&<', + [Op.noExtendLeft]: '&>', + [Op.any]: 'ANY', + [Op.all]: 'ALL', + [Op.and]: ' AND ', + [Op.or]: ' OR ', + [Op.col]: 'COL', + [Op.placeholder]: '$$PLACEHOLDER$$' + }, + + OperatorsAliasMap: {}, + + setOperatorsAliases(aliases) { + if (!aliases || _.isEmpty(aliases)) { + this.OperatorsAliasMap = false; + } else { + this.OperatorsAliasMap = Object.assign({}, aliases); + } + }, + + _replaceAliases(orig) { + const obj = {}; + if (!this.OperatorsAliasMap) { + return orig; + } + + Utils.getOperators(orig).forEach(op => { + const item = orig[op]; + if (_.isPlainObject(item)) { + obj[op] = this._replaceAliases(item); + } else { + obj[op] = item; + } + }); + + _.forOwn(orig, (item, prop) => { + prop = this.OperatorsAliasMap[prop] || prop; + if (_.isPlainObject(item)) { + item = this._replaceAliases(item); + } + obj[prop] = item; + }); + return obj; + } +}; + +module.exports = OperatorHelpers; diff --git a/lib/dialects/db2/query-interface.js b/lib/dialects/db2/query-interface.js new file mode 100644 index 000000000000..533955eaf5af --- /dev/null +++ b/lib/dialects/db2/query-interface.js @@ -0,0 +1,712 @@ +'use strict'; + +const DataTypes = require('../../data-types'); +// const Promise = require('../../promise'); +// const QueryTypes = require('../../query-types'); +const _ = require('lodash'); +const Utils = require('../../utils'); +const Op = require('../../operators'); +// const OpHelper = require('./query-generator/operators'); +const OpHelper = require('../abstract/query-generator/operators'); +const AsyncStream = require('async-streamjs').default; + +function wrapInArr(objOrArr) { + return Array.isArray(objOrArr) ? objOrArr : [objOrArr]; +} + +/** + Returns an object that handles Db2's inabilities to do certain queries. + + @class QueryInterface + @static + @private + */ + +function getWhereConditions(smth, tableName, factory, options, prepend) { + if (Array.isArray(tableName)) { + tableName = tableName[0]; + if (Array.isArray(tableName)) { + tableName = tableName[1]; + } + } + + options = options || {}; + + if (prepend === undefined) { + prepend = true; + } + + // TODO support + if (smth && smth instanceof Utils.SequelizeMethod) { // Checking a property is cheaper than a lot of instanceof calls + // return this.handleSequelizeMethod(smth, tableName, factory, options, prepend); + throw new Error('Unimplemented'); + } + if (_.isPlainObject(smth)) { + return whereItemsQuery(smth, { + model: factory, + // prefix: prepend && tableName, + type: options.type, + queryInterface: options.queryInterface + }); + } + if (typeof smth === 'number') { + return []; + } + if (typeof smth === 'string') { + return whereItemsQuery(smth, { + model: factory, + // prefix: prepend && tableName, + queryInterface: options.queryInterface + }); + } + if (Buffer.isBuffer(smth)) { + throw new Error('Unimplemented'); + // return this.escape(smth); + } + if (Array.isArray(smth)) { + if (smth.length === 0 || smth.length > 0 && smth[0].length === 0) return []; + if (Utils.canTreatArrayAsAnd(smth)) { + const _smth = { [Op.and]: smth }; + return getWhereConditions(_smth, tableName, factory, options, prepend); + } + throw new Error('Support for literal replacements in the `where` object has been removed.'); + } + if (smth === null) { + return whereItemsQuery(smth, { + model: factory, + // prefix: prepend && tableName, + queryInterface: options.queryInterface + }); + } + + return []; +} + +// return function or function[] +function whereItemsQuery(where, options, binding) { + if ( + where === null || + where === undefined || + Utils.getComplexSize(where) === 0 + ) { + // NO OP + return []; + } + + if (typeof where === 'string') { + throw new Error('Support for `{where: \'raw query\'}` has been removed.'); + } + + const items = []; + + if (_.isPlainObject(where)) { + Utils.getComplexKeys(where).forEach(prop => { + const item = where[prop]; + items.push(...wrapInArr(whereItemQuery(prop, item, options))); + }); + } else { + items.push(...wrapInArr(whereItemQuery(undefined, where, options))); + } + + // return items.length && items.filter(item => item && item.length).join(binding) || ''; + return /or/i.test(binding) ? _.overSome(items) : items; +} + +// return function or function[] +function whereItemQuery(key, value, options = {}) { + if (value === undefined) { + throw new Error(`WHERE parameter "${key}" has invalid "undefined" value`); + } + + const queryInterface = options.queryInterface; + if (typeof key === 'string' && key.includes('.') && options.model) { + const keyParts = key.split('.'); + if (options.model.rawAttributes[keyParts[0]] && options.model.rawAttributes[keyParts[0]].type instanceof DataTypes.JSON) { + const tmp = {}; + const field = options.model.rawAttributes[keyParts[0]]; + _.set(tmp, keyParts.slice(1), value); + return whereItemQuery(field.field || keyParts[0], tmp, Object.assign({ field }, options)); + } + } + + const field = _findField(key, options); + const fieldType = field && field.type || options.type; + + if (!options.hasOr && field && !(fieldType instanceof DataTypes.JSON)) { + return []; + } + const isPlainObject = _.isPlainObject(value); + const isArray = !isPlainObject && Array.isArray(value); + key = queryInterface.QueryGenerator.OperatorsAliasMap && queryInterface.QueryGenerator.OperatorsAliasMap[key] || key; + if (isPlainObject) { + value = queryInterface.QueryGenerator._replaceAliases(value); + } + const valueKeys = isPlainObject && Utils.getComplexKeys(value); + + if (key === undefined) { + if (typeof value === 'string') { + throw new Error('Unimplemented'); + // return value; + } + + if (isPlainObject && valueKeys.length === 1) { + return whereItemQuery(valueKeys[0], value[valueKeys[0]], options); + } + } + + if (value === null) { + // const opValue = options.bindParam ? 'NULL' : this.escape(value, field); + const opValue = value; + return _joinKeyValue(key, opValue, queryInterface.QueryGenerator.OperatorMap[Op.is], options.prefix); + } + + if (!value) { + // const opValue = options.bindParam ? this.format(value, field, options, options.bindParam) : this.escape(value, field); + const opValue = value; + return _joinKeyValue(key, opValue, queryInterface.QueryGenerator.OperatorMap[Op.eq], options.prefix); + } + + if (value instanceof Utils.SequelizeMethod && !(key !== undefined && value instanceof Utils.Fn)) { + throw new Error('Unimplemented'); + // return this.handleSequelizeMethod(value); + } + + // Convert where: [] to Op.and if possible, else treat as literal/replacements + if (key === undefined && isArray) { + if (Utils.canTreatArrayAsAnd(value)) { + key = Op.and; + } else { + throw new Error('Support for literal replacements in the `where` object has been removed.'); + } + } + + if (key === Op.or || key === Op.and || key === Op.not) { + if (key === Op.or) { + options.hasOr = true; + } + return _whereGroupBind(key, value, options); + } + + if (value[Op.or]) { + return _whereBind(queryInterface.QueryGenerator.OperatorMap[Op.or], key, value[Op.or], options); + } + + if (value[Op.and]) { + return _whereBind(queryInterface.QueryGenerator.OperatorMap[Op.and], key, value[Op.and], options); + } + + if (isArray && fieldType instanceof DataTypes.ARRAY) { + // const opValue = options.bindParam ? this.format(value, field, options, options.bindParam) : this.escape(value, field); + const opValue = value; + return _joinKeyValue(key, opValue, queryInterface.QueryGenerator.OperatorMap[Op.eq], options.prefix); + } + + if (isPlainObject && fieldType instanceof DataTypes.JSON && options.json !== false) { + return _whereJSON(key, value, options); + } + // If multiple keys we combine the different logic conditions + if (isPlainObject && valueKeys.length > 1) { + return _whereBind(queryInterface.QueryGenerator.OperatorMap[Op.and], key, value, options); + } + + if (isArray) { + return _whereParseSingleValueObject(key, field, Op.in, value, options); + } + if (isPlainObject) { + if (queryInterface.QueryGenerator.OperatorMap[valueKeys[0]]) { + return _whereParseSingleValueObject(key, field, valueKeys[0], value[valueKeys[0]], options); + } + return _whereParseSingleValueObject(key, field, queryInterface.QueryGenerator.OperatorMap[Op.eq], value, options); + } + + if (key === Op.placeholder) { + // const opValue = options.bindParam ? this.format(value, field, options, options.bindParam) : this.escape(value, field); + const opValue = value; + return _joinKeyValue(queryInterface.QueryGenerator.OperatorMap[key], opValue, queryInterface.QueryGenerator.OperatorMap[Op.eq], options.prefix); + } + + // const opValue = options.bindParam ? this.format(value, field, options, options.bindParam) : this.escape(value, field); + const opValue = value; + return _joinKeyValue(key, opValue, queryInterface.QueryGenerator.OperatorMap[Op.eq], options.prefix); +} + +// OR/AND/NOT grouping logic +function _whereGroupBind(key, value, options) { + const queryInterface = options.queryInterface; + const binding = key === Op.or ? queryInterface.QueryGenerator.OperatorMap[Op.or] : queryInterface.QueryGenerator.OperatorMap[Op.and]; + const outerBinding = key === Op.not ? 'NOT ': ''; + + if (Array.isArray(value)) { + value = _.flatMap(value, item => { + let filters = wrapInArr(whereItemsQuery(item, options, queryInterface.QueryGenerator.OperatorMap[Op.and])); + if (filters && filters.length && (Array.isArray(item) || _.isPlainObject(item)) && Utils.getComplexSize(item) > 1) { + // filters = `(${filters})`; + filters = _.overEvery(filters); + } + return filters; + }).filter(_.identity); + + // value = value.length && value.join(binding); + value = value.length && /or/i.test(binding) ? _.overSome(value) : value; + } else { + value = whereItemsQuery(value, options, binding); + } + // Op.or: [] should return no data. + // Op.not of no restriction should also return no data + if ((key === Op.or || key === Op.not) && !value) { + // return '0 = 1'; + return [() => false]; + } + + // return value ? `${outerBinding}(${value})` : undefined; + return value + ? outerBinding ? [_.negate(_.overEvery(value))] : [_.overEvery(value)] + : []; +} + +function _whereBind(binding, key, value, options) { + if (_.isPlainObject(value)) { + value = _.flatMap(Utils.getComplexKeys(value), prop => { + const item = value[prop]; + return whereItemQuery(key, { [prop]: item }, options); + }); + } else { + value = _.flatMap(value, item => whereItemQuery(key, item, options)); + } + + // value = value.filter(item => item && item.length); + value = value.filter(_.identity); + + // return value.length ? `(${value.join(binding)})` : undefined; + return value.length + ? /or/i.test(binding) ? _.overSome(value) : _.overEvery(value) + : []; +} + + +function _whereJSON(key, value, options) { + // const queryInterface = options.queryInterface; + const items = []; + // let baseKey = this.quoteIdentifier(key); + const baseKey = key; + // if (options.prefix) { + // if (options.prefix instanceof Utils.Literal) { + // baseKey = `${this.handleSequelizeMethod(options.prefix)}.${baseKey}`; + // } else { + // baseKey = `${this.quoteTable(options.prefix)}.${baseKey}`; + // } + // } + + Utils.getOperators(value).forEach(op => { + const where = { + [op]: value[op] + }; + items.push(...wrapInArr(whereItemQuery(key, where, Object.assign({}, options, { json: false })))); + }); + + _.forOwn(value, (item, prop) => { + _traverseJSON(items, baseKey, prop, item, [prop], _.pick(options, 'queryInterface')); + }); + + // const result = items.join(queryInterface.QueryGenerator.OperatorMap[Op.and]); + // return 1 < items.length ? `(${result})` : result; + return 1 < items.length + ? _.overEvery(items) + : items; +} + +function _whereParseSingleValueObject(key, field, prop, value, options) { + const queryInterface = options.queryInterface; + if (prop === Op.not) { + if (Array.isArray(value)) { + prop = Op.notIn; + } else if (value !== null && value !== true && value !== false) { + prop = Op.ne; + } + } + + const comparator = queryInterface.QueryGenerator.OperatorMap[prop] || queryInterface.QueryGenerator.OperatorMap[Op.eq]; + + switch (prop) { + case Op.in: + case Op.notIn: + if (value instanceof Utils.Literal) { + return _joinKeyValue(key, value.val, comparator, options.prefix); + } + + if (value.length) { + // return _joinKeyValue(key, `(${value.map(item => this.escape(item, field)).join(', ')})`, comparator, options.prefix); + return _joinKeyValue(key, value, comparator, options.prefix); + } + + if (comparator === queryInterface.QueryGenerator.OperatorMap[Op.in]) { + return _joinKeyValue(key, '(NULL)', comparator, options.prefix); + } + + return ''; + case Op.any: + case Op.all: + throw new Error('Unimplemented'); + // comparator = `${queryInterface.QueryGenerator.OperatorMap[Op.eq]} ${comparator}`; + // if (value[Op.values]) { + // return _joinKeyValue(key, `(VALUES ${value[Op.values].map(item => `(${this.escape(item)})`).join(', ')})`, comparator, options.prefix); + // } + // + // return _joinKeyValue(key, `(${this.escape(value, field)})`, comparator, options.prefix); + case Op.between: + case Op.notBetween: + throw new Error('Unimplemented'); + // return _joinKeyValue(key, `${this.escape(value[0], field)} AND ${this.escape(value[1], field)}`, comparator, options.prefix); + case Op.raw: + throw new Error('The `$raw` where property is no longer supported. Use `sequelize.literal` instead.'); + case Op.col: + throw new Error('Unimplemented'); + // comparator = queryInterface.QueryGenerator.OperatorMap[Op.eq]; + // value = value.split('.'); + // + // if (value.length > 2) { + // value = [ + // // join the tables by -> to match out internal namings + // value.slice(0, -1).join('->'), + // value[value.length - 1] + // ]; + // } + // + // return _joinKeyValue(key, value.map(identifier => this.quoteIdentifier(identifier)).join('.'), comparator, options.prefix); + case Op.startsWith: + throw new Error('Unimplemented'); + // comparator = queryInterface.QueryGenerator.OperatorMap[Op.like]; + // return _joinKeyValue(key, this.escape(`${value}%`), comparator, options.prefix); + case Op.endsWith: + throw new Error('Unimplemented'); + // comparator = queryInterface.QueryGenerator.OperatorMap[Op.like]; + // return _joinKeyValue(key, this.escape(`%${value}`), comparator, options.prefix); + case Op.substring: + throw new Error('Unimplemented'); + // comparator = queryInterface.QueryGenerator.OperatorMap[Op.like]; + // return _joinKeyValue(key, this.escape(`%${value}%`), comparator, options.prefix); + } + + // const escapeOptions = { + // acceptStrings: comparator.includes(queryInterface.QueryGenerator.OperatorMap[Op.like]) + // }; + + if (_.isPlainObject(value)) { + throw new Error('Unimplemented'); + + // if (value[Op.col]) { + // return _joinKeyValue(key, whereItemQuery(null, value), comparator, options.prefix); + // } + // if (value[Op.any]) { + // escapeOptions.isList = true; + // return _joinKeyValue(key, `(${this.escape(value[Op.any], field, escapeOptions)})`, `${comparator} ${queryInterface.QueryGenerator.OperatorMap[Op.any]}`, options.prefix); + // } + // if (value[Op.all]) { + // escapeOptions.isList = true; + // return _joinKeyValue(key, `(${this.escape(value[Op.all], field, escapeOptions)})`, `${comparator} ${queryInterface.QueryGenerator.OperatorMap[Op.all]}`, options.prefix); + // } + } + + if (value === null && comparator === queryInterface.QueryGenerator.OperatorMap[Op.eq]) { + return _joinKeyValue(key, value, queryInterface.QueryGenerator.OperatorMap[Op.is], options.prefix); + } + if (value === null && comparator === queryInterface.QueryGenerator.OperatorMap[Op.ne]) { + return _joinKeyValue(key, value, queryInterface.QueryGenerator.OperatorMap[Op.not], options.prefix); + } + + return _joinKeyValue(key, value, comparator, options.prefix); +} + + +function _traverseJSON(items, baseKey, prop, item, path, options) { + // let cast; + // + // if (path[path.length - 1].includes('::')) { + // const tmp = path[path.length - 1].split('::'); + // cast = tmp[1]; + // path[path.length - 1] = tmp[0]; + // } + + // const pathKey = this.jsonPathExtractionQuery(baseKey, path); + const pathKey = `${baseKey}.${_.isArray(path) ? path.join('.') : path}`; + + if (_.isPlainObject(item)) { + Utils.getOperators(item).forEach(op => { + // const value = this._toJSONValue(item[op]); + const value = item[op]; + items.push(...wrapInArr(whereItemQuery(pathKey, { [op]: value }, options))); + }); + _.forOwn(item, (value, itemProp) => { + _traverseJSON(items, baseKey, itemProp, value, path.concat([itemProp]), options); + }); + + return; + } + + // item = this._toJSONValue(item); + items.push(...wrapInArr(whereItemQuery(pathKey, { [Op.eq]: item }, options))); +} + +function _findField(key, options) { + if (options.field) { + return options.field; + } + + if (options.model && options.model.rawAttributes && options.model.rawAttributes[key]) { + return options.model.rawAttributes[key]; + } + + if (options.model && options.model.fieldRawAttributesMap && options.model.fieldRawAttributesMap[key]) { + return options.model.fieldRawAttributesMap[key]; + } +} + +/** + * 转义正则字符,以便将搜索条件插入正则 + * 参考于:https://stackoverflow.com/questions/2593637/how-to-escape-regular-expression-in-javascript + * + * @example + * escapeForRegex('a[0]') -> "a\[0\]" + * @param {string} str + */ +function escapeForRegex(str) { + return `${str}`.replace(/[.?*+^$[\]\\(){}|-]/g, '\\$&'); +} + +const likeToRegexMem = _.memoize((likeExp, flags) => { + if (10 < likeToRegexMem.cache.size) { + likeToRegexMem.cache.clear(); + } + const escaped = escapeForRegex(likeExp).replace(/%/g, '.*'); + return new RegExp(escaped, flags); +}, (likeExp, flags) => `${likeExp}|${flags}`); + +function dataValuetoString(a) { + if (_.isString(a)) { + return a; + } + if (_.isArray(a)) { + return a.toString(); + } + if (_.isObject(a)) { + return JSON.stringify(a); + } + return a; +} + +const JsOpDict = { + [Op.eq]: (a, b) => a === b, + [Op.ne]: (a, b) => a !== b, + [Op.gte]: (a, b) => _.isNumber(a) && a >= b, + [Op.gt]: (a, b) => _.isNumber(a) && a> b, + [Op.lte]: (a, b) => _.isNumber(a) && a <= b, + [Op.lt]: (a, b) => _.isNumber(a) && a < b, + [Op.not]: (a, b) => a !== b, // ? + [Op.is]: (a, b) => a === b, + [Op.in]: (a, b) => _.includes(b, a), + [Op.notIn]: (a, b) => !_.includes(b, a), + [Op.like]: (a, b) => { + const val = dataValuetoString(a); + return likeToRegexMem(b).test(val); + }, + [Op.notLike]: (a, b) => { + const val = dataValuetoString(a); + return !likeToRegexMem(b).test(val); + }, + [Op.iLike]: (a, b) => { + const val = dataValuetoString(a); + return likeToRegexMem(b, 'i').test(val); + }, + [Op.notILike]: (a, b) => { + const val = dataValuetoString(a); + return !likeToRegexMem(b, 'i').test(val); + }, + [Op.startsWith]: 'LIKE', + [Op.endsWith]: 'LIKE', + [Op.substring]: 'LIKE', + [Op.regexp]: (a, b) => b.test(a), + [Op.notRegexp]: '!~', + [Op.iRegexp]: '~*', + [Op.notIRegexp]: '!~*', + [Op.between]: (a, b) => { + const [start, end] = b; + return a >= start && a < end; + }, + [Op.notBetween]: (a, b) => { + const [start, end] = b; + return a < start && a >= end; + }, + [Op.overlap]: '&&', + [Op.contains]: (a, b) => { + if (_.isString(b)) { + return a.includes(b); + } if (_.isArray(b)) { + return _.difference(b, a).length === 0; + } if (_.isObject(b)) { + return JSON.stringify(_.pick(a, _.keys(b))) === JSON.stringify(b); + } + return false; + }, + [Op.contained]: '<@', + [Op.adjacent]: '-|-', + [Op.strictLeft]: '<<', + [Op.strictRight]: '>>', + [Op.noExtendRight]: '&<', + [Op.noExtendLeft]: '&>', + [Op.any]: 'ANY', + [Op.all]: 'ALL', + [Op.and]: ' AND ', + [Op.or]: ' OR ', + [Op.col]: 'COL', + [Op.placeholder]: '$$PLACEHOLDER$$' +}; + +const sqlOpToJsOpDict = Utils.getComplexKeys(OpHelper.OperatorMap).reduce((acc, symb) => { + const sqlOp = OpHelper.OperatorMap[symb]; + acc[sqlOp] = acc[sqlOp] || JsOpDict[symb]; + return acc; +}, {}); + +function _joinKeyValue(key, value, comparator, prefix) { + if (!key) { + return value; + } + if (comparator === undefined) { + throw new Error(`${key} and ${value} has no comparator`); + } + + const jsOp = sqlOpToJsOpDict[comparator]; + const path = prefix ? _.toPath(`${prefix}.${key}`) : _.toPath(key); + return obj => { + const colVal = _.get(obj, path, null); + return jsOp(colVal, value); + }; + // key = this._getSafeKey(key, prefix); + // return [key, value].join(` ${comparator} `); +} + + + +/** + * 针对 where 里的 json 筛选,生成客户端筛选函数 + * + * @param {string} tableName + * @param {Object} options + * @param {Model} model + * + * @returns {Function} + */ +function generateClientSideFilterForJsonCond(tableName, options, model) { + const filters = getWhereConditions(options.where, tableName, model, options); + return _.isEmpty(filters) + ? _.identity + : _.size(filters) === 1 + ? filters[0] + : _.overEvery(filters); +} + +/** + * 判断filter中是否包含json字段 + * + * @param {Object} model + * @param {Object} where + */ +function isJsonFilter(model, where) { + const filterKeys = getFilterAllKeys(where); + return _.some(filterKeys, key => { + return model.rawAttributes[key] && model.rawAttributes[key].type instanceof DataTypes.JSON; + }); +} + +/** + * 获取filter所有的key + * + * @param {*} filter + */ +function getFilterAllKeys(filter) { + const keys = _.keys(filter).map(k => { + const index = k.indexOf('.'); + return index > 0 ? k.substring(0, index) : k; + }); + const childKeys = _.values(filter).map(v => { + if (typeof v === 'object') { + return getFilterAllKeys(v); + } + return ''; + }).filter(_.identity); + return _.concat(keys, _.flatten(childKeys)); +} + +/** + * 生成主键的filter + * + * @param {Function} data + * @param {Object} primaryKeyField + */ +function getPrimaryKeyFilter(data, primaryKeyField) { + const pks = primaryKeyField.split(',').filter(_.identity); + if (!pks.length) { + //throw error + throw new Error('operation table does not exist primaryKey'); + } + if (!data) { + return { + [pks[0]]: { + $in: [] + } + }; + } + if (pks.length === 1) { + return { + [pks[0]]: { + $in: data.map(p => _.get(p, pks[0], '')) + } + }; + } + return { + $or: data.map(p => { + return _.reduce(pks, (r, v) => { + r[v] = _.get(p, v, ''); + return r; + }, {}); + }) + }; +} + +/** + * 分批查询,由于全部查询出来可能会造成内存问题,所以需要分批进行查询 + * + * @param {Function} batchQueryFunc ({dbOffset, dbLimit}) => [] + * @param {Function} jsFilterFunc + * @param {number} resultOffset + * @param {number} resultLimit + * @param {number} batchSize + */ +function batchQuery(batchQueryFunc, jsFilterFunc = _.identity, resultOffset = 0, resultLimit = Infinity, batchSize = 512) { + if (resultLimit === null || resultLimit === undefined) { + resultLimit = Infinity; + } + let done = false; + return AsyncStream.range() + .map(batchIndex => { + if (done) { + return []; + } + return batchQueryFunc({ dbOffset: batchIndex * batchSize, dbLimit: batchSize }).then(arr => { + if (_.size(arr) < batchSize) { + done = true; + } + return arr.filter(jsFilterFunc); + }); + }) + .takeWhile(arr => 0 < arr.length) + .flatMap(x => x) + .drop(resultOffset) + .take(resultLimit) + .toArray(); +} + +exports.generateClientSideFilterForJsonCond = generateClientSideFilterForJsonCond; +exports.getPrimaryKeyFilter = getPrimaryKeyFilter; +exports.isJsonFilter = isJsonFilter; +exports.batchQuery = batchQuery; diff --git a/lib/dialects/db2/query.js b/lib/dialects/db2/query.js index f12764dadfca..b798130fd979 100644 --- a/lib/dialects/db2/query.js +++ b/lib/dialects/db2/query.js @@ -138,8 +138,23 @@ class Query extends AbstractQuery { if (datalen > 0) { const coltypes = {}; for (let i = 0; i < metadata.length; i++) { - coltypes[metadata[i].SQL_DESC_CONCISE_TYPE] = - metadata[i].SQL_DESC_TYPE_NAME; + if (!this.model) { + coltypes[metadata[i].SQL_DESC_CONCISE_TYPE] = metadata[i].SQL_DESC_TYPE_NAME; + } else { + //处理include表数据类型 + let dataType = null; + const lastSplit = metadata[i].SQL_DESC_CONCISE_TYPE.lastIndexOf('.'); + const tablePrefix = lastSplit > 0 ? metadata[i].SQL_DESC_CONCISE_TYPE.substring(0, lastSplit) : ''; + const fieldName = lastSplit > 0 ? metadata[i].SQL_DESC_CONCISE_TYPE.substring(lastSplit + 1) : metadata[i].SQL_DESC_CONCISE_TYPE; + if (tablePrefix) { + dataType = _.get(this, `options.includeMap.${tablePrefix}.model.tableAttributes.${fieldName}.type`); + } else { + dataType = _.get(this.model, ['fieldRawAttributesMap', fieldName, 'type']); + } + coltypes[metadata[i].SQL_DESC_CONCISE_TYPE] = dataType && dataType.defaultType ? dataType.defaultType() : metadata[i].SQL_DESC_TYPE_NAME; + } + // coltypes[metadata[i].SQL_DESC_CONCISE_TYPE] = + // metadata[i].SQL_DESC_TYPE_NAME; } for (let i = 0; i < datalen; i++) { for (const column in data[i]) { @@ -480,7 +495,7 @@ class Query extends AbstractQuery { let id = null; let autoIncrementAttributeAlias = null; - if (this.model.rawAttributes.hasOwnProperty(autoIncrementAttribute) && + if (Object.prototype.hasOwnProperty.call(this.model.rawAttributes, autoIncrementAttribute) && this.model.rawAttributes[autoIncrementAttribute].field !== undefined) autoIncrementAttributeAlias = this.model.rawAttributes[autoIncrementAttribute].field; diff --git a/lib/hooks.js b/lib/hooks.js index 9d866be335b5..2599bf44f4cd 100644 --- a/lib/hooks.js +++ b/lib/hooks.js @@ -42,6 +42,8 @@ const hookTypes = { afterAssociate: { params: 2, sync: true }, beforeConnect: { params: 1, noModel: true }, afterConnect: { params: 2, noModel: true }, + beforeDisconnect: { params: 1, noModel: true }, + afterDisconnect: { params: 1, noModel: true }, beforeSync: { params: 1 }, afterSync: { params: 1 }, beforeBulkSync: { params: 1 }, @@ -501,11 +503,27 @@ exports.applyTo = applyTo; /** * A hook that is run after a connection is created * @param {string} name - * @param {Function} fn A callback function that is called with the connection object and thye config passed to connection + * @param {Function} fn A callback function that is called with the connection object and the config passed to connection * @name afterConnect * @memberof Sequelize */ +/** + * A hook that is run before a connection is disconnected + * @param {string} name + * @param {Function} fn A callback function that is called with the connection object + * @name beforeDisconnect + * @memberof Sequelize + */ + +/** + * A hook that is run after a connection is disconnected + * @param {string} name + * @param {Function} fn A callback function that is called with the connection object + * @name afterDisconnect + * @memberof Sequelize + */ + /** * A hook that is run before Model.sync call * @param {string} name diff --git a/lib/model.js b/lib/model.js index e742f4dc26c4..ab3af311d654 100644 --- a/lib/model.js +++ b/lib/model.js @@ -812,6 +812,12 @@ class Model { } }); } + // If we have a possible object/array to clone, we try it. + // Otherwise, we return the original value when it's not undefined, + // or the resulting object in that case. + if (srcValue) { + return Utils.cloneDeep(srcValue, true); + } return srcValue === undefined ? objValue : srcValue; } @@ -2011,13 +2017,8 @@ class Model { * * @returns {Promise} */ - static count(options) { + static count(options = {}) { return Promise.try(() => { - options = _.defaults(Utils.cloneDeep(options), { hooks: true }); - if (options.hooks) { - return this.runHooks('beforeCount', options); - } - }).then(() => { let col = options.col || '*'; if (options.include) { col = `${this.name}.${options.col || this.primaryKeyField}`; @@ -2027,12 +2028,11 @@ class Model { options.dataType = new DataTypes.INTEGER(); options.includeIgnoreAttributes = false; - // No limit, offset or order for the options max be given to count() - // Set them to null to prevent scopes setting those values + // No limit, offset or order for the options max be given to count() + // Set them to null to prevent scopes setting those values options.limit = null; options.offset = null; options.order = null; - return this.aggregate(col, 'count', options); }); } @@ -2222,7 +2222,7 @@ class Model { * The successful result of the promise will be (instance, built) * * @param {Object} options find options - * @param {Object} options.where A hash of search attributes. + * @param {Object} options.where A hash of search attributes. If `where` is a plain object it will be appended with defaults to build a new instance. * @param {Object} [options.defaults] Default values to use if building a new instance * @param {Object} [options.transaction] Transaction to run query under * @@ -2266,7 +2266,7 @@ class Model { * {@link Model.findAll} for a full specification of find and options * * @param {Object} options find and create options - * @param {Object} options.where where A hash of search attributes. + * @param {Object} options.where where A hash of search attributes. If `where` is a plain object it will be appended with defaults to build a new instance. * @param {Object} [options.defaults] Default values to use if creating a new instance * @param {Transaction} [options.transaction] Transaction to run query under * @@ -2376,7 +2376,7 @@ class Model { * {@link Model.findAll} for a full specification of find and options * * @param {Object} options find options - * @param {Object} options.where A hash of search attributes. + * @param {Object} options.where A hash of search attributes. If `where` is a plain object it will be appended with defaults to build a new instance. * @param {Object} [options.defaults] Default values to use if creating a new instance * * @returns {Promise} diff --git a/lib/query-interface.js b/lib/query-interface.js index 8fb875e42bb5..bf7667d3d06a 100644 --- a/lib/query-interface.js +++ b/lib/query-interface.js @@ -8,6 +8,7 @@ const SQLiteQueryInterface = require('./dialects/sqlite/query-interface'); const MSSQLQueryInterface = require('./dialects/mssql/query-interface'); const MySQLQueryInterface = require('./dialects/mysql/query-interface'); const PostgresQueryInterface = require('./dialects/postgres/query-interface'); +const Db2QueryInterface = require('./dialects/db2/query-interface'); const Transaction = require('./transaction'); const Promise = require('./promise'); const QueryTypes = require('./query-types'); @@ -1039,10 +1040,17 @@ class QueryInterface { options = Utils.cloneDeep(options); if (typeof identifier === 'object') identifier = Utils.cloneDeep(identifier); - const sql = this.QueryGenerator.updateQuery(tableName, values, identifier, options, attributes); const table = _.isObject(tableName) ? tableName : { tableName }; const model = _.find(this.sequelize.modelManager.models, { tableName: table.tableName }); + if (this.sequelize.options.dialect === 'db2' && Db2QueryInterface.isJsonFilter(model, options.where)) { + return this.hasJsonBulkOpertionToSelectQuer(tableName, options, model, filter => { + const sql = this.QueryGenerator.updateQuery(tableName, values, filter, options, attributes); + options.model = model; + return this.sequelize.query(sql, options); + }); + } + const sql = this.QueryGenerator.updateQuery(tableName, values, identifier, options, attributes); options.model = model; return this.sequelize.query(sql, options); } @@ -1110,6 +1118,14 @@ class QueryInterface { if (typeof identifier === 'object') where = Utils.cloneDeep(where); + if (this.sequelize.options.dialect === 'db2' && Db2QueryInterface.isJsonFilter(model, options.where)) { + return this.hasJsonBulkOpertionToSelectQuer(tableName, options, model, filter => { + return this.sequelize.query( + this.QueryGenerator.deleteQuery(tableName, filter, options, model), + options + ); + }); + } return this.sequelize.query( this.QueryGenerator.deleteQuery(tableName, where, options, model), options @@ -1117,13 +1133,35 @@ class QueryInterface { } select(model, tableName, options) { - options = Utils.cloneDeep(options); - options.type = QueryTypes.SELECT; - options.model = model; + const options0 = Utils.cloneDeep(options); + options0.type = QueryTypes.SELECT; + options0.model = model; + + if (this.sequelize.options.dialect === 'db2' && Db2QueryInterface.isJsonFilter(model, options0.where)) { + // db2 的 json 筛选通过在客户端进行 分批查询+数组filter 实现 + const options1 = Object.assign({ queryInterface: this }, options); + const filterFunc = Db2QueryInterface.generateClientSideFilterForJsonCond(tableName, options1, model); + + return Db2QueryInterface.batchQuery( + ({ dbOffset, dbLimit }) => { + const options2 = _.defaults({ offset: dbOffset, limit: dbLimit, plain: false }, options0); + return this.sequelize.query( + this.QueryGenerator.selectQuery(tableName, options2, model), + options2 + ); + }, + filterFunc, + options.offset, + options.limit + ) + .then(res => { + return options.plain ? res[0] || null : res; + }); + } return this.sequelize.query( - this.QueryGenerator.selectQuery(tableName, options, model), - options + this.QueryGenerator.selectQuery(tableName, options0, model), + options0 ); } @@ -1157,6 +1195,48 @@ class QueryInterface { type: QueryTypes.SELECT }); + if (this.sequelize.options.dialect === 'db2' && Db2QueryInterface.isJsonFilter(Model, options.where)) { + const operations1 = _.cloneDeep(options); + operations1.attributes = ['*']; + operations1.plain = false; + return this.hasJsonBulkOpertionToSelectQuer(tableName, operations1, Model, filter => { + options.where = filter; + const sql = this.QueryGenerator.selectQuery(tableName, options, Model); + + if (attributeSelector === undefined) { + throw new Error('Please pass an attribute selector!'); + } + return this.sequelize.query(sql, options).then(data => { + if (!options.plain) { + return data; + } + + const result = data ? data[attributeSelector] : null; + + if (!options || !options.dataType) { + return result; + } + + const dataType = options.dataType; + + if (dataType instanceof DataTypes.DECIMAL || dataType instanceof DataTypes.FLOAT) { + if (result !== null) { + return parseFloat(result); + } + } + if (dataType instanceof DataTypes.INTEGER || dataType instanceof DataTypes.BIGINT) { + return parseInt(result, 10); + } + if (dataType instanceof DataTypes.DATE) { + if (result !== null && !(result instanceof Date)) { + return new Date(result); + } + } + return result; + }); + }); + } + const sql = this.QueryGenerator.selectQuery(tableName, options, Model); if (attributeSelector === undefined) { @@ -1449,6 +1529,69 @@ class QueryInterface { return promise; } + + //将带有json的批量操作转成selectall生成主键filter + hasJsonBulkOpertionToSelectQuer(tableName, options, model, operation) { + const options1 = Object.assign({ queryInterface: this }, options); + const filterFunc = Db2QueryInterface.generateClientSideFilterForJsonCond(tableName, options1, model); + const options0 = Utils.cloneDeep(options); + options0.type = QueryTypes.SELECT; + options0.model = model; + + // db2 的 json 筛选通过在客户端进行 分批查询+数组filter 实现 + return Db2QueryInterface.batchQuery( + ({ dbOffset, dbLimit }) => { + const options2 = _.defaults({ offset: dbOffset, limit: dbLimit, plain: false }, options0); + return this.sequelize.query( + this.QueryGenerator.selectQuery(tableName, options2, model), + options2 + ); + }, + filterFunc, + options.offset, + options.limit + ) + .then(res => { + return options.plain ? res[0] || null : res; + }) + .then(data => { + const filter = Db2QueryInterface.getPrimaryKeyFilter(data, model.primaryKeyField); + return operation(filter); + }); + } + + /** + * count计数 + * + * @param {string} tableName + * @param {Object} options + * @param {Object} model + */ + count(tableName, options, model) { + let col = options.col || '*'; + if (options.include) { + col = `${this.name}.${options.col || this.primaryKeyField}`; + } + + options.plain = !options.group; + options.dataType = new DataTypes.INTEGER(); + options.includeIgnoreAttributes = false; + + // No limit, offset or order for the options max be given to count() + // Set them to null to prevent scopes setting those values + options.limit = null; + options.offset = null; + options.order = null; + + if (this.sequelize.options.dialect === 'db2' && Db2QueryInterface.isJsonFilter(model, options.where)) { + return this.hasJsonBulkOpertionToSelectQuer(tableName, options, model, filter => { + options.where = filter; + return options; + }); + } + + return { col, options }; + } } module.exports = QueryInterface; diff --git a/lib/sequelize.js b/lib/sequelize.js index 0f1c8d54bd63..4a2684ef64d8 100755 --- a/lib/sequelize.js +++ b/lib/sequelize.js @@ -167,7 +167,7 @@ class Sequelize { * @param {number} [options.retry.max] How many times a failing query is automatically retried. Set to 0 to disable retrying on SQL_BUSY error. * @param {boolean} [options.typeValidation=false] Run built in type validators on insert and update, e.g. validate that arguments passed to integer fields are integer-like. * @param {Object} [options.operatorsAliases] String based operator alias. Pass object to limit set of aliased operators. - * @param {Object} [options.hooks] An object of global hook functions that are called before and after certain lifecycle events. Global hooks will run after any model-specific hooks defined for the same event (See `Sequelize.Model.init()` for a list). Additionally, `beforeConnect()` and `afterConnect()` hooks may be defined here. + * @param {Object} [options.hooks] An object of global hook functions that are called before and after certain lifecycle events. Global hooks will run after any model-specific hooks defined for the same event (See `Sequelize.Model.init()` for a list). Additionally, `beforeConnect()`, `afterConnect()`, `beforeDisconnect()`, and `afterDisconnect()` hooks may be defined here. */ constructor(database, username, password, options) { let config; diff --git a/lib/sql-string.js b/lib/sql-string.js index 6f7dbc171662..f75a3f545b83 100644 --- a/lib/sql-string.js +++ b/lib/sql-string.js @@ -28,7 +28,7 @@ function escape(val, timeZone, dialect, format) { // SQLite doesn't have true/false support. MySQL aliases true/false to 1/0 // for us. Postgres actually has a boolean type with true/false literals, // but sequelize doesn't use it yet. - if (dialect === 'sqlite' || dialect === 'mssql') { + if (dialect === 'sqlite' || dialect === 'mssql' || dialect === 'db2') { return +!!val; } return (!!val).toString(); diff --git a/lib/utils.js b/lib/utils.js index e4f91f2cd294..04362254b7b9 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -123,7 +123,7 @@ function formatNamedParameters(sql, parameters, dialect) { } exports.formatNamedParameters = formatNamedParameters; -function cloneDeep(obj) { +function cloneDeep(obj, onlyPlain) { obj = obj || {}; return _.cloneDeepWith(obj, elem => { // Do not try to customize cloning of arrays or POJOs @@ -131,8 +131,9 @@ function cloneDeep(obj) { return undefined; } - // Don't clone stuff that's an object, but not a plain one - fx example sequelize models and instances - if (typeof elem === 'object') { + // If we specified to clone only plain objects & arrays, we ignore everyhing else + // In any case, don't clone stuff that's an object, but not a plain one - fx example sequelize models and instances + if (onlyPlain || typeof elem === 'object') { return elem; } diff --git a/package.json b/package.json index 49444b1f261b..199b0ef25da3 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,7 @@ { - "name": "sequelize", + "name": "sugo-sequelize", "description": "Multi dialect ORM for Node.JS", - "version": "0.0.0-development", - "author": "Sascha Depold ", - "contributors": [ - "Sascha Depold ", - "Jan Aagaard Meier ", - "Daniel Durante ", - "Mick Hansen ", - "Bimal Jha ", - "Sushant Dhiman " - ], + "version": "0.1.3", "repository": { "type": "git", "url": "https://github.com/sequelize/sequelize.git" @@ -31,6 +22,7 @@ ], "license": "MIT", "dependencies": { + "async-streamjs": "0.1.5", "bluebird": "^3.5.0", "cls-bluebird": "^2.1.0", "debug": "^4.1.1", diff --git a/test/integration/client-side-filter.test.js b/test/integration/client-side-filter.test.js new file mode 100644 index 000000000000..32d2c14beb5c --- /dev/null +++ b/test/integration/client-side-filter.test.js @@ -0,0 +1,700 @@ +'use strict'; + +const chai = require('chai'), + expect = chai.expect, + Support = require('../unit/support'), + Sequelize = Support.Sequelize, + _ = require('lodash'); + +describe('Client side filter', () => { + let sequelize, DB2Test; + const sampleData = _.range(50).map(i => ({ + id: `s${i + 1}`, + name: `slice_${i + 1}`, + visitCount: 10 + i, + price: _.round(5.2 + i * 5.2, 2), + is_private: i % 2 === 0, + params: _.pickBy({ + type: i % 3 === 0 ? 'line' : i % 3 === 1 ? 'bar' : 'pie', + dimensions: i % 2 === 0 ? ['event_name', 'duration'] : ['event_name'], + metrics: ['total'], + extraSettings: i % 5 === 0 + ? { timeZone: i % 3 === 0 ? 'Asia/Tokyo' : 'Asia/Shanghai', refreshInterval: 5 * i } + : undefined + }, _.identity) + })); + const needKeys = Object.keys(sampleData[0]); + const pickNeed = obj => _.pick(obj, needKeys); + + before(() => { + sequelize = Support.createSequelizeInstance({ + logging: console.log, + // 兼容旧版本调用方式 + operatorsAliases: _.mapKeys(Sequelize.Op, (v, k) => `$${k}`) + }); + DB2Test = sequelize.define('DB2Test2', { + id: { + type: Sequelize.STRING(32), + primaryKey: true + }, + name: Sequelize.STRING, + visitCount: Sequelize.INTEGER, + price: Sequelize.FLOAT, + is_private: Sequelize.BOOLEAN, + params: { + type: Sequelize.JSONB, + defaultValue: {} + } + }); + return DB2Test.sync({ force: true }); + }); + + after(() => { + }); + + describe('Insert test', () => { + it('should be create success', () => { + return DB2Test.create(sampleData[0]).then(slice1 => { + expect(sampleData[0]).to.deep.equal(pickNeed(slice1)); + }); + }); + + it('should be bulk create success', () => { + const records = _.drop(sampleData, 1); + return DB2Test.bulkCreate(records) + .then(() => { + return DB2Test.findAll({ + where: { + id: { $ne: sampleData[0].id } + }, + raw: true + }); + }) + .then(slices => { + expect(records).to.deep.equal(slices.map(pickNeed)); + }); + }); + }); + + describe('Non json find', () => { + it('should be find success(findOne)', () => { + return DB2Test.findOne({ + where: { + id: 's1' + }, + raw: true + }).then(slice1 => { + expect(sampleData[0]).to.deep.equal(pickNeed(slice1)); + }); + }); + + it('should be find success(findAll)', () => { + return DB2Test.findAll({ + where: { + id: sampleData[10].id + }, + raw: true + }).then(arr => { + expect(sampleData[10]).to.deep.equal(pickNeed(arr[0])); + }); + }); + + it('should be find nothing', () => { + return DB2Test.findOne({ + where: { + id: 's0' + }, + raw: true + }).then(slice1 => { + expect(true).to.equal(!slice1); + }); + }); + + it('should be find success(findAll gt)', () => { + const targets = _.filter(sampleData, s => s.visitCount > 30); + return DB2Test.findAll({ + where: { + visitCount: { $gt: 30 } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll lte)', () => { + const targets = _.filter(sampleData, s => s.visitCount <= 30); + return DB2Test.findAll({ + where: { + visitCount: { $lte: 30 } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll like)', () => { + const targets = _.filter(sampleData, s => _.startsWith(s.name, 'slice_1')); + return DB2Test.findAll({ + where: { + name: { $like: 'slice_1%' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll not like)', () => { + const targets = _.filter(sampleData, s => !_.startsWith(s.name, 'slice_1')); + return DB2Test.findAll({ + where: { + name: { + $notLike: 'slice_1%' + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll iLike)', () => { + const targets = _.filter(sampleData, s => _.startsWith(s.name, 'slice_1')); + return DB2Test.findAll({ + where: { + name: { $iLike: 'SLICE_1%' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll in)', () => { + const targets = _.filter(sampleData, s => s.name === 'slice_1' || s.name === 'slice_3'); + return DB2Test.findAll({ + where: { + name: { $in: ['slice_1', 'slice_3'] } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll ne)', () => { + const targets = _.filter(sampleData, s => s.name !== 'slice_1'); + return DB2Test.findAll({ + where: { + name: { $ne: 'slice_1' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll orderBy)', () => { + const targets = _.orderBy(sampleData, s => s.visitCount, 'desc'); + return DB2Test.findAll({ + order: [['visitCount', 'desc']], + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll offset limit)', () => { + const targets = _(sampleData).drop(5).take(5).value(); + return DB2Test.findAll({ + offset: 5, + limit: 5, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll or)', () => { + const targets = _.filter(sampleData, s => s.id === 's2' || s.name === 'slice_3'); + return DB2Test.findAll({ + where: { + $or: [{ id: 's2' }, { name: 'slice_3' }] + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll name or json)', () => { + const targets = _.filter(sampleData, s => s.params.type === 'pie' || s.name === 'slice_3'); + return DB2Test.findAll({ + where: { + $or: [{ name: 'slice_3' }, { params: { type: 'pie' } }] + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll or array)', () => { + const targets = _.filter(sampleData, s => s.visitCount === 10 || s.visitCount === 20); + return DB2Test.findAll({ + where: { + visitCount: { + $or: [10, 20] + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll and)', () => { + const targets = _.filter(sampleData, s => s.id === 's2' || s.name === 'slice_2'); + return DB2Test.findAll({ + where: { + $and: { + id: 's2', + name: 'slice_2' + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll not)', () => { + const targets = _.filter(sampleData, s => !(s.visitCount <= 50 && 6 <= s.price)); + return DB2Test.findAll({ + where: { + $not: { + visitCount: { $lte: 50 }, + price: { $gte: 6 } + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + }); + + describe('Json query', () => { + it('should be find success(json sub query)', () => { + const target = _.find(sampleData, s => _.get(s, 'params.type') === 'pie'); + return DB2Test.findOne({ + where: { + params: { type: 'pie' } + }, + raw: true + }).then(slice1 => { + expect(pickNeed(slice1)).to.deep.equal(target); + }); + }); + + it('should be find success(mix json filter with non json filter)', () => { + const targets = _.filter(sampleData, s => s.id === 's3' && _.get(s, 'params.type') === 'pie'); + return DB2Test.findAll({ + where: { + id: 's3', + params: { type: 'pie' } + }, + raw: true + }).then(slices => { + expect(targets).to.deep.equal(slices.map(pickNeed)); + }); + }); + + it('should be find success(json sub query with nested key)', () => { + const target = _.find(sampleData, s => _.get(s, 'params.type') === 'bar'); + return DB2Test.findOne({ + where: { + 'params.type': 'bar' + }, + raw: true + }).then(slice1 => { + expect(pickNeed(slice1)).to.deep.equal(target); + }); + }); + + it('should be find nothing(json sub query)', () => { + return DB2Test.findOne({ + where: { + params: { type: 'line1' } + }, + raw: true + }).then(slice1 => { + expect(!slice1).to.equal(true); + }); + }); + + it('should be find nothing(json sub query with nested key)', () => { + return DB2Test.findOne({ + where: { + 'params.type': 'line1' + }, + raw: true + }).then(slice1 => { + expect(!slice1).to.equal(true); + }); + }); + + it('should be find nothing(mix json filter with non json filter)', () => { + return DB2Test.findOne({ + where: { + id: 's1', + 'params.type': 'line1' + }, + raw: true + }).then(slice1 => { + expect(true).to.equal(!slice1); + }); + }); + + it('should be find success(json sub query)', () => { + const targets = _.filter(sampleData, s => _.get(s, 'params.type') === 'pie'); + return DB2Test.findAll({ + where: { + params: { type: 'pie' } + }, + raw: true + }).then(slices => { + expect(slices.map(pickNeed)).to.deep.equal(targets); + }); + }); + + it('should be find success(json sub query with nested key)', () => { + const targets = _.filter(sampleData, s => _.get(s, 'params.type') === 'bar'); + return DB2Test.findAll({ + where: { + 'params.type': 'bar' + }, + raw: true + }).then(slices => { + expect(slices.map(pickNeed)).to.deep.equal(targets); + }); + }); + + it('should be find nothing(json sub query)', () => { + return DB2Test.findAll({ + where: { + params: { type: 'pie0' } + }, + raw: true + }).then(slices => { + expect(slices).to.deep.equal([]); + }); + }); + + it('should be find nothing(json sub query with nested key)', () => { + return DB2Test.findAll({ + where: { + 'params.type': 'bar0' + }, + raw: true + }).then(slices => { + expect(slices).to.deep.equal([]); + }); + }); + + it('should be find success(findAll gt)', () => { + const targets = _.filter(sampleData, s => { + const refreshInterval = _.get(s, 'params.extraSettings.refreshInterval'); + return _.isNumber(refreshInterval) && 30 < refreshInterval; + }); + return DB2Test.findAll({ + where: { + 'params.extraSettings.refreshInterval': { $gt: 30 } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll lte)', () => { + const targets = _.filter(sampleData, s => { + const refreshInterval = _.get(s, 'params.extraSettings.refreshInterval'); + return _.isNumber(refreshInterval) && refreshInterval <= 30; + }); + return DB2Test.findAll({ + where: { + 'params.extraSettings.refreshInterval': { $lte: 30 } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll like)', () => { + const targets = _.filter(sampleData, s => _.startsWith(s.params.type, 'pi')); + return DB2Test.findAll({ + where: { + 'params.type': { $like: 'pi%' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll not like)', () => { + const targets = _.filter(sampleData, s => !_.startsWith(s.params.type, 'pi')); + return DB2Test.findAll({ + where: { + 'params.type': { $notLike: 'pi%' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll iLike)', () => { + const targets = _.filter(sampleData, s => _.startsWith(s.params.type, 'pi')); + return DB2Test.findAll({ + where: { + 'params.type': { $iLike: 'PI%' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll in)', () => { + const targets = _.filter(sampleData, s => s.params.type === 'line' || s.params.type === 'pie'); + return DB2Test.findAll({ + where: { + 'params.type': { $in: ['line', 'pie'] } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll ne)', () => { + const targets = _.filter(sampleData, s => s.params.type !== 'bar'); + return DB2Test.findAll({ + where: { + 'params.type': { $ne: 'bar' } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + // TODO + it.skip('should be find success(findAll json orderBy)', () => { + const targets = _.orderBy(sampleData, s => s.params.type, 'desc'); + return DB2Test.findAll({ + order: [['params.type', 'desc']], + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(json sub query with offset limit)', () => { + const targets = _(sampleData).filter(s => _.get(s, 'params.type') === 'pie').drop(5).take(5).value(); + return DB2Test.findAll({ + where: { + params: { type: 'pie' } + }, + offset: 5, + limit: 5, + raw: true + }).then(slices => { + expect(slices.map(pickNeed)).to.deep.equal(targets); + }); + }); + + it('should be find success(findAll or)', () => { + const targets = _.filter(sampleData, s => { + const type = _.get(s, 'params.type'); + return type === 'line' || type === 'pie'; + }); + return DB2Test.findAll({ + where: { + $or: [{ 'params.type': 'line' }, { 'params.type': 'pie' }] + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll or array)', () => { + const targets = _.filter(sampleData, s => { + const type = _.get(s, 'params.extraSettings.refreshInterval'); + return type === 25 || type === 50; + }); + return DB2Test.findAll({ + where: { + 'params.extraSettings.refreshInterval': { + $or: [25, 50] + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll and)', () => { + const targets = _.filter(sampleData, s => { + return _.get(s, 'params.type') === 'bar' + && _.get(s, 'params.extraSettings.refreshInterval') === 20; + }); + return DB2Test.findAll({ + where: { + $and: { + 'params.type': 'bar', + 'params.extraSettings.refreshInterval': 20 + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + + it('should be find success(findAll not)', () => { + const targets = _.filter(sampleData, s => { + return !(_.get(s, 'params.type') === 'pie' + && _.get(s, 'params.extraSettings.refreshInterval') === 25); + }); + return DB2Test.findAll({ + where: { + $not: { + 'params.type': 'pie', + 'params.extraSettings.refreshInterval': 25 + } + }, + raw: true + }).then(arr => { + expect(targets).to.deep.equal(arr.map(pickNeed)); + }); + }); + }); + + describe('findAndCountAll ', () => { + it('findAndCountAll by pk', () => { + return DB2Test.findAndCountAll({ + where: { + id: 's1' + } + }).then(res => { + console.log('findAndCountAll by pk ====>', res.count); + }); + }); + + it('findAndCountAll by bl', () => { + return DB2Test.findAndCountAll({ + where: { + is_private: true + } + }).then(res => { + console.log('findAndCountAll by bl ====>', res.count); + }); + }); + + it('findAndCountAll by json', () => { + return DB2Test.findAndCountAll({ + where: { + params: { type: 'bar' } + }, + raw: true + }).then(res => { + console.log('findAndCountAll by json ====>', res.count); + }); + }); + // TODO gt lte like iLike in ne orderby limit or and not + }); + + describe('update test', () => { + it('update by pk', () => { + return DB2Test.update({ + price: 999.99 + }, { + where: { + id: 's1' + } + }).then(res => { + console.log('update by pk', res); + }); + }); + + it('update by bl', () => { + return DB2Test.update({ + is_private: false + }, { + where: { + is_private: true + } + }).then(res => { + console.log('update by bl', res); + }); + }); + + it('update by json', () => { + return DB2Test.update({ + params: { type: 'bar999' } + }, { + where: { + params: { type: 'bar' } + }, + raw: true + }).then(res => { + console.log('update by json', res); + }); + }); + // TODO gt lte like iLike in ne orderby limit or and not + }); + + + describe('delete test', () => { + it('delete by pk', () => { + return DB2Test.destroy({ + where: { + id: 's1' + } + }).then(res => { + console.log('delete by pk ===>', res ); + }); + }); + + it('delete by bl', () => { + return DB2Test.destroy({ + where: { + is_private: true + } + }).then(res => { + console.log('delete by bl ===>', res ); + }); + }); + + it('delete by json', () => { + return DB2Test.destroy({ + where: { + params: { type: 'bar999' } + } + }).then(res => { + console.log('delete by json ===>', res ); + }); + }); + // TODO gt lte like iLike in ne orderby limit or and not + }); + +}); diff --git a/test/integration/dialects/sqlite/dao-factory.test.js b/test/integration/dialects/sqlite/dao-factory.test.js index c55e1c995217..0c127cb7c04b 100644 --- a/test/integration/dialects/sqlite/dao-factory.test.js +++ b/test/integration/dialects/sqlite/dao-factory.test.js @@ -110,7 +110,7 @@ if (dialect === 'sqlite') { }); }); - it.skip('should make aliased attributes available', function() { + it('should make aliased attributes available', function() { return this.User.findOne({ where: { name: 'user' }, attributes: ['id', ['name', 'username']] diff --git a/test/unit/connection-manager.test.js b/test/unit/connection-manager.test.js index b9895a597c48..a89d2d2c415a 100644 --- a/test/unit/connection-manager.test.js +++ b/test/unit/connection-manager.test.js @@ -66,4 +66,42 @@ describe('connection manager', () => { }); }); }); + + describe('_disconnect', () => { + beforeEach(function() { + this.connection = {}; + + this.dialect = { + connectionManager: { + disconnect: sinon.stub().resolves(this.connection) + } + }; + + this.sequelize = Support.createSequelizeInstance(); + }); + + it('should call beforeDisconnect', function() { + const spy = sinon.spy(); + this.sequelize.beforeDisconnect(spy); + + const connectionManager = new ConnectionManager(this.dialect, this.sequelize); + + return connectionManager._disconnect(this.connection).then(() => { + expect(spy.callCount).to.equal(1); + expect(spy.firstCall.args[0]).to.equal(this.connection); + }); + }); + + it('should call afterDisconnect', function() { + const spy = sinon.spy(); + this.sequelize.afterDisconnect(spy); + + const connectionManager = new ConnectionManager(this.dialect, this.sequelize); + + return connectionManager._disconnect(this.connection).then(() => { + expect(spy.callCount).to.equal(1); + expect(spy.firstCall.args[0]).to.equal(this.connection); + }); + }); + }); }); diff --git a/test/unit/hooks.test.js b/test/unit/hooks.test.js index df129a067a58..5e5d60ab466e 100644 --- a/test/unit/hooks.test.js +++ b/test/unit/hooks.test.js @@ -15,7 +15,7 @@ describe(Support.getTestDialectTeaser('Hooks'), () => { }); it('does not expose non-model hooks', function() { - for (const badHook of ['beforeDefine', 'afterDefine', 'beforeConnect', 'afterConnect', 'beforeInit', 'afterInit']) { + for (const badHook of ['beforeDefine', 'afterDefine', 'beforeConnect', 'afterConnect', 'beforeDisconnect', 'afterDisconnect', 'beforeInit', 'afterInit']) { expect(this.Model).to.not.have.property(badHook); } }); diff --git a/test/unit/model/scope.test.js b/test/unit/model/scope.test.js index aaed30912ea0..87e471a773fb 100644 --- a/test/unit/model/scope.test.js +++ b/test/unit/model/scope.test.js @@ -112,6 +112,10 @@ describe(Support.getTestDialectTeaser('Model'), () => { it('should unite attributes with array', () => { expect(User.scope('aScope', 'defaultScope')._scope.attributes).to.deep.equal({ exclude: ['value', 'password'] }); }); + + it('should not modify the original scopes when merging them', () => { + expect(User.scope('defaultScope', 'aScope').options.defaultScope.attributes).to.deep.equal({ exclude: ['password'] }); + }); }); it('defaultScope should be an empty object if not overridden', () => { diff --git a/test/unit/sql/create-table.test.js b/test/unit/sql/create-table.test.js index b946de090c3b..c74d2adc0ed2 100644 --- a/test/unit/sql/create-table.test.js +++ b/test/unit/sql/create-table.test.js @@ -110,31 +110,6 @@ describe(Support.getTestDialectTeaser('SQL'), () => { }); }); }); - - describe('Attempt to use different lodash template settings', () => { - before(() => { - // make handlebars - _.templateSettings.evaluate = /{{([\s\S]+?)}}/g; - _.templateSettings.interpolate = /{{=([\s\S]+?)}}/g; - _.templateSettings.escape = /{{-([\s\S]+?)}}/g; - }); - - after(() => { - // reset - const __ = require('lodash').runInContext(); - _.templateSettings.evaluate = __.templateSettings.evaluate; - _.templateSettings.interpolate = __.templateSettings.interpolate; - _.templateSettings.escape = __.templateSettings.escape; - }); - - it('it should be a okay!', () => { - expectsql(sql.createTableQuery(FooUser.getTableName(), sql.attributesToSQL(FooUser.rawAttributes), { - comment: 'This is a test of the lodash template settings.' - }), { - postgres: 'CREATE TABLE IF NOT EXISTS "foo"."users" ("id" SERIAL , "mood" "foo"."enum_users_mood", PRIMARY KEY ("id")); COMMENT ON TABLE "foo"."users" IS \'This is a test of the lodash template settings.\';' - }); - }); - }); } }); }); diff --git a/types/lib/hooks.d.ts b/types/lib/hooks.d.ts index c6f96c7c8ded..cc49ae9cdcd7 100644 --- a/types/lib/hooks.d.ts +++ b/types/lib/hooks.d.ts @@ -54,6 +54,8 @@ export interface SequelizeHooks extends ModelHooks { afterInit(sequelize: Sequelize): void; beforeConnect(config: Config): HookReturn; afterConnect(connection: unknown, config: Config): HookReturn; + beforeDisconnect(connection: unknown): HookReturn; + afterDisconnect(connection: unknown): HookReturn; } /** diff --git a/types/lib/model.d.ts b/types/lib/model.d.ts index 72b8141f6876..4da9a61612e4 100644 --- a/types/lib/model.d.ts +++ b/types/lib/model.d.ts @@ -121,7 +121,7 @@ export interface ScopeOptions { /** * The type accepted by every `where` option */ -export type WhereOptions = WhereAttributeHash | AndOperator | OrOperator | Where; +export type WhereOptions = WhereAttributeHash | AndOperator | OrOperator | Literal | Where; /** * Example: `[Op.any]: [2,3]` becomes `ANY ARRAY[2, 3]::INTEGER` @@ -376,7 +376,7 @@ export interface IncludeThroughOptions extends Filterable, Projectable {} /** * Options for eager-loading associated models, also allowing for all associations to be loaded at once */ -export type Includeable = typeof Model | Association | IncludeOptions | { all: true }; +export type Includeable = typeof Model | Association | IncludeOptions | { all: true } | string; /** * Complex include options @@ -450,14 +450,22 @@ export interface IncludeOptions extends Filterable, Projectable, Paranoid { subQuery?: boolean; } +type OrderItemModel = typeof Model | { model: typeof Model; as: string } | string +type OrderItemColumn = string | Col | Fn | Literal export type OrderItem = | string | Fn | Col | Literal - | [string | Col | Fn | Literal, string] - | [typeof Model | { model: typeof Model; as: string }, string, string] - | [typeof Model, typeof Model, string, string]; + | [OrderItemColumn, string] + | [OrderItemModel, OrderItemColumn] + | [OrderItemModel, OrderItemColumn, string] + | [OrderItemModel, OrderItemModel, OrderItemColumn] + | [OrderItemModel, OrderItemModel, OrderItemColumn, string] + | [OrderItemModel, OrderItemModel, OrderItemModel, OrderItemColumn] + | [OrderItemModel, OrderItemModel, OrderItemModel, OrderItemColumn, string] + | [OrderItemModel, OrderItemModel, OrderItemModel, OrderItemModel, OrderItemColumn] + | [OrderItemModel, OrderItemModel, OrderItemModel, OrderItemModel, OrderItemColumn, string] export type Order = string | Fn | Col | Literal | OrderItem[]; /** @@ -863,6 +871,11 @@ export interface UpdateOptions extends Logging, Transactionable { * How many rows to update (only for mysql and mariadb) */ limit?: number; + + /** + * If true, the updatedAt timestamp will not be updated. + */ + silent?: boolean; } /** diff --git a/types/lib/sequelize.d.ts b/types/lib/sequelize.d.ts index a189c315c9cb..f23b00a64d23 100644 --- a/types/lib/sequelize.d.ts +++ b/types/lib/sequelize.d.ts @@ -57,13 +57,13 @@ export interface SyncOptions extends Logging { * The schema that the tables should be created in. This can be overridden for each table in sequelize.define */ schema?: string; - - /** + + /** * An optional parameter to specify the schema search_path (Postgres only) */ searchPath?: string; - - /** + + /** * If hooks is true then beforeSync, afterSync, beforeBulkSync, afterBulkSync hooks will be called */ hooks?: boolean; @@ -621,6 +621,24 @@ export class Sequelize extends Hooks { public static afterConnect(name: string, fn: (connection: unknown, options: Config) => void): void; public static afterConnect(fn: (connection: unknown, options: Config) => void): void; + /** + * A hook that is run before a connection is released + * + * @param name + * @param fn A callback function that is called with options + */ + public static beforeDisconnect(name: string, fn: (connection: unknown) => void): void; + public static beforeDisconnect(fn: (connection: unknown) => void): void; + + /** + * A hook that is run after a connection is released + * + * @param name + * @param fn A callback function that is called with options + */ + public static afterDisconnect(name: string, fn: (connection: unknown) => void): void; + public static afterDisconnect(fn: (connection: unknown) => void): void; + /** * A hook that is run before a find (select) query, after any { include: {all: ...} } options are expanded * diff --git a/types/test/include.ts b/types/test/include.ts index 22f633bcb463..efeed55273ee 100644 --- a/types/test/include.ts +++ b/types/test/include.ts @@ -17,7 +17,7 @@ MyModel.findAll({ on: { a: 1, }, - order: [['id', 'DESC']], + order: [['id', 'DESC'], [ 'AssociatedModel', MyModel, 'id', 'DESC' ], [ MyModel, 'id' ] ], separate: true, where: { state: Sequelize.col('project.state') }, }, @@ -32,7 +32,7 @@ MyModel.findAll({ include: [{ limit: 1, association: 'relation', - order: [['id', 'DESC']], + order: [['id', 'DESC'], 'id', [ AssociatedModel, MyModel, 'id', 'ASC' ]], separate: true, where: { state: Sequelize.col('project.state') }, }] diff --git a/types/test/model.ts b/types/test/model.ts index 4cd83f9fa303..c6cff023b667 100644 --- a/types/test/model.ts +++ b/types/test/model.ts @@ -23,6 +23,10 @@ MyModel.findOne({ ] }); +MyModel.hasOne(OtherModel, { as: 'OtherModelAlias' }); + +MyModel.findOne({ include: ['OtherModelAlias'] }); + const sequelize = new Sequelize('mysql://user:user@localhost:3306/mydb'); MyModel.init({}, { diff --git a/types/test/usage.ts b/types/test/usage.ts index 06c2e7d664dc..0f7598ae5a30 100644 --- a/types/test/usage.ts +++ b/types/test/usage.ts @@ -15,6 +15,15 @@ async function test(): Promise { user = await User.findOne(); + if (!user) { + return; + } + + user.update({}, {}); + user.update({}, { + silent: true + }); + const user2 = await User.create({ firstName: 'John', groupId: 1 }); await User.findAndCountAll({ distinct: true }); diff --git a/types/test/where.ts b/types/test/where.ts index 0906aa99c824..e62581127b28 100644 --- a/types/test/where.ts +++ b/types/test/where.ts @@ -1,4 +1,4 @@ -import { AndOperator, fn, Model, Op, OrOperator, Sequelize, WhereOperators, WhereOptions, where as whereFn } from 'sequelize'; +import { AndOperator, fn, Model, Op, OrOperator, Sequelize, WhereOperators, WhereOptions, literal, where as whereFn } from 'sequelize'; import Transaction from '../lib/transaction'; class MyModel extends Model { @@ -266,6 +266,13 @@ where = whereFn('test', { [Op.gt]: new Date(), }); +// Literal as where +where = literal('true') + +MyModel.findAll({ + where: literal('true') +}) + // Where as having option MyModel.findAll({ having: where