diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index d1b1fd83b..927698ba6 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -1,3 +1,4 @@ +import { Document } from 'bson'; export enum TYPE { String = 'String', Number = 'Number', @@ -22,25 +23,6 @@ export enum SQLDataType { TIMESTAMP = 'TIMESTAMP', } -export enum MongoIndexType { - Ascending = 1, - Descending = -1, - GeoSpatial2d = '2d', - GeoSpatial2dSphere = '2dsphere', - GeoHaystack = 'geoHaystack', - Hashed = 'hashed', - Text = 'text', -} - -export enum PostgresIndexType { - BTREE = 'BTREE', - HASH = 'HASH', - GIST = 'GIST', - SPGIST = 'SPGIST', - GIN = 'GIN', - BRIN = 'BRIN', -} - export type Array = any[]; type BaseConduitModelField = { @@ -158,28 +140,72 @@ export interface ConduitSchemaOptions { enabled: boolean; }; }; - indexes?: ModelOptionsIndexes[]; + indexes?: ModelOptionsIndex[]; } +// Index types export interface SchemaFieldIndex { - type?: MongoIndexType | PostgresIndexType; - options?: MongoIndexOptions | PostgresIndexOptions; + name: string; + type?: MongoIndexType | SequelizeIndexType; + options?: MongoIndexOptions | SequelizeIndexOptions; +} +export interface ModelOptionsIndex { + name: string; + fields: string[]; + types?: MongoIndexType[] | SequelizeIndexType[]; + options?: MongoIndexOptions | SequelizeIndexOptions; [field: string]: any; } -export interface ModelOptionsIndexes { - fields: string[]; - types?: MongoIndexType[] | PostgresIndexType; - options?: MongoIndexOptions | PostgresIndexOptions; +export enum MongoIndexType { + Ascending = 1, + Descending = -1, + GeoSpatial2d = '2d', + GeoSpatial2dSphere = '2dsphere', + GeoHaystack = 'geoHaystack', + Hashed = 'hashed', + Text = 'text', +} - [field: string]: any; +export const ReverseMongoIndexTypeMap: { + [key: string]: string; +} = { + '1': 'Ascending', + '-1': 'Descending', + '2d': 'GeoSpatial2d', + '2dsphere': 'GeoSpatial2dSphere', + geoHaystack: 'GeoHaystack', + hashed: 'Hashed', + text: 'Text', +}; + +export enum PgIndexType { + BTREE = 'BTREE', + HASH = 'HASH', + GIST = 'GIST', + SPGIST = 'SPGIST', + GIN = 'GIN', + BRIN = 'BRIN', } +export enum MySQLMariaDBIndexType { + BTREE = 'BTREE', + HASH = 'HASH', + UNIQUE = 'UNIQUE', + FULLTEXT = 'FULLTEXT', + SPATIAL = 'SPATIAL', +} + +export enum SQLiteIndexType { + BTREE = 'BTREE', +} + +export type SequelizeIndexType = PgIndexType | MySQLMariaDBIndexType | SQLiteIndexType; + export interface MongoIndexOptions { background?: boolean; unique?: boolean; - name?: string; partialFilterExpression?: Document; sparse?: boolean; expireAfterSeconds?: number; @@ -199,15 +225,33 @@ export interface MongoIndexOptions { hidden?: boolean; } -export interface PostgresIndexOptions { - concurrently?: boolean; - name?: string; - operator?: string; - parser?: null | string; - prefix?: string; +export interface SequelizeIndexOptions { + parser?: string | null; unique?: boolean; - using?: PostgresIndexType; + // Used instead of ModelOptionsIndexes fields for more complex index definitions + fields?: { + name: string; + length?: number; + order?: 'ASC' | 'DESC'; + collate?: string; + operator?: string; + }[]; where?: { [opt: string]: any; }; + prefix?: string; +} + +export interface PgIndexOptions extends SequelizeIndexOptions { + concurrently?: boolean; + operator?: string; + using?: PgIndexType; +} + +export interface MySQLMariaDBIndexOptions extends SequelizeIndexOptions { + type?: MySQLMariaDBIndexType; +} + +export interface SQLiteIndexOptions extends SequelizeIndexOptions { + using?: SQLiteIndexType; } diff --git a/modules/authorization/src/controllers/permissions.controller.ts b/modules/authorization/src/controllers/permissions.controller.ts index 7088a217a..5846c6fbd 100644 --- a/modules/authorization/src/controllers/permissions.controller.ts +++ b/modules/authorization/src/controllers/permissions.controller.ts @@ -241,7 +241,7 @@ export class PermissionsController { }, ], sqlQuery: - dbType === 'PostgreSQL' + dbType === 'postgres' ? getPostgresAccessListQuery( objectTypeCollection, computedTuple, diff --git a/modules/authorization/src/models/ActorIndex.schema.ts b/modules/authorization/src/models/ActorIndex.schema.ts index 396ca3f24..7bbd6d669 100644 --- a/modules/authorization/src/models/ActorIndex.schema.ts +++ b/modules/authorization/src/models/ActorIndex.schema.ts @@ -22,6 +22,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, @@ -40,6 +41,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'entity_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/authorization/src/models/ObjectIndex.schema.ts b/modules/authorization/src/models/ObjectIndex.schema.ts index f3de35ad4..087235007 100644 --- a/modules/authorization/src/models/ObjectIndex.schema.ts +++ b/modules/authorization/src/models/ObjectIndex.schema.ts @@ -22,6 +22,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, @@ -46,6 +47,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'entity_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/authorization/src/models/Permission.schema.ts b/modules/authorization/src/models/Permission.schema.ts index c7cdef5f9..90622ff0f 100644 --- a/modules/authorization/src/models/Permission.schema.ts +++ b/modules/authorization/src/models/Permission.schema.ts @@ -18,6 +18,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'resource_1', type: MongoIndexType.Ascending, }, }, @@ -37,6 +38,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/authorization/src/models/Relationship.schema.ts b/modules/authorization/src/models/Relationship.schema.ts index d378d4d48..9ee6c5673 100644 --- a/modules/authorization/src/models/Relationship.schema.ts +++ b/modules/authorization/src/models/Relationship.schema.ts @@ -17,6 +17,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'resource_1', type: MongoIndexType.Ascending, }, }, @@ -36,6 +37,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index e2379c4eb..fd1cbcc31 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -3,7 +3,7 @@ import { ConduitModel, ConduitSchema, GrpcError, - ModelOptionsIndexes, + ModelOptionsIndex, RawMongoQuery, RawSQLQuery, TYPE, @@ -171,11 +171,11 @@ export abstract class DatabaseAdapter { abstract createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: ModelOptionsIndex[], callerModule: string, ): Promise; - abstract getIndexes(schemaName: string): Promise; + abstract getIndexes(schemaName: string): Promise; abstract createView( modelName: string, diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index b8c62b43c..e358b83f8 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -1,13 +1,15 @@ import { Schema } from 'mongoose'; import { + ConduitGrpcSdk, ConduitModelField, ConduitSchema, + ModelOptionsIndex, MongoIndexType, SchemaFieldIndex, } from '@conduitplatform/grpc-sdk'; import { cloneDeep, isArray, isNil, isObject } from 'lodash-es'; import { checkIfMongoOptions } from './utils.js'; - +import { ConduitDatabaseSchema } from '../../interfaces/index.js'; import * as deepdash from 'deepdash-es/standalone'; /** @@ -15,14 +17,14 @@ import * as deepdash from 'deepdash-es/standalone'; * @param jsonSchema */ export function schemaConverter(jsonSchema: ConduitSchema) { - let copy = cloneDeep(jsonSchema); + const copy = cloneDeep(jsonSchema); if (copy.fields.hasOwnProperty('_id')) { delete copy.fields['_id']; } - copy = convertSchemaFieldIndexes(copy); + convertSchemaFieldIndexes(copy); deepdash.eachDeep(copy.fields, convert); if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy.modelOptions.indexes = convertModelOptionsIndexes(copy); } iterDeep(copy.fields); return copy; @@ -71,7 +73,7 @@ function convert(value: any, key: any, parentValue: any) { _id: false, timestamps: false, }); - parentValue[key] = schemaConverter(typeSchema).fields; + parentValue[key] = schemaConverter(typeSchema as ConduitDatabaseSchema).fields; return true; } @@ -97,17 +99,24 @@ function convert(value: any, key: any, parentValue: any) { } function convertSchemaFieldIndexes(copy: ConduitSchema) { - for (const field of Object.entries(copy.fields)) { - const index = (field[1] as ConduitModelField).index; + for (const [, fieldObj] of Object.entries(copy.fields)) { + const index = (fieldObj as ConduitModelField).index; if (!index) continue; - const type = index.type; - const options = index.options; - if (type && !Object.values(MongoIndexType).includes(type)) { - throw new Error('Incorrect index type for MongoDB'); + const { type, options } = index; + if (type && !(type in MongoIndexType)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in: ${copy.name}. Index ignored`, + ); + delete (fieldObj as ConduitModelField).index; + continue; } if (options) { if (!checkIfMongoOptions(options)) { - throw new Error('Incorrect index options for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in: ${copy.name}. Index ignored`, + ); + delete (fieldObj as ConduitModelField).index; + continue; } for (const [option, optionValue] of Object.entries(options)) { index[option as keyof SchemaFieldIndex] = optionValue; @@ -115,40 +124,56 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { delete index.options; } } - return copy; } function convertModelOptionsIndexes(copy: ConduitSchema) { + const convertedIndexes: ModelOptionsIndex[] = []; + for (const index of copy.modelOptions.indexes!) { - // compound indexes are maintained in modelOptions in order to be created after schema creation - // single field index => add it to specified schema field - if (index.fields.length !== 1) continue; - const modelField = copy.fields[index.fields[0]] as ConduitModelField; - if (!modelField) { - throw new Error(`Field ${modelField} in index definition doesn't exist`); + const { name, fields, types, options } = index; + if (fields.length === 0) { + throw new Error('Undefined fields for index creation'); + } + if (fields.some(field => !Object.keys(copy.fields).includes(field))) { + throw new Error(`Invalid fields for index creation`); + } + // Compound indexes are maintained in modelOptions in order to be created after schema creation + // Single field index are added to specified schema field + if (fields.length !== 1) { + convertedIndexes.push(index); + continue; } - if (index.types) { + if (types) { + const indexField = index.fields[0]; if ( - !isArray(index.types) || - !Object.values(MongoIndexType).includes(index.types[0]) || - index.fields.length !== index.types.length + !isArray(types) || + !Object.values(MongoIndexType).includes(types[0]) || + fields.length !== types.length ) { - throw new Error('Invalid index type for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in: ${copy.name}. Index ignored`, + ); + continue; } - const type = index.types[0] as MongoIndexType; - modelField.index = { - type: type, + (copy.fields[indexField] as ConduitModelField).index = { + name, + type: types[0] as MongoIndexType, }; } - if (index.options) { - if (!checkIfMongoOptions(index.options)) { - throw new Error('Incorrect index options for MongoDB'); + if (options) { + if (!checkIfMongoOptions(options)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in: ${copy.name}. Index ignored`, + ); + continue; } - for (const [option, optionValue] of Object.entries(index.options)) { - modelField.index![option as keyof SchemaFieldIndex] = optionValue; + for (const [option, optionValue] of Object.entries(options)) { + (copy.fields[index.fields[0]] as ConduitModelField).index![ + option as keyof SchemaFieldIndex + ] = optionValue; } } - copy.modelOptions.indexes!.splice(copy.modelOptions.indexes!.indexOf(index), 1); + convertedIndexes.push(index); } - return copy; + return convertedIndexes; } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 3b46dd180..ff19561a1 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -3,15 +3,21 @@ import { MongooseSchema } from './MongooseSchema.js'; import { schemaConverter } from './SchemaConverter.js'; import { ConduitGrpcSdk, + ConduitModelField, ConduitSchema, GrpcError, Indexable, - ModelOptionsIndexes, + ModelOptionsIndex, MongoIndexType, RawMongoQuery, + ReverseMongoIndexTypeMap, } from '@conduitplatform/grpc-sdk'; import { DatabaseAdapter } from '../DatabaseAdapter.js'; -import { validateFieldChanges, validateFieldConstraints } from '../utils/index.js'; +import { + findAndRemoveIndex, + validateFieldChanges, + validateFieldConstraints, +} from '../utils/index.js'; import pluralize from '../../utils/pluralize.js'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils.js'; import { status } from '@grpc/grpc-js'; @@ -20,10 +26,11 @@ import { ConduitDatabaseSchema, introspectedSchemaCmsOptionsDefaults, } from '../../interfaces/index.js'; -import { isNil, isEqual } from 'lodash-es'; +import { isEqual, isNil } from 'lodash-es'; // @ts-ignore import * as parseSchema from 'mongodb-schema'; +import { validateIndexFields } from '../utils/indexValidations.js'; export class MongooseAdapter extends DatabaseAdapter { connected: boolean = false; @@ -264,67 +271,98 @@ export class MongooseAdapter extends DatabaseAdapter { async createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: ModelOptionsIndex[], callerModule: string, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - this.checkIndexes(schemaName, indexes, callerModule); const collection = this.mongoose.model(schemaName).collection; for (const index of indexes) { - const indexSpecs = []; - for (let i = 0; i < index.fields.length; i++) { - const spec: any = {}; - spec[index.fields[i]] = index.types ? index.types[i] : 1; - indexSpecs.push(spec); - } + const convIndex = this.checkAndConvertIndex(schemaName, index, callerModule); + const indexSpecs: Indexable[] = convIndex.fields.map( + (field: string, i: number) => ({ + [field]: convIndex.types?.[i] ?? 1, + }), + ); await collection - .createIndex(indexSpecs, index.options as IndexOptions) + .createIndex(indexSpecs, { + name: convIndex.name, + ...convIndex.options, + } as IndexOptions) .catch((e: Error) => { throw new GrpcError(status.INTERNAL, e.message); }); } + // Add indexes to modelOptions + const modelOptionsIndexes = + this.models[schemaName].originalSchema.modelOptions.indexes ?? []; + const indexMap = new Map( + modelOptionsIndexes.map((i: ModelOptionsIndex) => [i.name, i]), + ); + for (const i of indexes) { + if (!indexMap.has(i.name)) { + indexMap.set(i.name, i); + } + } + const foundSchema = await this.models['_DeclaredSchema'].findOne({ + name: schemaName, + }); + await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, { + 'modelOptions.indexes': [...indexMap.values()], + }); return 'Indexes created!'; } - async getIndexes(schemaName: string): Promise { + async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const collection = this.mongoose.model(schemaName).collection; - const result = await collection.indexes(); - result.filter(index => { - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if (indexEntry[0] === 'key' || indexEntry[0] === 'options') { - continue; - } - if (indexEntry[0] === 'v') { - delete index.v; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; - } - index.fields = []; - index.types = []; - for (const keyEntry of Object.entries(index.key)) { - index.fields.push(keyEntry[0]); - index.types.push(keyEntry[1]); - delete index.key; + // Find schema field indexes and convert them to modelOption indexes + const indexes = []; + const ogSchema = this.models[schemaName].originalSchema; + const fields = ogSchema.fields; + + for (const [field, value] of Object.entries(fields)) { + const index = (value as ConduitModelField).index; + if (index) { + indexes.push({ + name: index.name, + fields: [field], + types: index.type + ? [ReverseMongoIndexTypeMap[index.type.toString()]] + : undefined, + options: index.options ?? undefined, + }); } - }); - return result as ModelOptionsIndexes[]; + } + const modelOptionsIndexes = (ogSchema.modelOptions.indexes ?? []).map( + (i: ModelOptionsIndex) => ({ + ...i, + types: i.types?.map( + (t: string | number) => ReverseMongoIndexTypeMap[t.toString()], + ), + }), + ); + indexes.push(...modelOptionsIndexes); + return indexes; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) - throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const foundSchema = await this.models['_DeclaredSchema'].findOne({ + name: schemaName, + }); const collection = this.mongoose.model(schemaName).collection; + let newSchema; for (const name of indexNames) { collection.dropIndex(name).catch(() => { throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); }); + // Remove index from fields/compiledFields or modelOptions + newSchema = findAndRemoveIndex(foundSchema, name); } + await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, newSchema); return 'Indexes deleted'; } @@ -445,46 +483,38 @@ export class MongooseAdapter extends DatabaseAdapter { await this.compareAndStoreMigratedSchema(schema); await this.saveSchemaToDatabase(schema); } - if (indexes && !isInstanceSync) { await this.createIndexes(schema.name, indexes, schema.ownerModule); } return this.models[schema.name]; } - private checkIndexes( + private checkAndConvertIndex( schemaName: string, - indexes: ModelOptionsIndexes[], + index: ModelOptionsIndex, callerModule: string, ) { - for (const index of indexes) { - const options = index.options; - const types = index.types; - if (!options && !types) continue; - if (options) { - if (!checkIfMongoOptions(options)) { - throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', - ); - } - } - if (types) { - if (!Array.isArray(types) || types.length !== index.fields.length) { - throw new GrpcError(status.INTERNAL, 'Invalid index types format'); - } - for (const type of types) { - if (!Object.values(MongoIndexType).includes(type)) { - throw new GrpcError(status.INTERNAL, 'Invalid index type for mongoDB'); - } - } - } + const { types, options } = index; + validateIndexFields(this.models[schemaName].originalSchema, index, callerModule); + if (options && !checkIfMongoOptions(options)) { + throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); + } + if (!types) return index; + if ( + types.length !== index.fields.length || + types.some( + type => + !(type in MongoIndexType) && !Object.values(MongoIndexType).includes(type), + ) + ) { + throw new GrpcError(status.INTERNAL, 'Invalid index types'); } + // Convert index types (if called by endpoint index.types contains the keys of MongoIndexType enum) + index.types = types.map((type: unknown) => { + if (Object.keys(MongoIndexType).includes(type as keyof typeof MongoIndexType)) + return MongoIndexType[type as unknown as keyof typeof MongoIndexType]; + return type; + }) as MongoIndexType[]; + return index; } } diff --git a/modules/database/src/adapters/mongoose-adapter/utils.ts b/modules/database/src/adapters/mongoose-adapter/utils.ts index ea04cebac..250acecd9 100644 --- a/modules/database/src/adapters/mongoose-adapter/utils.ts +++ b/modules/database/src/adapters/mongoose-adapter/utils.ts @@ -2,7 +2,7 @@ import { ConduitModel, Indexable, MongoIndexOptions, - PostgresIndexOptions, + SequelizeIndexOptions, } from '@conduitplatform/grpc-sdk'; import { MongooseAdapter } from './index.js'; import { cloneDeep, isArray, isObject } from 'lodash-es'; @@ -79,7 +79,7 @@ async function _createWithPopulations( } } -export function checkIfMongoOptions(options: MongoIndexOptions | PostgresIndexOptions) { +export function checkIfMongoOptions(options: MongoIndexOptions | SequelizeIndexOptions) { const mongoOptions = [ 'background', 'unique', diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index b76e26693..814d518a1 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -2,22 +2,24 @@ import { Sequelize } from 'sequelize'; import { ConduitGrpcSdk, ConduitModel, + ConduitModelField, ConduitSchema, GrpcError, Indexable, - ModelOptionsIndexes, - PostgresIndexOptions, - PostgresIndexType, + ModelOptionsIndex, + MySQLMariaDBIndexOptions, + PgIndexOptions, RawSQLQuery, + SequelizeIndexType, sleep, - UntypedArray, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { SequelizeAuto } from 'sequelize-auto'; import { DatabaseAdapter } from '../DatabaseAdapter.js'; import { SequelizeSchema } from './SequelizeSchema.js'; import { - checkIfPostgresOptions, + checkIfSequelizeIndexOptions, + checkIfSequelizeIndexType, compileSchema, resolveRelatedSchemas, tableFetch, @@ -30,6 +32,8 @@ import { import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter.js'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter.js'; import { isEqual, isNil } from 'lodash-es'; +import { findAndRemoveIndex } from '../utils/index.js'; +import { validateIndexFields } from '../utils/indexValidations.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -244,8 +248,8 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const dialect = this.sequelize.getDialect(); const [newSchema, objectPaths, extractedRelations] = dialect === 'postgres' - ? pgSchemaConverter(compiledSchema) - : sqlSchemaConverter(compiledSchema); + ? pgSchemaConverter(compiledSchema, dialect) + : sqlSchemaConverter(compiledSchema, dialect); this.registeredSchemas.set( schema.name, Object.freeze(JSON.parse(JSON.stringify(schema))), @@ -340,78 +344,74 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } getDatabaseType(): string { - const type = this.sequelize.getDialect(); - if (type === 'postgres') { - return 'PostgreSQL'; // TODO: clean up - } - return type; + return this.sequelize.getDialect(); } async createIndexes( schemaName: string, - indexes: ModelOptionsIndexes[], + indexes: ModelOptionsIndex[], callerModule: string, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - indexes = this.checkAndConvertIndexes(schemaName, indexes, callerModule); - const queryInterface = this.sequelize.getQueryInterface(); - for (const index of indexes) { - await queryInterface - .addIndex('cnd_' + schemaName, index.fields, index.options) - .catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); - }); + const convertedIndexes = indexes.map(i => + this.checkAndConvertIndex(schemaName, i, callerModule), + ); + const schema = this.models[schemaName].originalSchema; + const modelOptionsIndexes = schema.modelOptions.indexes ?? []; + const indexMap = new Map( + modelOptionsIndexes.map((i: ModelOptionsIndex) => [i.name, i]), + ); + for (const i of convertedIndexes) { + if (!indexMap.has(i.name)) { + indexMap.set(i.name, i); + } } - await this.models[schemaName].sync(); + Object.assign(schema.modelOptions, { indexes: [...indexMap.values()] }); + await this.createSchemaFromAdapter(schema); return 'Indexes created!'; } - async getIndexes(schemaName: string): Promise { + async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const queryInterface = this.sequelize.getQueryInterface(); - const result = (await queryInterface.showIndex('cnd_' + schemaName)) as UntypedArray; - result.filter(index => { - const fieldNames = []; - for (const field of index.fields) { - fieldNames.push(field.attribute); - } - index.fields = fieldNames; - // extract index type from index definition - let tmp = index.definition.split('USING '); - tmp = tmp[1].split(' '); - index.types = tmp[0]; - delete index.definition; - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if ( - indexEntry[0] === 'options' || - indexEntry[0] === 'types' || - indexEntry[0] === 'fields' - ) { - continue; - } - if (indexEntry[0] === 'indkey') { - delete index.indkey; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; + const indexes: ModelOptionsIndex[] = []; + // Find schema field indexes and convert them to modelOption indexes + for (const [field, value] of Object.entries( + this.models[schemaName].originalSchema.fields, + )) { + const index = (value as ConduitModelField).index; + if (index) { + indexes.push({ + name: index.name, + fields: [field], + types: index.type ? [index.type as SequelizeIndexType] : undefined, + options: index.options ?? undefined, + }); } - }); - return result; + } + indexes.push(...(this.models[schemaName].originalSchema.modelOptions.indexes ?? [])); + return indexes; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const foundSchema = await this.models['_DeclaredSchema'].findOne({ + name: schemaName, + }); const queryInterface = this.sequelize.getQueryInterface(); + let newSchema; for (const name of indexNames) { - queryInterface.removeIndex('cnd_' + schemaName, name).catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); - }); + queryInterface + .removeIndex(this.models[schemaName].originalSchema.collectionName, name) + .catch(() => { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + }); + // Remove index from fields/compiledFields or modelOptions + newSchema = findAndRemoveIndex(foundSchema, name); } + await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, newSchema); return 'Indexes deleted'; } @@ -457,44 +457,33 @@ export abstract class SequelizeAdapter extends DatabaseAdapter protected abstract hasLegacyCollections(): Promise; - private checkAndConvertIndexes( + private checkAndConvertIndex( schemaName: string, - indexes: ModelOptionsIndexes[], + index: ModelOptionsIndex, callerModule: string, ) { - for (const index of indexes) { - if (!index.types && !index.options) continue; - if (index.types) { - if ( - Array.isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types) - ) { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Invalid index type for PostgreSQL', - ); - } - (index.options as PostgresIndexOptions).using = index.types; - delete index.types; - } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Invalid index options for PostgreSQL', - ); - } - if ( - Object.keys(index.options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', - ); - } - } + const dialect = this.sequelize.getDialect(); + const { types, options } = index; + validateIndexFields(this.models[schemaName].originalSchema, index, callerModule); + if (options && !checkIfSequelizeIndexOptions(options, dialect)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid index options for ${dialect}`, + ); } - return indexes; + if (!types) return index; + if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { + throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`); + } + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(types[0] as string) + ) { + index.options = { ...index.options, type: types[0] } as MySQLMariaDBIndexOptions; + } else { + index.options = { ...index.options, using: types[0] } as PgIndexOptions; + } + delete index.types; + return index; } } diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 8c023fd7a..cfdf4d98b 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -18,12 +18,16 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; +import { ConduitDatabaseSchema } from '../../../interfaces/index.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent * @param jsonSchema */ -export function pgSchemaConverter(jsonSchema: ConduitSchema): [ +export function pgSchemaConverter( + jsonSchema: ConduitDatabaseSchema, + dialect: string, +): [ ConduitSchema, { [key: string]: { parentKey: string; childKey: string }; @@ -37,13 +41,13 @@ export function pgSchemaConverter(jsonSchema: ConduitSchema): [ delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy.modelOptions.indexes = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, dialect); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index fc869ca37..70167dce9 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -18,12 +18,16 @@ import { extractRelations, RelationType, } from '../utils/extractors/index.js'; +import { ConduitDatabaseSchema } from '../../../interfaces/index.js'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent * @param jsonSchema */ -export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ +export function sqlSchemaConverter( + jsonSchema: ConduitDatabaseSchema, + dialect: string, +): [ ConduitSchema, { [key: string]: { parentKey: string; childKey: string }; @@ -35,13 +39,13 @@ export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy.modelOptions.indexes = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, dialect); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/utils/index.ts b/modules/database/src/adapters/sequelize-adapter/utils/index.ts index aa71368f3..c2d6191f3 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/index.ts @@ -1,11 +1,9 @@ -import { MongoIndexOptions, PostgresIndexOptions } from '@conduitplatform/grpc-sdk'; - export * from './schema.js'; export * from './collectionUtils.js'; +export * from './indexChecks.js'; +import { MongoIndexOptions, PgIndexOptions } from '@conduitplatform/grpc-sdk'; -export function checkIfPostgresOptions( - options: MongoIndexOptions | PostgresIndexOptions, -) { +export function checkIfPostgresOptions(options: MongoIndexOptions | PgIndexOptions) { const postgresOptions = [ 'concurrently', 'name', diff --git a/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts b/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts new file mode 100644 index 000000000..f2aa5885e --- /dev/null +++ b/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts @@ -0,0 +1,42 @@ +import { + MySQLMariaDBIndexType, + PgIndexType, + SQLiteIndexType, +} from '@conduitplatform/grpc-sdk'; + +export function checkIfSequelizeIndexType(type: any, dialect?: string) { + switch (dialect) { + case 'postgres': + return type in PgIndexType; + case 'mysql' || 'mariadb': + return type in MySQLMariaDBIndexType; + case 'sqlite': + return type in SQLiteIndexType; + default: + return ( + type in PgIndexType || type in MySQLMariaDBIndexType || type in SQLiteIndexType + ); + } +} + +export function checkIfSequelizeIndexOptions(options: any, dialect?: string) { + const sequelizeOptions = [ + 'name', + 'parser', + 'unique', + 'fields', + 'where', + 'prefix', + 'using', + ]; + const pgOptions = sequelizeOptions.concat(['concurrently', 'operator']); + const mySqlMariaDbOptions = sequelizeOptions.concat(['type']); + switch (dialect) { + case 'postgres': + return !Object.keys(options).some(option => !pgOptions.includes(option)); + case 'mysql' || 'mariadb': + return !Object.keys(options).some(option => !mySqlMariaDbOptions.includes(option)); + default: + return !Object.keys(options).some(option => !sequelizeOptions.includes(option)); + } +} diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index d2f6b86e5..45667f669 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,13 +1,28 @@ -import { isArray, isBoolean, isNumber, isString } from 'lodash-es'; +import { + isArray, + isBoolean, + isNumber, + isString, + isObject, + isNil, + forEach, + has, +} from 'lodash-es'; import { ConduitGrpcSdk, ConduitModelField, - ConduitSchema, Indexable, - PostgresIndexOptions, - PostgresIndexType, + ModelOptionsIndex, + MySQLMariaDBIndexType, + PgIndexType, + SequelizeIndexOptions, + SQLiteIndexType, } from '@conduitplatform/grpc-sdk'; -import { checkIfPostgresOptions } from '../sequelize-adapter/utils/index.js'; +import { + checkIfSequelizeIndexOptions, + checkIfSequelizeIndexType, +} from '../sequelize-adapter/utils/index.js'; +import { ConduitDatabaseSchema } from '../../interfaces/index.js'; export function checkDefaultValue(type: string, value: string) { switch (type) { @@ -28,78 +43,99 @@ export function checkDefaultValue(type: string, value: string) { } } -export function convertModelOptionsIndexes(copy: ConduitSchema) { +export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: string) { + const convertedIndexes = []; + for (const index of copy.modelOptions.indexes!) { - if (index.types) { + const { fields, types, options } = index; + const compiledFields = Object.keys(copy.compiledFields); + if (fields.length === 0) { + throw new Error('Undefined fields for index creation'); + } + if (fields.some(field => !compiledFields.includes(field))) { + throw new Error(`Invalid fields for index creation`); + } + // Convert conduit indexes to sequelize indexes + if (options) { + if (!checkIfSequelizeIndexOptions(options, dialect)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index options for ${dialect} found in: ${copy.name}. Index ignored`, + ); + continue; + } + // Used instead of ModelOptionsIndexes fields for more complex index definitions + const seqOptions = options as SequelizeIndexOptions; if ( - isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types as PostgresIndexType) + !isNil(seqOptions.fields) && + seqOptions.fields.every(f => compiledFields.includes(f.name)) ) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn('Invalid index type for PostgreSQL, ignoring index'); - continue; - // throw new Error('Incorrect index type for PostgreSQL'); + (index.fields as any) = seqOptions.fields; + delete (index.options as SequelizeIndexOptions).fields; } - index.using = index.types as PostgresIndexType; - delete index.types; + Object.assign(index, options); + delete index.options; } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - // ignore index instead of error + if (types) { + if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { ConduitGrpcSdk.Logger.warn( - 'Invalid index options for PostgreSQL, ignoring index', + `Invalid index type for ${dialect} found in: ${copy.name}. Index ignored`, ); continue; - // throw new Error('Incorrect index options for PostgreSQL'); } - for (const [option, value] of Object.entries(index.options)) { - index[option as keyof PostgresIndexOptions] = value; + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(types[0] as string) + ) { + index.type = types[0] as MySQLMariaDBIndexType; + } else { + index.using = types[0] as PgIndexType | SQLiteIndexType; } - delete index.options; + delete index.types; } + convertedIndexes.push(index); } - return copy; + return convertedIndexes; } -export function convertSchemaFieldIndexes(copy: ConduitSchema) { +export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: string) { const indexes = []; - for (const field of Object.entries(copy.fields)) { - const fieldName = field[0]; - const index = (copy.fields[fieldName] as ConduitModelField).index; + for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { + const index = (fieldValue as ConduitModelField).index; if (!index) continue; - const newIndex: any = { - fields: [fieldName], - }; - if (index.type) { - if (!Object.values(PostgresIndexType).includes(index.type as PostgresIndexType)) { - // ignore index instead of error - ConduitGrpcSdk.Logger.warn('Invalid index type for PostgreSQL, ignoring index'); - continue; - // throw new Error('Invalid index type for PostgreSQL'); - } - newIndex.using = index.type; - } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - // ignore index instead of error + // Convert conduit indexes to sequelize indexes + const { name, type, options } = index; + const newIndex: any = { name, fields: [fieldName] }; + if (type) { + if (isArray(type) || !checkIfSequelizeIndexType(type, dialect)) { ConduitGrpcSdk.Logger.warn( - 'Invalid index options for PostgreSQL, ignoring index', + `Invalid index type for ${dialect} found in: ${copy.name}. Index ignored`, ); + delete (copy.fields[fieldName] as ConduitModelField).index; continue; - // throw new Error('Invalid index options for PostgreSQL'); } - for (const [option, value] of Object.entries(index.options)) { - newIndex[option] = value; + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(type as string) + ) { + newIndex.type = type as MySQLMariaDBIndexType; + } else { + newIndex.using = type as PgIndexType | SQLiteIndexType; } } + if (options && !checkIfSequelizeIndexOptions(options, dialect)) { + ConduitGrpcSdk.Logger.warn( + `Invalid index options for ${dialect} found in: ${copy.name}. Index ignored`, + ); + delete (copy.fields[fieldName] as ConduitModelField).index; + continue; + } + Object.assign(newIndex, options); indexes.push(newIndex); - delete copy.fields[fieldName]; - } - if (copy.modelOptions.indexes) { - copy.modelOptions.indexes = [...copy.modelOptions.indexes, ...indexes]; - } else { - copy.modelOptions.indexes = indexes; + delete (copy.fields[fieldName] as ConduitModelField).index; } + copy.modelOptions.indexes = copy.modelOptions.indexes + ? [...copy.modelOptions.indexes, ...indexes] + : indexes; return copy; } @@ -126,3 +162,21 @@ export function extractFieldProperties( return res; } + +export function findAndRemoveIndex(schema: any, indexName: string) { + const arrayIndex = schema.modelOptions.indexes.findIndex( + (i: ModelOptionsIndex) => i.name === indexName, + ); + if (arrayIndex !== -1) { + schema.modelOptions.indexes.splice(arrayIndex, 1); + return schema; + } + forEach(schema.fields, (value: ConduitModelField, key: string, fields: any) => { + if (isObject(value) && has(value, 'index') && value.index!.name === indexName) { + delete fields[key].index; + delete schema.compiledFields[key].index; + return schema; + } + }); + throw new Error('Index not found in schema'); +} diff --git a/modules/database/src/adapters/utils/indexValidations.ts b/modules/database/src/adapters/utils/indexValidations.ts new file mode 100644 index 000000000..47a0b9e99 --- /dev/null +++ b/modules/database/src/adapters/utils/indexValidations.ts @@ -0,0 +1,32 @@ +import { ConduitModel, GrpcError, ModelOptionsIndex } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { ConduitDatabaseSchema } from '../../interfaces/index.js'; + +export function validateIndexFields( + schema: ConduitDatabaseSchema, + index: ModelOptionsIndex, + callerModule: string, +) { + const compiledFields = schema.compiledFields; + if (index.fields.some(field => !Object.keys(compiledFields).includes(field))) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid fields for index creation'); + } + + let ownedExtensionFields: ConduitModel = {}; + for (const ext of schema.extensions) { + if (ext.ownerModule === callerModule) { + ownedExtensionFields = ext.fields; + break; + } + } + + const isOwnerOfSchema = schema.ownerModule === callerModule; + for (const field of index.fields) { + if (index.options?.unique && !(field in ownedExtensionFields) && !isOwnerOfSchema) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Not authorized to create unique index', + ); + } + } +} diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 0776c9420..cfe0c17d6 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -597,6 +597,45 @@ export class AdminHandlers { }), this.customEndpointsAdmin.schemaDetailsForOperation.bind(this.customEndpointsAdmin), ); + this.routingManager.route( + { + path: '/schemas/indexes/import', + action: ConduitRouteActions.POST, + description: `Imports indexes.`, + bodyParams: { + indexes: [ + { + schemaName: ConduitString.Required, + name: ConduitString.Required, + fields: [ConduitString.Required], + types: [ConduitString.Optional], + options: ConduitJson.Optional, + } as never, + ], + }, + }, + new ConduitRouteReturnDefinition('ImportIndexes', 'String'), + this.schemaAdmin.importIndexes.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/indexes/export', + action: ConduitRouteActions.GET, + description: `Exports indexes.`, + }, + new ConduitRouteReturnDefinition('ExportIndexes', { + indexes: [ + { + schemaName: ConduitString.Required, + fields: [ConduitString.Required], + name: ConduitString.Optional, + types: [ConduitString.Optional], + options: ConduitJson.Optional, + }, + ], + }), + this.schemaAdmin.exportIndexes.bind(this.schemaAdmin), + ); this.routingManager.route( { path: '/schemas/:id/indexes', @@ -606,7 +645,14 @@ export class AdminHandlers { id: { type: TYPE.String, required: true }, }, bodyParams: { - indexes: [ConduitJson.Required], + indexes: [ + { + fields: [ConduitString.Required], + types: [ConduitString.Required], + name: ConduitString.Optional, + options: ConduitJson.Optional, + } as never, + ], }, }, new ConduitRouteReturnDefinition('CreateSchemaIndexes', 'String'), @@ -622,7 +668,14 @@ export class AdminHandlers { }, }, new ConduitRouteReturnDefinition('getSchemaIndexes', { - indexes: [ConduitJson.Required], + indexes: [ + { + name: ConduitString.Required, + fields: [ConduitString.Required], + types: [ConduitString.Optional], + options: ConduitJson.Optional, + }, + ], }), this.schemaAdmin.getIndexes.bind(this.schemaAdmin), ); diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index a862fcb11..9c7268361 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -3,6 +3,7 @@ import { ConduitSchema, GrpcError, Indexable, + ModelOptionsIndex, ParsedRouterRequest, UnparsedRouterResponse, } from '@conduitplatform/grpc-sdk'; @@ -641,6 +642,50 @@ export class SchemaAdmin { return this.database.getDatabaseType(); } + async importIndexes(call: ParsedRouterRequest): Promise { + const { indexes } = call.request.params; + const indexMap = new Map(); + for (const i of indexes) { + const { schemaName, ...rest } = i; + if (indexMap.has(schemaName)) { + indexMap.get(schemaName)!.push(rest); + } else { + indexMap.set(i.schemaName, [rest]); + } + } + for (const [schemaName, indexes] of indexMap) { + await this.database.createIndexes(schemaName, indexes, 'database').catch(e => { + throw new GrpcError(status.INTERNAL, `Index creation failed: ${e.message}`); + }); + } + return 'Indexes imported successfully'; + } + + async exportIndexes(): Promise { + const indexes = []; + const schemas = await this.database.getSchemaModel('_DeclaredSchema').model.findMany({ + $or: [ + { 'modelOptions.conduit.cms.enabled': true }, + { + $and: [ + { 'modelOptions.conduit.cms': { $exists: false } }, + { 'modelOptions.conduit.permissions.extendable': true }, + { extensions: { $exists: true } }, + ], + }, + ], + }); + for (const schema of schemas) { + const schemaIndexes = await this.database.getIndexes(schema.name); + if (!isNil(schemaIndexes) && !isEmpty(schemaIndexes)) { + indexes.push( + ...schemaIndexes.map(index => ({ ...index, schemaName: schema.name })), + ); + } + } + return { indexes }; + } + async createIndexes(call: ParsedRouterRequest): Promise { const { id, indexes } = call.request.params; const requestedSchema = await this.database @@ -660,7 +705,8 @@ export class SchemaAdmin { if (isNil(requestedSchema)) { throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); } - return this.database.getIndexes(requestedSchema.name); + const indexes = await this.database.getIndexes(requestedSchema.name); + return { indexes }; } async deleteIndexes(call: ParsedRouterRequest): Promise { diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index 524fb2c94..cc2069a21 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -13,6 +13,7 @@ import { ConduitModelOptionsPermModifyType as ValidModifyPermValues, ConduitSchemaOptions, Indexable, + ModelOptionsIndex, TYPE, } from '@conduitplatform/grpc-sdk'; @@ -148,8 +149,8 @@ export function populateArray(pop: any) { function validateModelOptions(modelOptions: ConduitSchemaOptions) { if (!isPlainObject(modelOptions)) throw new Error('Model options must be an object'); Object.keys(modelOptions).forEach(key => { - if (key !== 'conduit' && key !== 'timestamps') - throw new Error("Only 'conduit' and 'timestamps' options allowed"); + if (key !== 'conduit' && key !== 'timestamps' && key !== 'indexes') + throw new Error("Only 'conduit', 'timestamps' and 'indexes' options allowed"); else if (key === 'timestamps' && !isBoolean(modelOptions.timestamps)) throw new Error("Option 'timestamps' must be of type Boolean"); else if (key === 'conduit') { @@ -163,7 +164,7 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { conduitKey !== 'imported' ) throw new Error( - "Only 'cms' and 'permissions' fields allowed inside 'conduit' field", + "Only 'cms', 'permissions', 'authorization' and 'imported' fields allowed inside 'conduit' field", ); if (conduitKey === 'imported') { if (!isBoolean(modelOptions.conduit!.imported)) @@ -183,6 +184,32 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { if (modelOptions.conduit!.permissions) { validatePermissions(modelOptions.conduit.permissions); } + } else if (key === 'indexes') { + if (!isArray(modelOptions.indexes)) + throw new Error("Option 'indexes' must be an array"); + modelOptions.indexes.forEach((index: ModelOptionsIndex) => { + if ( + Object.keys(index).some( + key => !['name', 'fields', 'types', 'options'].includes(key), + ) + ) { + throw new Error( + "Only 'name', 'fields', 'types' and 'options' fields allowed inside 'indexes' array", + ); + } + if (!isString(index.name)) { + throw new Error("Index field option 'name' must be of type String"); + } + if (!isArray(index.fields)) { + throw new Error("Index field option 'fields' must be of type Array"); + } + if (index.types && !isArray(index.types)) { + throw new Error("Index field option 'types' must be of type Array"); + } + if (index.options && !isObject(index.options)) { + throw new Error("Index field option 'options' must be of type Object"); + } + }); } }); }