diff --git a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts index dabd2900ac523..cda85e9b969e9 100644 --- a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts +++ b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts @@ -3,7 +3,9 @@ import { inferLogicalModels } from './inferLogicalModel'; describe('inferLogicalModels', () => { it('returns a logical model', () => { const document = { - _id: '11123-123-123', + _id: { + $oid: '11123-123-123', + }, name: 'John', age: 30, isActive: true, @@ -28,7 +30,9 @@ describe('inferLogicalModels', () => { it('returns multiple logical models with array', () => { const document = { - _id: '11123-123-123', + _id: { + $oid: '11123-123-123', + }, name: 'John', age: 30, isActive: true, @@ -108,7 +112,9 @@ describe('inferLogicalModels', () => { it('returns multiple logical models with object', () => { const document = { - _id: 'asd', + _id: { + $oid: 'asd', + }, name: 'Stu', year: 2018, gpa: 3.5, @@ -264,4 +270,43 @@ describe('inferLogicalModels', () => { }, ]); }); + + it('handles documents with object ids with names other than _id', () => { + const document = { + _id: { + $oid: '5a9427648b0beebeb69579cc', + }, + name: 'Andrea Le', + email: 'andrea_le@fakegmail.com', + movie_id: { + $oid: '573a1390f29313caabcd418c', + }, + text: 'Rem officiis eaque repellendus amet eos doloribus. Porro dolor voluptatum voluptates neque culpa molestias. Voluptate unde nulla temporibus ullam.', + date: { + $date: '2012-03-26T23:20:16.000Z', + }, + }; + + const logicalModels = inferLogicalModels( + 'new-documents', + JSON.stringify(document) + ); + + expect(logicalModels).toEqual([ + { + fields: [ + { name: '_id', type: { nullable: false, scalar: 'objectId' } }, + { name: 'name', type: { nullable: false, scalar: 'string' } }, + { name: 'email', type: { nullable: false, scalar: 'string' } }, + { + name: 'movie_id', + type: { nullable: false, scalar: 'objectId' }, + }, + { name: 'text', type: { nullable: false, scalar: 'string' } }, + { name: 'date', type: { nullable: false, scalar: 'date' } }, + ], + name: 'newdocuments', + }, + ]); + }); }); diff --git a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts index 2023dcf78b48e..98a766c6c4476 100644 --- a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts +++ b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts @@ -32,111 +32,168 @@ const getLogicalModelsFromProperties = ( ): LogicalModel[] => { const logicalModels: LogicalModel[] = []; const fields: LogicalModelField[] = []; - - type ItemSchemaTypes = { - anyOf?: Array<{ type: string }>; - type?: string; - }; - const isMixedArray = (itemsSchema: ItemSchemaTypes): boolean => { - if (itemsSchema.anyOf) { - const types = itemsSchema.anyOf.map(subSchema => subSchema.type); - return ( - types.includes('object') && !types.every(type => type === 'object') - ); - } - return false; - }; - for (const [rawFieldName, fieldSchema] of Object.entries(properties)) { const fieldName = sanitizeGraphQLFieldNames(rawFieldName); - - if (fieldName === '_id') { - fields.push({ - name: fieldName, - type: { - scalar: 'objectId', - nullable: false, - }, - }); - continue; - } - const nullable = !requiredProperties.includes(fieldName); const logicalModelPath = parentName ? `${parentName}_${fieldName}` : fieldName; - if (fieldSchema.type === 'object') { - if (fieldSchema.properties) { - const newLogicalModels = getLogicalModelsFromProperties( - collectionName, - `${collectionName}_${logicalModelPath}`, - fieldSchema.properties, - fieldSchema.required, - logicalModelPath - ); - - logicalModels.push(...newLogicalModels); + // Get scalars from MongoDB objectid and date objects + const handleMongoDBFieldTypes = ( + properties: ObjectSchema['properties'] + ): { type: 'objectId' | 'date' | 'string' | 'none'; name?: string } => { + if (!properties) { + return { type: 'string' }; + } + if (Object.prototype.hasOwnProperty.call(properties, '$oid')) { + return { type: 'objectId' }; + } + if (Object.prototype.hasOwnProperty.call(properties, '$date')) { + return { type: 'date' }; + } + return { type: 'none' }; + }; + if (fieldSchema.type === 'object') { + // Seperate MongoDB objectid and date scalars from logical model objects + const mongoDBFieldType = handleMongoDBFieldTypes(fieldSchema.properties); + if (mongoDBFieldType.type !== 'none') { fields.push({ name: fieldName, type: { - logical_model: `${collectionName}_${logicalModelPath}`, - nullable, - }, - }); - } else { - // Empty object just being casted to `string` - fields.push({ - name: fieldName, - type: { - scalar: 'string', - nullable, + scalar: mongoDBFieldType.type, + nullable: false, }, }); + continue; } + // Make new logical model + const newLogicalModels = getLogicalModelsFromProperties( + collectionName, + `${collectionName}_${logicalModelPath}`, + fieldSchema.properties, + fieldSchema.required, + logicalModelPath + ); + logicalModels.push(...newLogicalModels); + fields.push({ + name: fieldName, + type: { + logical_model: `${collectionName}_${logicalModelPath}`, + nullable, + }, + }); } if (fieldSchema.type === 'array') { - if (isMixedArray(fieldSchema.items)) { + // Throw error for mixed object / scalar array + // Schema inferer returns anyOf if there are any type conflicts + const hasNestedAnyOf = (function checkNestedAnyOf(obj: any): boolean { + if (typeof obj !== 'object' || obj === null) return false; + if ('anyOf' in obj) return true; + return Object.values(obj).some( + val => typeof val === 'object' && checkNestedAnyOf(val) + ); + })(fieldSchema.items); + if (hasNestedAnyOf) { throw new Error( - `The array for field "${fieldName}" contains both objects and scalars (string, int, etc.). Please check and ensure it only contains one for inference. \n Exact key with issue: "${logicalModelPath}"` + `The array for field "${fieldName}" contains both multiple types (objects, string, int, etc.). Please check and ensure it only contains one for inference. \n Exact key with issue: "${logicalModelPath}"` ); } + + // Array of objects if (fieldSchema.items.type === 'object') { - const newLogicalModels = getLogicalModelsFromProperties( - collectionName, - `${collectionName}_${logicalModelPath}`, - fieldSchema.items.properties, - fieldSchema.items?.required || [], - logicalModelPath + // Check for special mongo scalars + const mongoDBFieldType = handleMongoDBFieldTypes( + fieldSchema.items.properties ); + if (mongoDBFieldType.type !== 'none') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: mongoDBFieldType.type, + nullable, + }, + }, + }); + } else { + // Make new logical model for array + const newLogicalModels = getLogicalModelsFromProperties( + collectionName, + `${collectionName}_${logicalModelPath}`, + fieldSchema.items.properties, + fieldSchema.items?.required || [], + logicalModelPath + ); - logicalModels.push(...newLogicalModels); + logicalModels.push(...newLogicalModels); - fields.push({ - name: fieldName, - type: { - array: { - logical_model: `${collectionName}_${logicalModelPath}`, - nullable, + fields.push({ + name: fieldName, + type: { + array: { + logical_model: `${collectionName}_${logicalModelPath}`, + nullable, + }, }, - }, - }); - } else { - // scalar array - fields.push({ - name: fieldName, - type: { - array: { - scalar: fieldSchema.items.type, - nullable, + }); + } + continue; + } + // Array of scalars + if (fieldSchema.items.type !== 'object') { + // Process scalar types in array + // TODO: DRY this out with scalar processing below + if (fieldSchema.items.type === 'string') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'string', + nullable, + }, }, - }, - }); + }); + } + if (fieldSchema.items.type === 'integer') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'int', + nullable, + }, + }, + }); + } + if (fieldSchema.items.type === 'number') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'double', + nullable, + }, + }, + }); + } + if (fieldSchema.items.type === 'boolean') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'bool', + nullable, + }, + }, + }); + } } } + // Process scalars if (fieldSchema.type === 'string') { fields.push({ name: fieldName, @@ -177,7 +234,6 @@ const getLogicalModelsFromProperties = ( }); } } - return [ { name,