Index SPI - types&interfaces, implementation for MySQL

.eslintrc.json

@@ -61,7 +61,8 @@
             "@typescript-eslint/no-unused-vars": [ "error", { "ignoreRestSiblings": true, "argsIgnorePattern": "^_" } ],
             "@typescript-eslint/no-explicit-any": "off",
             "@typescript-eslint/ban-ts-comment": "warn",
-            "eol-last": [ "error", "always" ]
+            "eol-last": [ "error", "always" ], 
+            "@typescript-eslint/no-empty-interface": "off"
           }
         }
       ]

.github/workflows/main.yml

@@ -38,15 +38,7 @@ jobs:
     strategy:
       matrix:
         database: [
-                   "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9",
-                   "spanner",
-                   "mysql", "mysql5",
-                   "mssql", "mssql17",
-                   "mongo", "mongo4",
-                   "firestore",
-                   "airtable",
-                   "dynamodb",
-                   "google-sheets"
+                   "mysql", "mysql5"
                    ]
 
     env:

apps/velo-external-db/src/app.ts

@@ -4,10 +4,20 @@ import { ExternalDbRouter, Hooks } from '@wix-velo/velo-external-db-core'
 import { engineConnectorFor } from './storage/factory'
 
 
-const initConnector = async(hooks?: Hooks) => {
-    const { vendor, type: adapterType } = readCommonConfig()
+process.env.CLOUD_VENDOR = 'azure'
+process.env.TYPE = 'mysql'
+process.env.EXTERNAL_DATABASE_ID = ''
+process.env.ALLOWED_METASITES = ''
+process.env['TYPE'] = 'mysql'
+process.env['HOST'] = 'localhost'
+process.env['USER'] = 'test-user'
+process.env['PASSWORD'] = 'password'
+process.env['DB'] = 'test-db'
+
+const initConnector = async(wixDataBaseUrl?: string, hooks?: Hooks) => {
+    const { vendor, type: adapterType, externalDatabaseId, allowedMetasites } = readCommonConfig()
     const configReader = create()
-    const { authorization, secretKey, ...dbConfig } = await configReader.readConfig()
+    const { authorization, ...dbConfig } = await configReader.readConfig()
 
     const { connector: engineConnector, providers, cleanup } = await engineConnectorFor(adapterType, dbConfig)
 
@@ -17,22 +27,24 @@ const initConnector = async(hooks?: Hooks) => {
             authorization: {
                 roleConfig: authorization
             },
-            secretKey,
+            externalDatabaseId,
+            allowedMetasites,
             vendor,
             adapterType,
-            commonExtended: true
+            commonExtended: true,
+            wixDataBaseUrl: wixDataBaseUrl || 'https://www.wixapis.com/wix-data'
         },
         hooks
     })
 
     return { externalDbRouter, cleanup: async() => await cleanup(), schemaProvider: providers.schemaProvider }
 }
 
-export const createApp = async() => {
+export const createApp = async(wixDataBaseUrl?: string) => {
     const app = express()
-    const initConnectorResponse = await initConnector()
+    const initConnectorResponse = await initConnector(wixDataBaseUrl)
     app.use(initConnectorResponse.externalDbRouter.router)
     const server = app.listen(8080, () => console.log('Connector listening on port 8080'))
 
-    return { server, ...initConnectorResponse, reload: () => initConnector() }
+    return { server, ...initConnectorResponse, reload: () => initConnector(wixDataBaseUrl) }
 }

apps/velo-external-db/src/storage/factory.ts

@@ -3,46 +3,46 @@ import { DatabaseFactoryResponse } from '@wix-velo/velo-external-db-commons'
 export const engineConnectorFor = async(_type: string, config: any): Promise<DatabaseFactoryResponse> => {
     const type = _type || ''
     switch ( type.toLowerCase() ) {
-        case 'postgres': {
-            const { postgresFactory } = require('@wix-velo/external-db-postgres')
-            return await postgresFactory(config)
-        }
-        case 'spanner': {
-            const { spannerFactory } = require('@wix-velo/external-db-spanner')
-            return await spannerFactory(config)
-        }
-        case 'firestore': {
-            const { firestoreFactory } = require('@wix-velo/external-db-firestore')
-            return await firestoreFactory(config)
-        }
-        case 'mssql': {
-            const { mssqlFactory } = require('@wix-velo/external-db-mssql')
-            return await mssqlFactory(config)
-        }
+        // case 'postgres': {
+        //     const { postgresFactory } = require('@wix-velo/external-db-postgres')
+        //     return await postgresFactory(config)
+        // }
+        // case 'spanner': {
+        //     const { spannerFactory } = require('@wix-velo/external-db-spanner')
+        //     return await spannerFactory(config)
+        // }
+        // case 'firestore': {
+        //     const { firestoreFactory } = require('@wix-velo/external-db-firestore')
+        //     return await firestoreFactory(config)
+        // }
+        // case 'mssql': {
+        //     const { mssqlFactory } = require('@wix-velo/external-db-mssql')
+        //     return await mssqlFactory(config)
+        // }
         case 'mysql': {
             const { mySqlFactory } = require('@wix-velo/external-db-mysql')
             return await mySqlFactory(config)
         }
-        case 'mongo': {
-            const { mongoFactory } = require('@wix-velo/external-db-mongo')
-            return await mongoFactory(config)
-        }
-        case 'google-sheet': {
-            const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
-            return await googleSheetFactory(config)
-        }
-        case 'airtable': {
-            const { airtableFactory } = require('@wix-velo/external-db-airtable')
-            return await airtableFactory(config)
-        }
-        case 'dynamodb': {
-            const { dynamoDbFactory } = require('@wix-velo/external-db-dynamodb')
-            return await dynamoDbFactory(config)
-        }
-        case 'bigquery': {
-            const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
-            return await bigqueryFactory(config)
-        }
+        // case 'mongo': {
+        //     const { mongoFactory } = require('@wix-velo/external-db-mongo')
+        //     return await mongoFactory(config)
+        // }
+        // case 'google-sheet': {
+        //     const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
+        //     return await googleSheetFactory(config)
+        // }
+        // case 'airtable': {
+        //     const { airtableFactory } = require('@wix-velo/external-db-airtable')
+        //     return await airtableFactory(config)
+        // }
+        // case 'dynamodb': {
+        //     const { dynamoDbFactory } = require('@wix-velo/external-db-dynamodb')
+        //     return await dynamoDbFactory(config)
+        // }
+        // case 'bigquery': {
+        //     const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
+        //     return await bigqueryFactory(config)
+        // }
         default: {
             const { stubFactory } = require('./stub-db/stub-connector')
             return await stubFactory(type, config)

apps/velo-external-db/test/drivers/data_api_rest_test_support.ts

@@ -1,9 +1,70 @@
+import axios from 'axios'
 import { Item } from '@wix-velo/velo-external-db-types'
+import { dataSpi } from '@wix-velo/velo-external-db-core'
+import { streamToArray } from '@wix-velo/test-commons'
 
-const axios = require('axios').create({
+const axiosInstance = axios.create({
     baseURL: 'http://localhost:8080'
 })
 
-export const givenItems = async(items: Item[], collectionName: string, auth: any) => await axios.post('/data/insert/bulk', { collectionName: collectionName, items: items }, auth)
+export const insertRequest = (collectionName: string, items: Item[], overwriteExisting: boolean): dataSpi.InsertRequest => ({
+    collectionId: collectionName,
+    items: items,
+    overwriteExisting,
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    }
+})
+
+export const updateRequest = (collectionName: string, items: Item[]): dataSpi.UpdateRequest => ({
+    collectionId: collectionName,
+    items: items,
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    }
+})
+
+export const countRequest = (collectionName: string): dataSpi.CountRequest => ({
+    collectionId: collectionName,
+    filter: '',
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    },
+})
+
+export const queryRequest = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], filter?: dataSpi.Filter): dataSpi.QueryRequest => ({
+    collectionId: collectionName,
+    query: {
+        filter: filter ?? '',
+        sort: sort,
+        fields: fields,
+        fieldsets: undefined,
+        paging: {
+            limit: 25,
+            offset: 0,
+        },
+        cursorPaging: null
+    },
+    includeReferencedItems: [],
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    },
+    omitTotalCount: false
+})
+
+
+export const queryCollectionAsArray = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], auth: any, filter?: dataSpi.Filter) =>
+    axiosInstance.post('/data/query',
+        queryRequest(collectionName, sort, fields, filter), { responseType: 'stream', transformRequest: auth.transformRequest })
+        .then(response => streamToArray(response.data))
+
+
+export const pagingMetadata = (total: number, count: number): dataSpi.QueryResponsePart => ({ pagingMetadata: { count: count, offset: 0, total: total, tooManyToCount: false } })
+
 
-export const expectAllDataIn = async(collectionName: string, auth: any) => (await axios.post('/data/find', { collectionName: collectionName, filter: '', sort: '', skip: 0, limit: 25 }, auth)).data
+export const givenItems = async(items: Item[], collectionName: string, auth: any) =>
+    await axiosInstance.post('/data/insert', insertRequest(collectionName, items, false), { responseType: 'stream', transformRequest: auth.transformRequest })

apps/velo-external-db/test/drivers/index_api_rest_matchers.ts

@@ -0,0 +1,46 @@
+import { indexSpi } from '@wix-velo/velo-external-db-core'
+const { IndexFieldOrder, IndexStatus } = indexSpi
+
+const responseWith = (matcher: any) => expect.objectContaining({ data: matcher })
+
+
+const indexWith = (index: indexSpi.Index, extraProps: Partial<indexSpi.Index>) => ({
+    ...index,
+    fields: index.fields.map(field => ({
+        ...field,
+        order: expect.toBeOneOf([IndexFieldOrder.ASC, IndexFieldOrder.DESC]),
+    })),
+    caseInsensitive: expect.any(Boolean), // TODO: remove this when we support case insensitive indexes
+    ...extraProps
+})
+
+
+export const listIndexResponseWithDefaultIndex = () =>
+    expect.arrayContaining([toHaveDefaultIndex()])
+
+export const listIndexResponseWith = (indexes: indexSpi.Index[]) =>
+    expect.arrayContaining(
+        [...indexes.map((index: indexSpi.Index) => indexWith(index, { status: IndexStatus.ACTIVE }))]
+    )
+
+export const toHaveDefaultIndex = () => ({
+    name: expect.any(String),
+    fields: expect.arrayContaining([
+        expect.objectContaining({
+            path: '_id',
+            order: expect.toBeOneOf([IndexFieldOrder.ASC, IndexFieldOrder.DESC])
+        })
+    ]),
+    caseInsensitive: expect.any(Boolean),
+    status: IndexStatus.ACTIVE,
+    unique: true
+})
+
+
+export const createIndexResponseWith = (index: indexSpi.Index) => responseWith(({ index: indexWith(index, { status: IndexStatus.BUILDING }) }))
+
+export const removeIndexResponse = () => responseWith(({}))
+
+export const listIndexResponseWithFailedIndex = (index: indexSpi.Index) => {
+    return expect.arrayContaining([indexWith(index, { status: IndexStatus.FAILED })])
+}

apps/velo-external-db/test/drivers/index_api_rest_test_support.ts

@@ -0,0 +1,24 @@
+import { streamToArray } from '@wix-velo/test-commons'
+import waitUntil from 'async-wait-until'
+import { indexSpi } from '@wix-velo/velo-external-db-core'
+
+const axios = require('axios').create({
+    baseURL: 'http://localhost:8080'
+})
+
+export const givenIndexes = async(collectionName: string, indexes: indexSpi.Index[], auth: any) => {
+    for (const index of indexes) {
+        await axios.post('/indexes/create', { dataCollectionId: collectionName, index } as indexSpi.CreateIndexRequest, auth)
+    }
+    await Promise.all(indexes.map(index => indexCreated(collectionName, index.name, auth)))
+}
+
+const indexCreated = async(collectionName: string, indexName: string, auth: any) => {
+    await waitUntil(async() => {
+        const indexes = await retrieveIndexesFor(collectionName, auth) as indexSpi.Index[]
+        return indexes.some(index => index.name === indexName)
+    })
+}
+
+export const retrieveIndexesFor = async(collectionName: string, auth: any) => axios.post('/indexes/list', { dataCollectionId: collectionName }, { responseType: 'stream', ...auth })
+    .then(response => streamToArray(response.data))

apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts

@@ -1,5 +1,6 @@
 import { SystemFields, asWixSchemaHeaders } from '@wix-velo/velo-external-db-commons'
 import { InputField } from '@wix-velo/velo-external-db-types'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
 
 export const responseWith = (matcher: any) => expect.objectContaining( { data: matcher } )
 
@@ -40,3 +41,35 @@ const toHaveCollections = (collections: string[]) => expect.objectContaining( {
 const listToHaveCollection = (collectionName: string) => expect.objectContaining( {
     schemas: expect.arrayContaining( [ expect.objectContaining( { id: collectionName } ) ] )
 } )
+
+const collectionCapabilities = (_collectionOperations: any[], _dataOperations: any[], _fieldTypes: any[]) => ({
+    collectionOperations: expect.any(Array),
+    dataOperations: expect.any(Array),
+    fieldTypes: expect.any(Array)
+})
+
+const fieldCapabilitiesMatcher = () => expect.objectContaining({
+    queryOperators: expect.any(Array),
+    sortable: expect.any(Boolean),
+})
+
+const filedMatcher = (field: InputField) => ({
+    key: field.name,
+    capabilities: fieldCapabilitiesMatcher(),
+    encrypted: expect.any(Boolean),
+    type: schemaUtils.fieldTypeToWixDataEnum(field.type)
+})
+
+const fieldsMatcher = (fields: InputField[]) => expect.toIncludeSameMembers(fields.map(filedMatcher))
+
+export const collectionResponsesWith = (collectionName: string, fields: InputField[]) => ({
+    id: collectionName,
+    capabilities: collectionCapabilities([], [], []),
+    fields: fieldsMatcher(fields),
+})
+
+export const createCollectionResponse = (collectionName: string, fields: InputField[]) => ({
+    id: collectionName,
+    capabilities: collectionCapabilities([], [], []),
+    fields: fieldsMatcher(fields),
+})

apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts

@@ -1,14 +1,34 @@
+import axios from 'axios'
 import { InputField } from '@wix-velo/velo-external-db-types'
+import { streamToArray } from '@wix-velo/test-commons'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
 
-const axios = require('axios').create({
+
+const axiosClient = axios.create({
     baseURL: 'http://localhost:8080'
 })
 
 export const givenCollection = async(name: string, columns: InputField[], auth: any) => {
-    await axios.post('/schemas/create', { collectionName: name }, auth)
-    for (const column of columns) {
-        await axios.post('/schemas/column/add', { collectionName: name, column: column }, auth)
+    const collection = {
+        id: name,
+        fields: columns.map(schemaUtils.InputFieldToWixFormatField)
     }
+    await axiosClient.post('/collections/create', { collection }, { ...auth, responseType: 'stream' })
 }
 
-export const retrieveSchemaFor = async(collectionName: string, auth: any) => axios.post('/schemas/find', { schemaIds: [collectionName] }, auth)
+export const deleteAllCollections = async(auth: any) => {
+    const res = await axiosClient.post('/collections/get', { collectionIds: [] }, { ...auth, responseType: 'stream' })
+    const dataRes = await streamToArray(res.data) as any []
+    const collectionIds = dataRes.map(d => d.id)
+
+    for (const collectionId of collectionIds) {
+        await axiosClient.post('/collections/delete', { collectionId }, { ...auth, responseType: 'stream' })
+    }
+
+}
+
+export const retrieveSchemaFor = async(collectionName: string, auth: any) => {
+    const collectionGetStream = await axiosClient.post('/collections/get', { collectionIds: [collectionName] }, { ...auth, responseType: 'stream' })
+    const [collectionGetRes] = await streamToArray(collectionGetStream.data) as any[]
+    return collectionGetRes
+}