feat(sf): deploy analytics toolbox native app

.github/workflows/snowflake.yml

@@ -15,7 +15,7 @@ on:
   workflow_call:
 
 env:
-  NODE_VERSION: 14
+  NODE_VERSION: 16
   PYTHON3_VERSION: 3.8.10
   VIRTUALENV_VERSION: 20.21.1
   GCLOUD_VERSION: 290.0.1
@@ -88,6 +88,37 @@ jobs:
           cd clouds/snowflake
           make deploy diff="$GIT_DIFF" production=1
 
+  deploy-internal-app:
+    if: github.ref_name == 'main'
+    needs: test
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    env:
+      APP_PACKAGE_NAME: ${{ secrets.SF_NATIVE_APP_PACKAGE_NAME_CD }}
+      APP_NAME: ${{ secrets.SF_NATIVE_APP_NAME_CD }}
+      SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_NATIVE_APP }}
+      SF_DATABASE: ${{ secrets.SF_DATABASE_NATIVE_APP }}
+      SF_USER: ${{ secrets.SF_USER_NATIVE_APP }}
+      SF_PASSWORD: ${{ secrets.SF_PASSWORD_NATIVE_APP }}
+      SF_ROLE: ${{ secrets.SF_ROLE_NATIVE_APP }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Check diff
+        uses: technote-space/get-diff-action@v4
+      - name: Setup node
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Deploy native app package
+        run: |
+          cd clouds/snowflake
+          make deploy-native-app-package production=1
+      - name: Deploy native app locally
+        run: |
+          cd clouds/snowflake
+          make deploy-native-app production=1
+
   deploy-share:
     if: github.ref_name == 'stable'
     needs: test

CONTRIBUTING.md

@@ -221,4 +221,9 @@ in the case of BigQuery, the backtick quotes. For example:
 Extra dependencies:
 `@@BQ_DATASET@@.SOME_FUNCTION`()
 */
-```
+```
+
+## Known Limitations
+### Snowflake
+
+Due to Snowflake Native Apps limitations at this moment TEMPORARY tables cannot be used within procedures. For the time being create normal tables and ensure that they are dropped.

clouds/snowflake/Makefile

@@ -13,10 +13,10 @@ include $(COMMON_DIR)/Makefile
 
 .SILENT:
 
-.PHONY: help lint build deploy test remove clean create-package
+.PHONY: help lint build deploy test remove clean create-package deploy-native-app-package deploy-native-app
 
 help:
-	echo "Available targets: lint build deploy test remove clean create-package"
+	echo "Available targets: lint build deploy test remove clean create-package deploy-native-app-package deploy-native-app"
 
 lint:
 	$(MAKE) lint-libraries
@@ -54,6 +54,13 @@ build-modules:
 	$(MAKE) -C modules build
 	cp modules/build/modules.sql $(BUILD_DIR)
 
+build-native-app-setup-script:
+	rm -rf $(BUILD_DIR)
+	$(MAKE) build-libraries
+	mkdir -p $(BUILD_DIR)
+	$(MAKE) -C modules build-native-app-setup-script
+	cp modules/build/setup_script.sql $(BUILD_DIR)
+
 deploy:
 	$(MAKE) build-libraries
 	$(MAKE) deploy-modules
@@ -102,3 +109,11 @@ create-package:
 	echo '{"latest_version": "$(PACKAGE_VERSION)"}' > $(DIST_DIR)/metadata.json
 
 extra-package::
+
+deploy-native-app-package:
+	$(MAKE) build-native-app-setup-script
+	$(MAKE) -C native_app build
+	$(MAKE) -C native_app deploy-app-package
+
+deploy-native-app:
+	$(MAKE) -C native_app deploy-app

clouds/snowflake/README.md

@@ -38,6 +38,7 @@ SF_SHARE=<share>  # optional
   - `doc`: contains the functions' documentation
   - `sql`: contains the functions' SQL code
   - `test`: contains the functions' tests
+- `native_app`
 
 ## Make commands
 
@@ -49,12 +50,14 @@ SF_SHARE=<share>  # optional
 - `make remove`: removes the SQL scripts from the Snowflake database
 - `make clean`: cleans the installed dependencies and generated files locally
 - `make create-package`: creates the installation package in the dist folder (zip)
+- `make deploy-native-app-package`: builds the JS libraries and SQL scripts and deploys a native app package. When the new version does not imply a major version change a patch is deployed.
+- `make deploy-native-app`: deploys a native app from a deployed native app package or upgrade it if already exists.
 
 Make commands can be run also inside `libraries/javascript` and `modules` folders, or be called like `make lint-libraries`, `make deploy-modules`.
 
 **Filtering**
 
-Commands `build-libraries`, `build-modules`, `deploy-modules`, `test-libraries`, `test-modules` and `create-package` can be filtered by the following. All the filters are additive:
+Commands `build-libraries`, `build-modules`, `deploy-modules`, `test-libraries`, `test-modules`, `create-package` and `deploy-native-app-package` can be filtered by the following. All the filters are additive:
 
 - `diff`: list of changed files
 - `modules`: list of modules to filter
@@ -68,6 +71,8 @@ make build-modules diff=modules/sql/quadbin/QUADBIN_RESOLUTION.sql
 make deploy-modules modules=quadbin,constructors
 make test-modules functions=ST_MAKEENVELOPE
 make create-package modules=quadbin
+make deploy-native-app-package modules=quadbin
+make deploy-native-app
 ```
 
 Command `test-libraries` can be also filtered by setting the `test` variable with a path of the test file. It supports passing the name of the test. Note that `build-libraries` will rebuild the libraries to make them suitable for testing.

clouds/snowflake/common/Makefile

@@ -13,8 +13,10 @@ endif
 
 ifeq ($(production),1)
 export SF_SCHEMA = $(SF_DATABASE).$(SF_SCHEMA_DEFAULT)
+export SF_UNQUALIFIED_SCHEMA = $(SF_SCHEMA_DEFAULT)
 else
 export SF_SCHEMA = $(SF_DATABASE).$(SF_PREFIX)$(SF_SCHEMA_DEFAULT)
+export SF_UNQUALIFIED_SCHEMA = $(SF_PREFIX)$(SF_SCHEMA_DEFAULT)
 endif
 
 .PHONY: check venv3 $(NODE_MODULES_DEV)

clouds/snowflake/common/build_native_app_setup_script.js

@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+
+// Build the setup_script for the native app file based on the input filters
+// and ordered to solve the dependencies
+
+// ./build_native_app_setup_script.js modules --output=build --diff="clouds/snowflake/modules/sql/quadbin/QUADBIN_TOZXY.sql"
+// ./build_native_app_setup_script.js modules --output=build --functions=ST_TILEENVELOPE
+// ./build_native_app_setup_script.js modules --output=build --modules=quadbin
+// ./build_native_app_setup_script.js modules --output=build --production --dropfirst
+
+const fs = require('fs');
+const path = require('path');
+const argv = require('minimist')(process.argv.slice(2));
+
+const inputDirs = argv._[0] && argv._[0].split(',');
+const outputDir = argv.output || 'build';
+const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build';
+const diff = argv.diff || [];
+const nodeps = argv.nodeps;
+let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
+let functionsFilter = (argv.functions && argv.functions.split(',')) || [];
+let all = !(diff.length || modulesFilter.length || functionsFilter.length);
+
+if (all) {
+    console.log('- Build all');
+} else if (diff && diff.length) {
+    console.log(`- Build input diff: ${argv.diff}`);
+} else if (modulesFilter && modulesFilter.length) {
+    console.log(`- Build input modules: ${argv.modules}`);
+} else if (functionsFilter && functionsFilter.length) {
+    console.log(`- Build input functions: ${argv.functions}`);
+}
+
+// Convert diff to modules
+if (diff.length) {
+    const patternsAll = [
+        /\.github\/workflows\/snowflake\.yml/,
+        /clouds\/snowflake\/common\/.+/,
+        /clouds\/snowflake\/libraries\/.+/,
+        /clouds\/snowflake\/.*Makefile/,
+        /clouds\/snowflake\/version/
+    ];
+    const patternModulesSql = /clouds\/snowflake\/modules\/sql\/([^\s]*?)\//g;
+    const patternModulesTest = /clouds\/snowflake\/modules\/test\/([^\s]*?)\//g;
+    const diffAll = patternsAll.some(p => diff.match(p));
+    if (diffAll) {
+        console.log('-- all');
+        all = diffAll;
+    } else {
+        const modulesSql = [...diff.matchAll(patternModulesSql)].map(m => m[1]);
+        const modulesTest = [...diff.matchAll(patternModulesTest)].map(m => m[1]);
+        const diffModulesFilter = [...new Set(modulesSql.concat(modulesTest))];
+        if (diffModulesFilter) {
+            console.log(`-- modules: ${diffModulesFilter}`);
+            modulesFilter = diffModulesFilter;
+        }
+    }
+}
+
+// Extract functions
+const functions = [];
+for (let inputDir of inputDirs) {
+    const sqldir = path.join(inputDir, 'sql');
+    const modules = fs.readdirSync(sqldir);
+    modules.forEach(module => {
+        const moduledir = path.join(sqldir, module);
+        if (fs.statSync(moduledir).isDirectory()) {
+            const files = fs.readdirSync(moduledir);
+            files.forEach(file => {
+                if (file.endsWith('.sql')) {
+                    const name = path.parse(file).name;
+                    const content = fs.readFileSync(path.join(moduledir, file)).toString().replace(/--.*\n/g, '');
+                    functions.push({
+                        name,
+                        module,
+                        content,
+                        dependencies: []
+                    });
+                }
+            });
+        }
+    });
+}
+
+// Check filters
+modulesFilter.forEach(m => {
+    if (!functions.map(fn => fn.module).includes(m)) {
+        console.log(`ERROR: Module not found ${m}`);
+        process.exit(1);
+    }
+});
+functionsFilter.forEach(f => {
+    if (!functions.map(fn => fn.name).includes(f)) {
+        console.log(`ERROR: Function not found ${f}`);
+        process.exit(1);
+    }
+});
+
+// Extract function dependencies
+if (!nodeps) {
+    functions.forEach(mainFunction => {
+        functions.forEach(depFunction => {
+            if (mainFunction.name != depFunction.name) {
+                if (mainFunction.content.includes(`SCHEMA@@.${depFunction.name}(`)) {
+                    mainFunction.dependencies.push(depFunction.name);
+                }
+            }
+        });
+    });
+}
+
+// Check circular dependencies
+if (!nodeps) {
+    functions.forEach(mainFunction => {
+        functions.forEach(depFunction => {
+            if (mainFunction.dependencies.includes(depFunction.name) &&
+                depFunction.dependencies.includes(mainFunction.name)) {
+                console.log(`ERROR: Circular dependency between ${mainFunction.name} and ${depFunction.name}`);
+                process.exit(1);
+            }
+        });
+    });
+}
+
+// Filter and order functions
+const output = [];
+function add (f, include) {
+    include = include || all || functionsFilter.includes(f.name) || modulesFilter.includes(f.module);
+    for (const dependency of f.dependencies) {
+        add(functions.find(f => f.name === dependency), include);
+    }
+    if (!output.map(f => f.name).includes(f.name) && include) {
+        output.push({
+            name: f.name,
+            content: f.content
+        });
+    }
+}
+functions.forEach(f => add(f));
+
+// Replace environment variables
+let separator;
+if (argv.production) {
+    separator = '\n';
+} else {
+    separator = '\n-->\n';  // marker to future SQL split
+}
+let content = output.map(f => fetchPermissionsGrant(f.content)).join(separator);
+
+function apply_replacements (text) {
+    const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))];
+    for (let library of libraries) {
+        const libraryName = library.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js';
+        const libraryPath = path.join(libsBuildDir, libraryName);
+        if (fs.existsSync(libraryPath)) {
+            const libraryContent = fs.readFileSync(libraryPath).toString();
+            text = text.replace(new RegExp(library, 'g'), libraryContent);
+        }
+        else {
+            console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
+            process.exit(1);
+        }
+    }
+    const replacements = process.env.REPLACEMENTS.split(' ');
+    for (let replacement of replacements) {
+        if (replacement) {
+            const pattern = new RegExp(`@@${replacement}@@`, 'g');
+            text = text.replace(pattern, process.env[replacement]);
+        }
+    }
+    return text;
+}
+
+function getFunctionSignatures (functionMatches)
+{
+    const functSignatures = []
+    for (const functionMatch of functionMatches) {
+        //Remove spaces and diacritics
+        let qualifiedFunctName = functionMatch[0].split('(')[0].replace(/\s+/gm,'');
+        qualifiedFunctNameArr = qualifiedFunctName.split('.');
+        const functName = qualifiedFunctNameArr[qualifiedFunctNameArr.length - 1];
+        if (functName.startsWith('_'))
+        {
+            continue;
+        }
+        //Remove diacritics and go greedy to take the outer parentheses
+        let functArgs = functionMatch[0].matchAll(new RegExp('(?<=\\()(.*)(?=\\))','g')).next().value;
+        if (functArgs)
+        {
+            functArgs = functArgs[0];
+        }
+        else
+        {
+            continue;
+        }
+        functArgs = functArgs.split(',')
+        let functArgsTypes = [];
+        for (const functArg of functArgs) {
+            const functArgSplitted = functArg.trim(' ').split(' ');
+            functArgsTypes.push(functArgSplitted[functArgSplitted.length - 1]);
+        }
+        const functSignature = qualifiedFunctName + '(' + functArgsTypes.join(',') + ')';
+        functSignatures.push(functSignature)
+    }
+    return functSignatures
+}
+
+function fetchPermissionsGrant (content)
+{
+    let fileContent = content.split('\n');
+    for (let i = 0 ; i < fileContent.length; i++)
+    {
+        if (fileContent[i].startsWith('--'))
+        {
+            delete fileContent[i];
+        }
+    }
+    fileContent = fileContent.join(' ').replace(/[\p{Diacritic}]/gu, '').replace(/\s+/gm,' ');
+    const functionMatches = fileContent.matchAll(new RegExp(/(?<=(?<!TEMP )FUNCTION)(.*?)(?=RETURNS)/gs));
+    const functSignatures = getFunctionSignatures(functionMatches).map(f => `GRANT USAGE ON FUNCTION ${f} TO APPLICATION ROLE @@APP_ROLE@@;`).join('\n')
+    const procMatches = fileContent.matchAll(new RegExp(/(?<=PROCEDURE)(.*?)(?=AS)/gs));
+    const procSignatures = getFunctionSignatures(procMatches).map(f => `GRANT USAGE ON PROCEDURE ${f} TO APPLICATION ROLE @@APP_ROLE@@;`).join('\n')
+    return content + functSignatures +procSignatures
+}
+
+if (argv.dropfirst) {
+    const header = fs.readFileSync(path.resolve(__dirname, 'DROP_FUNCTIONS.sql')).toString();
+    content = header + separator + content
+}
+
+const header = `CREATE OR REPLACE APPLICATION ROLE @@APP_ROLE@@;
+CREATE OR ALTER VERSIONED SCHEMA @@SF_SCHEMA@@;
+GRANT USAGE ON SCHEMA @@SF_SCHEMA@@ TO APPLICATION ROLE @@APP_ROLE@@;\n`;
+
+const footer = fetchPermissionsGrant (fs.readFileSync(path.resolve(__dirname, 'VERSION.sql')).toString());
+content = header + separator + content + separator + footer;
+
+content = apply_replacements(content);
+
+// Execute as caller replacement
+content = content.replace(/EXECUTE\s+AS\s+CALLER/g, 'EXECUTE AS OWNER');
+
+// Write setup_script.sql file
+fs.writeFileSync(path.join(outputDir, 'setup_script.sql'), content);
+console.log(`Write ${outputDir}/setup_script.sql`);