diff --git a/.github/workflows/snowflake.yml b/.github/workflows/snowflake.yml index 8c37e4726..cbb102107 100644 --- a/.github/workflows/snowflake.yml +++ b/.github/workflows/snowflake.yml @@ -15,7 +15,7 @@ on: workflow_call: env: - NODE_VERSION: 14 + NODE_VERSION: 16 PYTHON3_VERSION: 3.8.10 VIRTUALENV_VERSION: 20.21.1 GCLOUD_VERSION: 290.0.1 @@ -88,6 +88,37 @@ jobs: cd clouds/snowflake make deploy diff="$GIT_DIFF" production=1 + deploy-internal-app: + if: github.ref_name == 'main' + needs: test + runs-on: ubuntu-20.04 + timeout-minutes: 20 + env: + APP_PACKAGE_NAME: ${{ secrets.SF_NATIVE_APP_PACKAGE_NAME_CD }} + APP_NAME: ${{ secrets.SF_NATIVE_APP_NAME_CD }} + SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_NATIVE_APP }} + SF_DATABASE: ${{ secrets.SF_DATABASE_NATIVE_APP }} + SF_USER: ${{ secrets.SF_USER_NATIVE_APP }} + SF_PASSWORD: ${{ secrets.SF_PASSWORD_NATIVE_APP }} + SF_ROLE: ${{ secrets.SF_ROLE_NATIVE_APP }} + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Check diff + uses: technote-space/get-diff-action@v4 + - name: Setup node + uses: actions/setup-node@v1 + with: + node-version: ${{ env.NODE_VERSION }} + - name: Deploy native app package + run: | + cd clouds/snowflake + make deploy-native-app-package production=1 + - name: Deploy native app locally + run: | + cd clouds/snowflake + make deploy-native-app production=1 + deploy-share: if: github.ref_name == 'stable' needs: test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 551567508..90c5e1b75 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -221,4 +221,9 @@ in the case of BigQuery, the backtick quotes. For example: Extra dependencies: `@@BQ_DATASET@@.SOME_FUNCTION`() */ -``` \ No newline at end of file +``` + +## Known Limitations +### Snowflake + +Due to Snowflake Native Apps limitations at this moment TEMPORARY tables cannot be used within procedures. For the time being create normal tables and ensure that they are dropped. \ No newline at end of file diff --git a/clouds/snowflake/Makefile b/clouds/snowflake/Makefile index b61dccc75..391bd6391 100644 --- a/clouds/snowflake/Makefile +++ b/clouds/snowflake/Makefile @@ -13,10 +13,10 @@ include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint build deploy test remove clean create-package +.PHONY: help lint build deploy test remove clean create-package deploy-native-app-package deploy-native-app help: - echo "Available targets: lint build deploy test remove clean create-package" + echo "Available targets: lint build deploy test remove clean create-package deploy-native-app-package deploy-native-app" lint: $(MAKE) lint-libraries @@ -54,6 +54,13 @@ build-modules: $(MAKE) -C modules build cp modules/build/modules.sql $(BUILD_DIR) +build-native-app-setup-script: + rm -rf $(BUILD_DIR) + $(MAKE) build-libraries + mkdir -p $(BUILD_DIR) + $(MAKE) -C modules build-native-app-setup-script + cp modules/build/setup_script.sql $(BUILD_DIR) + deploy: $(MAKE) build-libraries $(MAKE) deploy-modules @@ -102,3 +109,11 @@ create-package: echo '{"latest_version": "$(PACKAGE_VERSION)"}' > $(DIST_DIR)/metadata.json extra-package:: + +deploy-native-app-package: + $(MAKE) build-native-app-setup-script + $(MAKE) -C native_app build + $(MAKE) -C native_app deploy-app-package + +deploy-native-app: + $(MAKE) -C native_app deploy-app diff --git a/clouds/snowflake/README.md b/clouds/snowflake/README.md index 623c3fa00..d87854d61 100644 --- a/clouds/snowflake/README.md +++ b/clouds/snowflake/README.md @@ -38,6 +38,7 @@ SF_SHARE= # optional - `doc`: contains the functions' documentation - `sql`: contains the functions' SQL code - `test`: contains the functions' tests +- `native_app` ## Make commands @@ -49,12 +50,14 @@ SF_SHARE= # optional - `make remove`: removes the SQL scripts from the Snowflake database - `make clean`: cleans the installed dependencies and generated files locally - `make create-package`: creates the installation package in the dist folder (zip) +- `make deploy-native-app-package`: builds the JS libraries and SQL scripts and deploys a native app package. When the new version does not imply a major version change a patch is deployed. +- `make deploy-native-app`: deploys a native app from a deployed native app package or upgrade it if already exists. Make commands can be run also inside `libraries/javascript` and `modules` folders, or be called like `make lint-libraries`, `make deploy-modules`. **Filtering** -Commands `build-libraries`, `build-modules`, `deploy-modules`, `test-libraries`, `test-modules` and `create-package` can be filtered by the following. All the filters are additive: +Commands `build-libraries`, `build-modules`, `deploy-modules`, `test-libraries`, `test-modules`, `create-package` and `deploy-native-app-package` can be filtered by the following. All the filters are additive: - `diff`: list of changed files - `modules`: list of modules to filter @@ -68,6 +71,8 @@ make build-modules diff=modules/sql/quadbin/QUADBIN_RESOLUTION.sql make deploy-modules modules=quadbin,constructors make test-modules functions=ST_MAKEENVELOPE make create-package modules=quadbin +make deploy-native-app-package modules=quadbin +make deploy-native-app ``` Command `test-libraries` can be also filtered by setting the `test` variable with a path of the test file. It supports passing the name of the test. Note that `build-libraries` will rebuild the libraries to make them suitable for testing. diff --git a/clouds/snowflake/common/Makefile b/clouds/snowflake/common/Makefile index 1935c8dbe..5b1d5f00c 100644 --- a/clouds/snowflake/common/Makefile +++ b/clouds/snowflake/common/Makefile @@ -13,8 +13,10 @@ endif ifeq ($(production),1) export SF_SCHEMA = $(SF_DATABASE).$(SF_SCHEMA_DEFAULT) +export SF_UNQUALIFIED_SCHEMA = $(SF_SCHEMA_DEFAULT) else export SF_SCHEMA = $(SF_DATABASE).$(SF_PREFIX)$(SF_SCHEMA_DEFAULT) +export SF_UNQUALIFIED_SCHEMA = $(SF_PREFIX)$(SF_SCHEMA_DEFAULT) endif .PHONY: check venv3 $(NODE_MODULES_DEV) diff --git a/clouds/snowflake/common/build_native_app_setup_script.js b/clouds/snowflake/common/build_native_app_setup_script.js new file mode 100755 index 000000000..6db6f6f2f --- /dev/null +++ b/clouds/snowflake/common/build_native_app_setup_script.js @@ -0,0 +1,245 @@ +#!/usr/bin/env node + +// Build the setup_script for the native app file based on the input filters +// and ordered to solve the dependencies + +// ./build_native_app_setup_script.js modules --output=build --diff="clouds/snowflake/modules/sql/quadbin/QUADBIN_TOZXY.sql" +// ./build_native_app_setup_script.js modules --output=build --functions=ST_TILEENVELOPE +// ./build_native_app_setup_script.js modules --output=build --modules=quadbin +// ./build_native_app_setup_script.js modules --output=build --production --dropfirst + +const fs = require('fs'); +const path = require('path'); +const argv = require('minimist')(process.argv.slice(2)); + +const inputDirs = argv._[0] && argv._[0].split(','); +const outputDir = argv.output || 'build'; +const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build'; +const diff = argv.diff || []; +const nodeps = argv.nodeps; +let modulesFilter = (argv.modules && argv.modules.split(',')) || []; +let functionsFilter = (argv.functions && argv.functions.split(',')) || []; +let all = !(diff.length || modulesFilter.length || functionsFilter.length); + +if (all) { + console.log('- Build all'); +} else if (diff && diff.length) { + console.log(`- Build input diff: ${argv.diff}`); +} else if (modulesFilter && modulesFilter.length) { + console.log(`- Build input modules: ${argv.modules}`); +} else if (functionsFilter && functionsFilter.length) { + console.log(`- Build input functions: ${argv.functions}`); +} + +// Convert diff to modules +if (diff.length) { + const patternsAll = [ + /\.github\/workflows\/snowflake\.yml/, + /clouds\/snowflake\/common\/.+/, + /clouds\/snowflake\/libraries\/.+/, + /clouds\/snowflake\/.*Makefile/, + /clouds\/snowflake\/version/ + ]; + const patternModulesSql = /clouds\/snowflake\/modules\/sql\/([^\s]*?)\//g; + const patternModulesTest = /clouds\/snowflake\/modules\/test\/([^\s]*?)\//g; + const diffAll = patternsAll.some(p => diff.match(p)); + if (diffAll) { + console.log('-- all'); + all = diffAll; + } else { + const modulesSql = [...diff.matchAll(patternModulesSql)].map(m => m[1]); + const modulesTest = [...diff.matchAll(patternModulesTest)].map(m => m[1]); + const diffModulesFilter = [...new Set(modulesSql.concat(modulesTest))]; + if (diffModulesFilter) { + console.log(`-- modules: ${diffModulesFilter}`); + modulesFilter = diffModulesFilter; + } + } +} + +// Extract functions +const functions = []; +for (let inputDir of inputDirs) { + const sqldir = path.join(inputDir, 'sql'); + const modules = fs.readdirSync(sqldir); + modules.forEach(module => { + const moduledir = path.join(sqldir, module); + if (fs.statSync(moduledir).isDirectory()) { + const files = fs.readdirSync(moduledir); + files.forEach(file => { + if (file.endsWith('.sql')) { + const name = path.parse(file).name; + const content = fs.readFileSync(path.join(moduledir, file)).toString().replace(/--.*\n/g, ''); + functions.push({ + name, + module, + content, + dependencies: [] + }); + } + }); + } + }); +} + +// Check filters +modulesFilter.forEach(m => { + if (!functions.map(fn => fn.module).includes(m)) { + console.log(`ERROR: Module not found ${m}`); + process.exit(1); + } +}); +functionsFilter.forEach(f => { + if (!functions.map(fn => fn.name).includes(f)) { + console.log(`ERROR: Function not found ${f}`); + process.exit(1); + } +}); + +// Extract function dependencies +if (!nodeps) { + functions.forEach(mainFunction => { + functions.forEach(depFunction => { + if (mainFunction.name != depFunction.name) { + if (mainFunction.content.includes(`SCHEMA@@.${depFunction.name}(`)) { + mainFunction.dependencies.push(depFunction.name); + } + } + }); + }); +} + +// Check circular dependencies +if (!nodeps) { + functions.forEach(mainFunction => { + functions.forEach(depFunction => { + if (mainFunction.dependencies.includes(depFunction.name) && + depFunction.dependencies.includes(mainFunction.name)) { + console.log(`ERROR: Circular dependency between ${mainFunction.name} and ${depFunction.name}`); + process.exit(1); + } + }); + }); +} + +// Filter and order functions +const output = []; +function add (f, include) { + include = include || all || functionsFilter.includes(f.name) || modulesFilter.includes(f.module); + for (const dependency of f.dependencies) { + add(functions.find(f => f.name === dependency), include); + } + if (!output.map(f => f.name).includes(f.name) && include) { + output.push({ + name: f.name, + content: f.content + }); + } +} +functions.forEach(f => add(f)); + +// Replace environment variables +let separator; +if (argv.production) { + separator = '\n'; +} else { + separator = '\n-->\n'; // marker to future SQL split +} +let content = output.map(f => fetchPermissionsGrant(f.content)).join(separator); + +function apply_replacements (text) { + const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))]; + for (let library of libraries) { + const libraryName = library.replace('@@SF_LIBRARY_', '').replace('@@', '').toLowerCase() + '.js'; + const libraryPath = path.join(libsBuildDir, libraryName); + if (fs.existsSync(libraryPath)) { + const libraryContent = fs.readFileSync(libraryPath).toString(); + text = text.replace(new RegExp(library, 'g'), libraryContent); + } + else { + console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`); + process.exit(1); + } + } + const replacements = process.env.REPLACEMENTS.split(' '); + for (let replacement of replacements) { + if (replacement) { + const pattern = new RegExp(`@@${replacement}@@`, 'g'); + text = text.replace(pattern, process.env[replacement]); + } + } + return text; +} + +function getFunctionSignatures (functionMatches) +{ + const functSignatures = [] + for (const functionMatch of functionMatches) { + //Remove spaces and diacritics + let qualifiedFunctName = functionMatch[0].split('(')[0].replace(/\s+/gm,''); + qualifiedFunctNameArr = qualifiedFunctName.split('.'); + const functName = qualifiedFunctNameArr[qualifiedFunctNameArr.length - 1]; + if (functName.startsWith('_')) + { + continue; + } + //Remove diacritics and go greedy to take the outer parentheses + let functArgs = functionMatch[0].matchAll(new RegExp('(?<=\\()(.*)(?=\\))','g')).next().value; + if (functArgs) + { + functArgs = functArgs[0]; + } + else + { + continue; + } + functArgs = functArgs.split(',') + let functArgsTypes = []; + for (const functArg of functArgs) { + const functArgSplitted = functArg.trim(' ').split(' '); + functArgsTypes.push(functArgSplitted[functArgSplitted.length - 1]); + } + const functSignature = qualifiedFunctName + '(' + functArgsTypes.join(',') + ')'; + functSignatures.push(functSignature) + } + return functSignatures +} + +function fetchPermissionsGrant (content) +{ + let fileContent = content.split('\n'); + for (let i = 0 ; i < fileContent.length; i++) + { + if (fileContent[i].startsWith('--')) + { + delete fileContent[i]; + } + } + fileContent = fileContent.join(' ').replace(/[\p{Diacritic}]/gu, '').replace(/\s+/gm,' '); + const functionMatches = fileContent.matchAll(new RegExp(/(?<=(? `GRANT USAGE ON FUNCTION ${f} TO APPLICATION ROLE @@APP_ROLE@@;`).join('\n') + const procMatches = fileContent.matchAll(new RegExp(/(?<=PROCEDURE)(.*?)(?=AS)/gs)); + const procSignatures = getFunctionSignatures(procMatches).map(f => `GRANT USAGE ON PROCEDURE ${f} TO APPLICATION ROLE @@APP_ROLE@@;`).join('\n') + return content + functSignatures +procSignatures +} + +if (argv.dropfirst) { + const header = fs.readFileSync(path.resolve(__dirname, 'DROP_FUNCTIONS.sql')).toString(); + content = header + separator + content +} + +const header = `CREATE OR REPLACE APPLICATION ROLE @@APP_ROLE@@; +CREATE OR ALTER VERSIONED SCHEMA @@SF_SCHEMA@@; +GRANT USAGE ON SCHEMA @@SF_SCHEMA@@ TO APPLICATION ROLE @@APP_ROLE@@;\n`; + +const footer = fetchPermissionsGrant (fs.readFileSync(path.resolve(__dirname, 'VERSION.sql')).toString()); +content = header + separator + content + separator + footer; + +content = apply_replacements(content); + +// Execute as caller replacement +content = content.replace(/EXECUTE\s+AS\s+CALLER/g, 'EXECUTE AS OWNER'); + +// Write setup_script.sql file +fs.writeFileSync(path.join(outputDir, 'setup_script.sql'), content); +console.log(`Write ${outputDir}/setup_script.sql`); \ No newline at end of file diff --git a/clouds/snowflake/common/native-app-utils.js b/clouds/snowflake/common/native-app-utils.js new file mode 100755 index 000000000..5fff5bbfb --- /dev/null +++ b/clouds/snowflake/common/native-app-utils.js @@ -0,0 +1,151 @@ +#!/usr/bin/env node + +// Perform operations related with the versioning of the native app + +// -- Set the app package release to the max patch of a given version. This is the version that will be used by default by customers. +// ./native-app-utils.js SET_PACKAGE_RELEASE=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) VERSION=$(APP_MAJOR_VERSION) + +// -- Drop the previous version of the app package. This is done to avoid having more than two versions of the package. +// ./native-app-utils.js DROP_PREVIOUS_VERSION=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) VERSION=$(APP_MAJOR_VERSION) + +// -- Check if a given version exists in the app package. This is checked in order to deploy a version or a patch +// ./native-app-utils.js CHECK_VERSION_EXISTENCE=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) VERSION=$(APP_MAJOR_VERSION) + +// -- Count the number of versions in an app package. This is checked in order to drop a version if necessary +// ./native-app-utils.js COUNT_VERSIONS=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) + +// -- Check if a given app package exists. This is checked in order to create a package or not +// ./native-app-utils.js CHECK_APP_PACKAGE_EXISTENCE=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) + +// -- Check if a given app exists. This is checked in order to create an app or launch an upgrade +// ./native-app-utils.js CHECK_APP_EXISTENCE=1 APP_NAME=$(APP_NAME) + +const snowflake = require('snowflake-sdk'); + +snowflake.configure({ insecureConnect: true }); + +const connection = snowflake.createConnection({ + account: process.env.SF_ACCOUNT, + username: process.env.SF_USER, + password: process.env.SF_PASSWORD, + role: process.env.SF_ROLE +}); + +connection.connect((err) => { + if (err) { + console.error(`Unable to connect: ${err.message}`); + } +}); + +function runSetRelease (err, stmt, rows) { + // Get the patches for a given version + patchesArr = rows.filter(obj => obj['version'] === process.env.VERSION.toUpperCase()) + // Get the max patch number for a given version + maxPatch = patchesArr.reduce((maxObj, currentObj) => { + return currentObj['patch'] > maxObj['patch'] ? currentObj : maxObj; + }, patchesArr[0])['patch'] + const query = ` + ALTER APPLICATION PACKAGE ${process.env.APP_PACKAGE_NAME} + SET DEFAULT RELEASE DIRECTIVE + VERSION = ${process.env.VERSION} + PATCH = ${maxPatch};` + connection.execute({ + sqlText: query, + complete: (err, stmt, rows) => { + if (err) { + console.log(err.message); + } + } + }); +} + +function runDropPreviousVersionQuery (err, stmt, rows) { + // Get the patches different of a given version + patchesArr = rows.filter(obj => obj['version'] !== process.env.VERSION.toUpperCase()) + if (patchesArr.length > 0) { + // As there can only be two versions we take any patch + const previousVersion = patchesArr[0]['version'] + const query = ` + ALTER APPLICATION PACKAGE ${process.env.APP_PACKAGE_NAME} + DROP VERSION ${previousVersion};` + connection.execute({ + sqlText: query, + complete: (err, stmt, rows) => { + if (err) { + console.log(err.message); + } + } + }); + } +} + +function runCheckVersionExistenceQuery (err, stmt, rows) { + // Get the different patches for a given version + patchesArr = rows.filter(obj => obj['version'] === process.env.VERSION.toUpperCase()) + if (patchesArr.length > 0) { + console.log('1') + } else { + console.log('0') + } +} + +function runCountVersionsQuery (err, stmt, rows) { + uniqueVersions = new Set(rows.map(obj => obj['version'])) + console.log(uniqueVersions.size) +} + +function runGetVersionsQuery (callback) { + const query = `SHOW VERSIONS IN APPLICATION PACKAGE ${process.env.APP_PACKAGE_NAME};` + connection.execute({ + sqlText: query, + complete: callback + }); +} + +function runCheckAppPackageExistenceQuery (err, stmt, rows) { + packagesArr = rows.filter(obj => obj['name'] === process.env.APP_PACKAGE_NAME.toUpperCase()) + if (packagesArr.length > 0) { + console.log('1') + } else { + console.log('0') + } +} + +function runGetAppPackagesQuery (callback) { + const query = 'SHOW APPLICATION PACKAGES;' + connection.execute({ + sqlText: query, + complete: callback + }); +} + +function runCheckAppExistenceQuery (err, stmt, rows) { + packagesArr = rows.filter(obj => obj['name'] === process.env.APP_NAME.toUpperCase()) + if (packagesArr.length > 0) { + console.log('1') + } else { + console.log('0') + } +} + +function runGetAppsQuery (callback) { + const query = 'SHOW APPLICATIONS;' + connection.execute({ + sqlText: query, + complete: callback + }); +} + +if (process.env.SET_PACKAGE_RELEASE) { + runGetVersionsQuery(runSetRelease) +} else if (process.env.DROP_PREVIOUS_VERSION) { + runGetVersionsQuery (runDropPreviousVersionQuery) +} else if (process.env.CHECK_VERSION_EXISTENCE) { + runGetVersionsQuery (runCheckVersionExistenceQuery) +} else if (process.env.COUNT_VERSIONS) { + runGetVersionsQuery (runCountVersionsQuery) +} else if (process.env.CHECK_APP_PACKAGE_EXISTENCE) { + runGetAppPackagesQuery(runCheckAppPackageExistenceQuery) +} else if (process.env.CHECK_APP_EXISTENCE) { + runGetAppsQuery(runCheckAppExistenceQuery) +} \ No newline at end of file diff --git a/clouds/snowflake/common/package.json b/clouds/snowflake/common/package.json index d25a03ddb..f852c7570 100644 --- a/clouds/snowflake/common/package.json +++ b/clouds/snowflake/common/package.json @@ -11,6 +11,6 @@ "rollup": "^2.47.0", "rollup-plugin-bundle-size": "^1.0.3", "rollup-plugin-terser": "^7.0.2", - "snowflake-sdk": "^1.6.0" + "snowflake-sdk": "^1.6.6" } } diff --git a/clouds/snowflake/modules/Makefile b/clouds/snowflake/modules/Makefile index 02f2be59f..dc1df1988 100644 --- a/clouds/snowflake/modules/Makefile +++ b/clouds/snowflake/modules/Makefile @@ -25,13 +25,13 @@ else BAIL=--bail endif -REPLACEMENTS = "SF_SCHEMA SF_VERSION_FUNCTION SF_PACKAGE_VERSION SF_SHARE" +REPLACEMENTS = "SF_SCHEMA SF_VERSION_FUNCTION SF_PACKAGE_VERSION SF_SHARE APP_ROLE" include $(COMMON_DIR)/Makefile .SILENT: -.PHONY: help lint build build-share deploy deploy-share test remove remove-share clean +.PHONY: help lint build build-share build-native-app-setup-script deploy deploy-share test remove remove-share clean help: echo "Available targets: lint build deploy test remove clean" @@ -68,6 +68,16 @@ build-share: $(NODE_MODULES_DEV) --output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --diff="$(diff)" \ --modules=$(modules) --functions=$(functions) --production=$(production) --nodeps=$(nodeps) --dropfirst=$(dropfirst) +build-native-app-setup-script: $(NODE_MODULES_DEV) + echo "Building native app setup script..." + rm -rf $(BUILD_DIR) + mkdir $(BUILD_DIR) + SF_SCHEMA=$(SF_UNQUALIFIED_SCHEMA) APP_ROLE=app_public \ + REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \ + $(COMMON_DIR)/build_native_app_setup_script.js $(MODULES_DIRS) \ + --output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --diff="$(diff)" \ + --modules=$(modules) --functions=$(functions) --production=$(production) --nodeps=$(nodeps) --dropfirst=1 + deploy: check build echo "Deploying modules..." $(COMMON_DIR)/run-query.js "CREATE SCHEMA IF NOT EXISTS $(SF_SCHEMA);" diff --git a/clouds/snowflake/native_app/Makefile b/clouds/snowflake/native_app/Makefile new file mode 100644 index 000000000..beab7de0f --- /dev/null +++ b/clouds/snowflake/native_app/Makefile @@ -0,0 +1,101 @@ +# Makefile native app for Snowflake + +ROOT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) + +DIST_DIR ?= $(ROOT_DIR)/../dist +ENV_DIR ?= $(ROOT_DIR)/.. +BUILD_DIR ?= $(ROOT_DIR)/../build +COMMON_DIR = $(ROOT_DIR)/../common +APP_DIR ?= $(ROOT_DIR) + +include $(COMMON_DIR)/Makefile + +APP_NAME ?= APP_$(SF_UNQUALIFIED_SCHEMA) +APP_PACKAGE_NAME ?= APP_PACKAGE_$(SF_UNQUALIFIED_SCHEMA) +APP_PACKAGE_DIST_NAME ?= carto-analytics-toolbox-core-snowflake-native-app-$(APP_VERSION) +APP_VERSION ?= $(shell cat $(ROOT_DIR)/../version) +APP_MAJOR_VERSION = v$(firstword $(subst ., ,$(APP_VERSION))) +APP_FORMATTED_VERSION = v$(subst .,_,$(APP_VERSION)) + +APP_STAGE_NAME ?= $(SF_SCHEMA).$(APP_NAME) +ifeq ($(production),1) + APP_STAGE_LOCATION ?= $(APP_STAGE_NAME)/$(APP_FORMATTED_VERSION) +else + APP_STAGE_LOCATION ?= $(APP_STAGE_NAME) +endif + +.SILENT: + +.PHONY: help build deploy-app-package deploy-app drop-app-package drop-app + +help: + echo "Available targets: help build deploy-app-package deploy-app drop-app-package drop-app" + +build: + rm -rf $(DIST_DIR) + mkdir -p $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME) + cp $(BUILD_DIR)/setup_script.sql $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/ + sed 's/@@VERSION@@/$(APP_VERSION)/g' $(APP_DIR)/manifest.yml > $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/manifest.yml + cp $(APP_DIR)/README.md $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/ + + +deploy-app-package: + $(COMMON_DIR)/run-query.js "CREATE STAGE IF NOT EXISTS $(APP_STAGE_NAME);" + $(COMMON_DIR)/run-query.js "PUT file://$(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/* @$(APP_STAGE_LOCATION) overwrite=true auto_compress=false;" + + result=$$(CHECK_APP_PACKAGE_EXISTENCE=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) $(COMMON_DIR)/native-app-utils.js) && \ + if [ $${result} -eq 0 ]; then \ + echo "Creating native app package..."; \ + $(COMMON_DIR)/run-query.js "CREATE APPLICATION PACKAGE $(APP_PACKAGE_NAME);"; \ + fi + + result=$$(CHECK_VERSION_EXISTENCE=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) VERSION=$(APP_MAJOR_VERSION) $(COMMON_DIR)/native-app-utils.js) && \ + if [ $${result} -eq 1 ]; then \ + echo "Deploying native app package patch..."; \ + $(COMMON_DIR)/run-query.js "ALTER APPLICATION PACKAGE $(APP_PACKAGE_NAME) \ + ADD PATCH FOR VERSION $(APP_MAJOR_VERSION) \ + USING @$(APP_STAGE_LOCATION);"; \ + else \ + echo "Deploying native app package version..."; \ + $(COMMON_DIR)/run-query.js "ALTER APPLICATION PACKAGE $(APP_PACKAGE_NAME) \ + ADD VERSION $(APP_MAJOR_VERSION) \ + USING @$(APP_STAGE_LOCATION);"; \ + fi + + echo "Setting release package to the newest patch..." + SET_PACKAGE_RELEASE=1 \ + APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) \ + VERSION=$(APP_MAJOR_VERSION) \ + $(COMMON_DIR)/native-app-utils.js + +# This ensures that there are no more than 2 versions which is a Snowflake limitations + result=$$(COUNT_VERSIONS=1 APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) $(COMMON_DIR)/native-app-utils.js) && \ + if [ $${result} -eq 2 ]; then \ + echo "Requesting previous native app package version drop..."; \ + DROP_PREVIOUS_VERSION=1 \ + APP_PACKAGE_NAME=$(APP_PACKAGE_NAME) \ + VERSION=$(APP_MAJOR_VERSION) \ + $(COMMON_DIR)/native-app-utils.js; \ + fi + +drop-app-package: + echo "Dropping native app package..." + $(COMMON_DIR)/run-query.js "DROP APPLICATION PACKAGE IF EXISTS $(APP_PACKAGE_NAME);" + +deploy-app: + result=$$(CHECK_APP_EXISTENCE=1 APP_NAME=$(APP_NAME) $(COMMON_DIR)/native-app-utils.js) && \ + if [ $${result} -eq 0 ]; then \ + echo "Installing native app... (this may take a while)"; \ + $(COMMON_DIR)/run-query.js "CREATE APPLICATION $(APP_NAME) \ + FROM APPLICATION PACKAGE $(APP_PACKAGE_NAME);"; \ + $(MAKE) extra-app-deploy; \ + else \ + echo "Upgrading native app... (this may take a while)"; \ + $(COMMON_DIR)/run-query.js "ALTER APPLICATION $(APP_NAME) UPGRADE;"; \ + fi + +extra-app-deploy:: + +drop-app: + echo "Dropping native app..." + $(COMMON_DIR)/run-query.js "DROP APPLICATION IF EXISTS $(APP_NAME);" \ No newline at end of file diff --git a/clouds/snowflake/native_app/README.md b/clouds/snowflake/native_app/README.md new file mode 100644 index 000000000..dedecaecf --- /dev/null +++ b/clouds/snowflake/native_app/README.md @@ -0,0 +1,47 @@ +## ANALYTICS TOOLBOX CORE + +### Introduction + +The CARTO Analytics Toolbox for Snowflake is composed of a set of user-defined functions and procedures organized in a set of modules based on the functionality they offer. This app gives you access to the Open Source modules included in the CARTO's Analytics Toolbox, supporting different spatial indexes and other geospatial operations: quadkeys, H3, S2, placekey, geometry constructors, accessors, transformations, etc. + +We recommend that at the moment of installing the app you name the app "CARTO". The next guidelines and examples will assume that in order to simplify the onboarding process. + +### Installation + +#### Grant Usage + +This step is required by procedures that have input tables/queries or output tables. The user will have to manually provide permissions to access the tables of a given database. Providing ALL permissions on an SCHEMA ensures that the app can write new tables in that schema. + +On the other hand, those tables generated by a procedure belong to the app itself. If the user wants to recover control over those tables (performing SELECT, UPDATE, DELETE...), updating the ownership on those tables is required. + +``` +-- Set read and write permissions +GRANT USAGE ON DATABASE TO APPLICATION CARTO; +GRANT ALL ON SCHEMA . TO APPLICATION CARTO; +GRANT SELECT ON TABLE IN SCHEMA .. TO APPLICATION CARTO; + +-- Update ownership (when a table is created within the app) +GRANT OWNERSHIP ON TABLE .. TO ROLE ACCOUNTADMIN; +``` + +### Usage Examples + +Please refer to CARTO's [SQL reference](https://docs.carto.com/data-and-analysis/analytics-toolbox-for-snowflake/sql-reference) to find the full list of available functions and procedures as well as examples. + +#### H3_POLYFILL + +Returns an array with all the H3 cell indexes **with centers** contained in a given polygon. + +``` +SELECT carto.H3_POLYFILL( + TO_GEOGRAPHY('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))'), 4); +-- 842da29ffffffff +-- 843f725ffffffff +-- 843eac1ffffffff +-- 8453945ffffffff +-- ... +``` + +Learn how to visualize the result of these queries in CARTO by visiting [https://docs.carto.com/carto-user-manual/maps/data-sources#add-source-from-a-custom-query](https://docs.carto.com/carto-user-manual/maps/data-sources#add-source-from-a-custom-query). + +Get a CARTO account in [https://app.carto.com/signup](https://app.carto.com/signup). \ No newline at end of file diff --git a/clouds/snowflake/native_app/manifest.yml b/clouds/snowflake/native_app/manifest.yml new file mode 100644 index 000000000..9b8163b12 --- /dev/null +++ b/clouds/snowflake/native_app/manifest.yml @@ -0,0 +1,10 @@ +manifest_version: 1 # required +version: + name: @@VERSION@@ + label: "@@VERSION@@" + comment: "Analytics Toolbox Core" + +artifacts: + readme: README.md + setup_script: setup_script.sql + extension_code: true