diff --git a/.eslintignore b/.eslintignore index 3e3cbbd8966..2d4d2bc62d8 100644 --- a/.eslintignore +++ b/.eslintignore @@ -27,7 +27,11 @@ scripts/skipPrepareScript.js .prettierignore *.json Dockerfile* +*.properties # Ignore Go module files go/**/*.mod -go/**/*.sum \ No newline at end of file +go/**/*.sum + + + diff --git a/.github/workflows/draft-new-release.yml b/.github/workflows/draft-new-release.yml index 602ded980fd..ce38023d6ae 100644 --- a/.github/workflows/draft-new-release.yml +++ b/.github/workflows/draft-new-release.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest # Only allow release stakeholders to initiate releases - if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'saikumarrs' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'shrouti1507' || github.actor == 'anantjain45823' || github.actor == 'chandumlg' || github.actor == 'mihir-4116' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'utsabc') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc') + if: (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/hotfix/')) && (github.actor == 'ItsSudip' || github.actor == 'krishna2020' || github.actor == 'sandeepdsvs' || github.actor == 'koladilip' || github.actor == 'yashasvibajpai' || github.actor == 'sanpj2292' || github.actor == 'utsabc' || github.actor == 'vinayteki95') && (github.triggering_actor == 'ItsSudip' || github.triggering_actor == 'krishna2020' || github.triggering_actor == 'koladilip' || github.triggering_actor == 'saikumarrs' || github.triggering_actor == 'sandeepdsvs' || github.triggering_actor == 'shrouti1507' || github.triggering_actor == 'anantjain45823' || github.triggering_actor == 'chandumlg' || github.triggering_actor == 'mihir-4116' || github.triggering_actor == 'yashasvibajpai' || github.triggering_actor == 'sanpj2292' || github.triggering_actor == 'utsabc') steps: - name: Checkout uses: actions/checkout@v4.2.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index fa45ed9b5b3..2e0c2254396 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,85 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.90.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.90.0...v1.90.1) (2025-02-06) + + +### Bug Fixes + +* update google ads api version for gaoc ([#4047](https://github.com/rudderlabs/rudder-transformer/issues/4047)) ([62705c0](https://github.com/rudderlabs/rudder-transformer/commit/62705c0613f3c38b8f3e6baf047467ce97878625)) + +## [1.90.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.89.1...v1.90.0) (2025-02-03) + + +### Features + +* **http:** enclose constants with quotes in mappings ([#4037](https://github.com/rudderlabs/rudder-transformer/issues/4037)) ([d62ba40](https://github.com/rudderlabs/rudder-transformer/commit/d62ba40ff26d1f4fc27b54f1546094f384ee7266)) +* **http:** minor bug fixes and enhancements related to XML and FORM formats ([#4022](https://github.com/rudderlabs/rudder-transformer/issues/4022)) ([241250d](https://github.com/rudderlabs/rudder-transformer/commit/241250d10e1d8cb904ec188312e1ec5532379e93)) +* onboarding customerio segment destination ([#4028](https://github.com/rudderlabs/rudder-transformer/issues/4028)) ([16b927a](https://github.com/rudderlabs/rudder-transformer/commit/16b927a97b64d3033e33a047429d0bbd15c7e1fd)) + + +### Bug Fixes + +* add phone e164 format validation for klaviyo ([#4025](https://github.com/rudderlabs/rudder-transformer/issues/4025)) ([ea46afe](https://github.com/rudderlabs/rudder-transformer/commit/ea46afef39bc85727a5258737987e4da1104ac3d)) + +### [1.89.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.89.0...v1.89.1) (2025-01-29) + + +### Bug Fixes + +* extend statuscode check to 502 to retry the request ([#4029](https://github.com/rudderlabs/rudder-transformer/issues/4029)) ([d003676](https://github.com/rudderlabs/rudder-transformer/commit/d00367691400402011d624c5b993d0f152191edd)) + +## [1.89.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.88.3...v1.89.0) (2025-01-27) + + +### Features + +* add DATA WAREHOUSE to integrations object in shopify pixel source ([#3980](https://github.com/rudderlabs/rudder-transformer/issues/3980)) ([3c20393](https://github.com/rudderlabs/rudder-transformer/commit/3c20393a0e4e3ec98d316820d187310bfec1faea)), closes [#3973](https://github.com/rudderlabs/rudder-transformer/issues/3973) [#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957) [#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957) +* add redis support in shopify pixel for id stitching ([#4001](https://github.com/rudderlabs/rudder-transformer/issues/4001)) ([23ad10a](https://github.com/rudderlabs/rudder-transformer/commit/23ad10a4bd470f09a75e9230d8c860b703a5a1f9)), closes [#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957) +* add support for form format ([#4000](https://github.com/rudderlabs/rudder-transformer/issues/4000)) ([1fc15bf](https://github.com/rudderlabs/rudder-transformer/commit/1fc15bfd4b88e0b252c780a278376cea4c594057)) +* **http:** resolves bug fixes raised during testing ([#3991](https://github.com/rudderlabs/rudder-transformer/issues/3991)) ([a86f009](https://github.com/rudderlabs/rudder-transformer/commit/a86f0094cbef9428e7aeda4965b580f29c89f706)) + + +### Bug Fixes + +* add mappings to mp event ([#3997](https://github.com/rudderlabs/rudder-transformer/issues/3997)) ([2eab465](https://github.com/rudderlabs/rudder-transformer/commit/2eab46557743702a834597d0d0267a17e561516d)) +* sonar issues in adobe analytics ([#3999](https://github.com/rudderlabs/rudder-transformer/issues/3999)) ([d74c4ab](https://github.com/rudderlabs/rudder-transformer/commit/d74c4ab7ef582eb83d67c4b295aa5d3845c15166)) +* sonar issues in user transformations static lookup ([#3998](https://github.com/rudderlabs/rudder-transformer/issues/3998)) ([a7d6b8f](https://github.com/rudderlabs/rudder-transformer/commit/a7d6b8fca8681d2eb3bf1751bf7855a7cc220d3b)) +* sonar issues in various components ([#4006](https://github.com/rudderlabs/rudder-transformer/issues/4006)) ([454451d](https://github.com/rudderlabs/rudder-transformer/commit/454451dba3260a3664088e2bb009fd8a11cbf957)) + +### [1.88.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.88.2...v1.88.3) (2025-01-24) + + +### Bug Fixes + +* iterable device token registration api issues ([#4013](https://github.com/rudderlabs/rudder-transformer/issues/4013)) ([a986138](https://github.com/rudderlabs/rudder-transformer/commit/a986138d43d2cabcaa597b70fa7871b383cb1427)) + +### [1.88.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.88.1...v1.88.2) (2025-01-22) + + +### Bug Fixes + +* revert feat added drop traits in track call feature for mixpanel ([#4003](https://github.com/rudderlabs/rudder-transformer/issues/4003)) ([ba2accd](https://github.com/rudderlabs/rudder-transformer/commit/ba2accd14a9d99f2c55a9eab34f65e6e4d989e4a)) + +### [1.88.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.88.0...v1.88.1) (2025-01-21) + +## [1.88.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.87.1...v1.88.0) (2025-01-20) + + +### Features + +* add redis support in shopify pixel for id stitching ([#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957)) ([165e06d](https://github.com/rudderlabs/rudder-transformer/commit/165e06de1ac22cda5c4db0b0b23e6e9972d5fa94)) +* added drop traits in track call feature for mixpanel ([#3986](https://github.com/rudderlabs/rudder-transformer/issues/3986)) ([6ec3d55](https://github.com/rudderlabs/rudder-transformer/commit/6ec3d552b845535965bc6c7b9990161a168b9a4e)) + + +### Bug Fixes + +* add email validation in HS ([#3972](https://github.com/rudderlabs/rudder-transformer/issues/3972)) ([1c45db8](https://github.com/rudderlabs/rudder-transformer/commit/1c45db8e6ec6b9436ef08eaa9dfcc99c8953397b)) +* fixing mismatch of schedule field value in clicksend destination ([#3975](https://github.com/rudderlabs/rudder-transformer/issues/3975)) ([12e8a52](https://github.com/rudderlabs/rudder-transformer/commit/12e8a528148454e2113c9c68b575d25695c1d55a)) +* refactor code and add validation for values ([#3971](https://github.com/rudderlabs/rudder-transformer/issues/3971)) ([fc09080](https://github.com/rudderlabs/rudder-transformer/commit/fc090806235f2599703098c9f3fc83c4bdf7d599)) +* removing device token from if condition in mixpanel destination ([#3982](https://github.com/rudderlabs/rudder-transformer/issues/3982)) ([b0e0b15](https://github.com/rudderlabs/rudder-transformer/commit/b0e0b15ee3cb7aeab0162c91fdcf98fca1e16722)) +* sonar issues in regex expressions ([#3979](https://github.com/rudderlabs/rudder-transformer/issues/3979)) ([0e0944d](https://github.com/rudderlabs/rudder-transformer/commit/0e0944d2372fa53b1a6db56fd382fc42c57e5068)) + ### [1.87.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.87.0...v1.87.1) (2025-01-06) diff --git a/benchmark/index.js b/benchmark/index.js deleted file mode 100644 index 919fa3c6de0..00000000000 --- a/benchmark/index.js +++ /dev/null @@ -1,193 +0,0 @@ -/* eslint-disable import/no-extraneous-dependencies */ -/* eslint-disable func-names */ -/* eslint-disable no-restricted-syntax */ -/* eslint-disable guard-for-in */ -/* eslint-disable no-await-in-loop */ -/* eslint-disable no-use-before-define */ -/* eslint-disable import/no-dynamic-require */ -/* eslint-disable global-require */ -const Benchmark = require('benchmark-suite'); -const fs = require('fs'); -const path = require('path'); -const Commander = require('commander'); -const logger = require('./metaLogger'); -const versionedRouter = require('../src/versionedRouter'); -const cdkV2Handler = require('../src/cdk/v2/handler'); - -const supportedDestinations = ['algolia', 'pinterest_tag']; - -logger.info(); - -const command = new Commander.Command(); -command - .allowUnknownOption() - .option( - '-d, --destinations ', - 'Enter destination names separated by comma', - supportedDestinations.toString(), - ) - .option( - '-bt, --benchmarktype ', - 'Enter the benchmark type (Operations or Memory)', - 'Operations', - ) - .option('-f, --feature ', 'Enter feature name (proc or rt)', 'proc') - .parse(); - -const getTestFileName = (intg, testSufix) => { - const featureSufix = cmdOpts.feature === 'rt' ? '_router' : ''; - return `${intg}${featureSufix}${testSufix}.json`; -}; - -const testDataDir = path.join(__dirname, '../test/__tests__/data'); -const getTestData = (intgList, fileNameSuffixes) => { - const intgTestData = {}; - intgList.forEach((intg) => { - // Use the last valid test data file - fileNameSuffixes.forEach((fileNameSuffix) => { - try { - intgTestData[intg] = JSON.parse( - fs.readFileSync(path.join(testDataDir, getTestFileName(intg, fileNameSuffix)), { - encoding: 'utf-8', - }), - ); - } catch (err) { - // logger.error( - // `Unable to load the data for: "${intg}" suffix: "${fileNameSuffix}"` - // ); - // logger.error(`Raw error: "${err}"`); - } - }); - }); - return intgTestData; -}; - -const cmdOpts = command.opts(); - -// Initialize data for destinations -const destinationsList = cmdOpts.destinations - .split(',') - .map((x) => x.trim()) - .filter((x) => x !== ''); -logger.info('Destinations:', destinationsList, 'feature:', cmdOpts.feature); -logger.info(); -const destDataset = getTestData(destinationsList, ['_input', '']); - -const nativeDestHandlers = {}; -const destCdKWorkflowEngines = {}; - -const benchmarkType = cmdOpts.benchmarktype.trim(); - -const getNativeHandleName = () => { - let handleName = 'process'; - if (cmdOpts.feature === 'rt') { - handleName = 'processRouterDest'; - } - return handleName; -}; - -async function initializeHandlers() { - for (const idx in destinationsList) { - const dest = destinationsList[idx]; - - // Native destination handler - nativeDestHandlers[dest] = versionedRouter.getDestHandler('v0', dest)[getNativeHandleName()]; - - // Get the CDK 2.0 workflow engine instance - destCdKWorkflowEngines[dest] = await cdkV2Handler.getWorkflowEngine(dest, cmdOpts.feature); - } -} - -async function runDataset(suitDesc, input, intg, params) { - logger.info('=========================================='); - logger.info(suitDesc); - logger.info('=========================================='); - - const results = {}; - const suite = new Benchmark(suitDesc, benchmarkType); - - Object.keys(params).forEach((opName) => { - const handler = params[opName].handlerResolver(intg); - const args = params[opName].argsResolver(intg, input); - suite.add(opName, async () => { - try { - await handler(...args); - } catch (err) { - // logger.info(err); - // Do nothing - } - }); - }); - - suite - .on('cycle', (result) => { - results[result.end.name] = { stats: result.end.stats }; - }) - .on('complete', (result) => { - logger.info( - benchmarkType === 'Operations' ? 'Fastest: ' : 'Memory intensive: ', - `"${result.end.name}"`, - ); - logger.info(); - Object.keys(results).forEach((impl) => { - logger.info(`"${impl}" - `, suite.formatStats(results[impl].stats)); - - if (result.end.name !== impl) { - if (benchmarkType === 'Operations') { - logger.info( - `-> "${result.end.name}" is faster by ${( - results[impl].stats.mean / result.end.stats.mean - ).toFixed(1)} times to "${impl}"`, - ); - } else { - logger.info( - `-> "${result.end.name}" consumed ${( - result.end.stats.mean / results[impl].stats.mean - ).toFixed(1)} times memory compared to "${impl}"`, - ); - } - } - - logger.info(); - }); - }); - - await suite.run({ time: 1000 }); -} - -async function runIntgDataset(dataset, type, params) { - for (const intg in dataset) { - for (const tc in dataset[intg]) { - const curTcData = dataset[intg][tc]; - let tcInput = curTcData; - let tcDesc = `${type} - ${intg} - ${cmdOpts.feature} - ${tc}`; - // New test data file structure - if ('description' in curTcData && 'input' in curTcData && 'output' in curTcData) { - tcInput = curTcData.input; - tcDesc += ` - "${curTcData.description}"`; - } - - await runDataset(tcDesc, tcInput, intg, params); - } - } -} - -async function run() { - // Initialize CDK and native handlers - await initializeHandlers(); - - // Destinations - await runIntgDataset(destDataset, 'Destination', { - native: { - handlerResolver: (intg) => nativeDestHandlers[intg], - argsResolver: (_intg, input) => [input], - }, - 'CDK 2.0': { - handlerResolver: () => cdkV2Handler.process, - argsResolver: (intg, input) => [destCdKWorkflowEngines[intg], input], - }, - }); -} - -// Start suites -run(); diff --git a/benchmark/metaLogger.js b/benchmark/metaLogger.js deleted file mode 100644 index b89ad71066d..00000000000 --- a/benchmark/metaLogger.js +++ /dev/null @@ -1,36 +0,0 @@ -/* istanbul ignore file */ - -const logger = require('../src/logger'); - -logger.setLogLevel('random'); - -const debug = (...args) => { - logger.setLogLevel('debug'); - logger.debug(...args); - logger.setLogLevel('random'); -}; - -const info = (...args) => { - logger.setLogLevel('info'); - logger.info(...args); - logger.setLogLevel('random'); -}; - -const warn = (...args) => { - logger.setLogLevel('warn'); - logger.warn(...args); - logger.setLogLevel('random'); -}; - -const error = (...args) => { - logger.setLogLevel('error'); - logger.error(...args); - logger.setLogLevel('random'); -}; - -module.exports = { - debug, - info, - warn, - error, -}; diff --git a/package-lock.json b/package-lock.json index b3631b2e57f..319c5b43aa8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.87.1", + "version": "1.90.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.87.1", + "version": "1.90.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", @@ -20,7 +20,7 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.13", - "@rudderstack/json-template-engine": "^0.18.0", + "@rudderstack/json-template-engine": "^0.19.5", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", @@ -32,6 +32,7 @@ "component-each": "^0.2.6", "crypto-js": "^4.2.0", "dotenv": "^16.0.3", + "fast-xml-parser": "^4.5.1", "flat": "^5.0.2", "form-data": "^4.0.0", "get-value": "^3.0.1", @@ -46,11 +47,10 @@ "json-diff": "^1.0.3", "json-size": "^1.0.0", "jsontoxml": "^1.0.1", - "jstoxml": "^5.0.2", "koa": "^2.15.3", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", - "libphonenumber-js": "^1.11.12", + "libphonenumber-js": "^1.11.18", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", @@ -1134,6 +1134,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-personalize/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-s3": { "version": "3.637.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.637.0.tgz", @@ -1570,6 +1591,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-s3/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-sso": { "version": "3.650.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.650.0.tgz", @@ -2014,6 +2056,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-sso-oidc/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-sso/node_modules/@aws-sdk/util-user-agent-browser": { "version": "3.649.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.649.0.tgz", @@ -2481,6 +2544,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-sts/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/core": { "version": "3.649.0", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.649.0.tgz", @@ -2502,6 +2586,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/core/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/credential-provider-cognito-identity": { "version": "3.650.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.650.0.tgz", @@ -3604,6 +3709,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/middleware-ssec": { "version": "3.609.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.609.0.tgz", @@ -6642,8 +6768,9 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.18.0", - "license": "MIT" + "version": "0.19.5", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.19.5.tgz", + "integrity": "sha512-lA45cp8caboMECfNk/CXuP45hCx+W09mclEDiqqTQXilY1hdWp+r/zpcOBzFGZWSxIdrAp5cfhIYnUENx3XX5g==" }, "node_modules/@rudderstack/workflow-engine": { "version": "0.8.13", @@ -12330,7 +12457,9 @@ "license": "MIT" }, "node_modules/fast-xml-parser": { - "version": "4.4.1", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.1.tgz", + "integrity": "sha512-y655CeyUQ+jj7KBbYMc4FG01V8ZQqjN+gDYGJ50RtfsUB8iG9AmwmwoAgeKLJdmueKKMrH1RJ7yXHTSoczdv5w==", "funding": [ { "type": "github", @@ -12341,7 +12470,6 @@ "url": "https://paypal.me/naturalintelligence" } ], - "license": "MIT", "dependencies": { "strnum": "^1.0.5" }, @@ -16385,10 +16513,6 @@ "node": ">=0.2.0" } }, - "node_modules/jstoxml": { - "version": "5.0.2", - "license": "MIT" - }, "node_modules/keygrip": { "version": "1.1.0", "license": "MIT", @@ -16639,9 +16763,9 @@ } }, "node_modules/libphonenumber-js": { - "version": "1.11.12", - "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.12.tgz", - "integrity": "sha512-QkJn9/D7zZ1ucvT++TQSvZuSA2xAWeUytU+DiEQwbPKLyrDpvbul2AFs1CGbRAPpSCCk47aRAb5DX5mmcayp4g==" + "version": "1.11.18", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.18.tgz", + "integrity": "sha512-okMm/MCoFrm1vByeVFLBdkFIXLSHy/AIK2AEGgY3eoicfWZeOZqv3GfhtQgICkzs/tqorAMm3a4GBg5qNCrqzg==" }, "node_modules/lilconfig": { "version": "2.1.0", @@ -18188,7 +18312,9 @@ "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==" }, "node_modules/nanoid": { - "version": "3.3.7", + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", "dev": true, "funding": [ { diff --git a/package.json b/package.json index cf72ab26def..b6fb5b42884 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.87.1", + "version": "1.90.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -65,7 +65,7 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.13", - "@rudderstack/json-template-engine": "^0.18.0", + "@rudderstack/json-template-engine": "^0.19.5", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", @@ -77,6 +77,7 @@ "component-each": "^0.2.6", "crypto-js": "^4.2.0", "dotenv": "^16.0.3", + "fast-xml-parser": "^4.5.1", "flat": "^5.0.2", "form-data": "^4.0.0", "get-value": "^3.0.1", @@ -91,11 +92,10 @@ "json-diff": "^1.0.3", "json-size": "^1.0.0", "jsontoxml": "^1.0.1", - "jstoxml": "^5.0.2", "koa": "^2.15.3", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", - "libphonenumber-js": "^1.11.12", + "libphonenumber-js": "^1.11.18", "lodash": "^4.17.21", "match-json": "^1.3.5", "md5": "^2.3.0", diff --git a/sonar-project.properties b/sonar-project.properties index 6c45b0ce39d..284f159cb3d 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -18,7 +18,7 @@ sonar.testExecutionReportPaths=reports/sonar/results-report.xml sonar.eslint.reportPaths=reports/eslint.json # Path to sources -sonar.sources=src,benchmark +sonar.sources=src sonar.inclusions=**/*.js sonar.exclusions=**/*.json,**/*.html,**/*.png,**/*.jpg,**/*.gif,**/*.svg,**/*.yml,src/util/libExtractor.js,src/util/url-search-params.min.js,src/util/lodash-es-core.js diff --git a/src/adapters/network.js b/src/adapters/network.js index aeb1cc128b3..13ebc89aadc 100644 --- a/src/adapters/network.js +++ b/src/adapters/network.js @@ -323,7 +323,7 @@ function stringifyQueryParam(value) { * @param {Object} payload * @returns {String} */ -function getFormData(payload) { +function getFormData(payload = {}) { const data = new URLSearchParams(); Object.keys(payload).forEach((key) => { const payloadValStr = stringifyQueryParam(payload[key]); @@ -332,6 +332,22 @@ function getFormData(payload) { return data; } +function extractPayloadForFormat(payload, format) { + switch (format) { + case 'JSON_ARRAY': + return payload?.batch; + case 'JSON': + return payload; + case 'XML': + return payload?.payload; + case 'FORM': + return getFormData(payload); + default: + logger.debug(`Unknown payload format: ${format}`); + return undefined; + } +} + /** * Prepares the proxy request * @param {*} request @@ -340,33 +356,29 @@ function getFormData(payload) { const prepareProxyRequest = (request) => { const { body, method, params, endpoint, headers, destinationConfig: config } = request; const { payload, payloadFormat } = getPayloadData(body); - let data; - - switch (payloadFormat) { - case 'JSON_ARRAY': - data = payload.batch; - // TODO: add headers - break; - case 'JSON': - data = payload; - break; - case 'XML': - data = payload.payload; - break; - case 'FORM': - data = getFormData(payload); - break; - case 'MULTIPART-FORM': - // TODO: - break; - default: - logger.debug(`body format ${payloadFormat} not supported`); - } + const data = extractPayloadForFormat(payload, payloadFormat); // Ref: https://github.com/rudderlabs/rudder-server/blob/master/router/network.go#L164 headers['User-Agent'] = 'RudderLabs'; return removeUndefinedValues({ endpoint, data, params, headers, method, config }); }; +const getHttpWrapperMethod = (requestType) => { + switch (requestType) { + case 'get': + return httpGET; + case 'put': + return httpPUT; + case 'patch': + return httpPATCH; + case 'delete': + return httpDELETE; + case 'constructor': + return httpSend; + default: + return httpPOST; + } +}; + /** * handles http request and sends the response in a simple format that is followed in transformer * @@ -392,27 +404,7 @@ const prepareProxyRequest = (request) => { }) */ const handleHttpRequest = async (requestType = 'post', ...httpArgs) => { - let httpWrapperMethod; - switch (requestType.toLowerCase()) { - case 'get': - httpWrapperMethod = httpGET; - break; - case 'put': - httpWrapperMethod = httpPUT; - break; - case 'patch': - httpWrapperMethod = httpPATCH; - break; - case 'delete': - httpWrapperMethod = httpDELETE; - break; - case 'constructor': - httpWrapperMethod = httpSend; - break; - default: - httpWrapperMethod = httpPOST; - break; - } + const httpWrapperMethod = getHttpWrapperMethod(requestType.toLowerCase()); const httpResponse = await httpWrapperMethod(...httpArgs); const processedResponse = processAxiosResponse(httpResponse); return { httpResponse, processedResponse }; diff --git a/src/adapters/network.test.js b/src/adapters/network.test.js index 7894925ccd6..a4a2787504f 100644 --- a/src/adapters/network.test.js +++ b/src/adapters/network.test.js @@ -1,7 +1,20 @@ const mockLoggerInstance = { info: jest.fn(), }; -const { getFormData, httpPOST, httpGET, httpSend, fireHTTPStats } = require('./network'); +const { + getFormData, + httpPOST, + httpGET, + httpSend, + fireHTTPStats, + proxyRequest, + prepareProxyRequest, + handleHttpRequest, + httpDELETE, + httpPUT, + httpPATCH, + getPayloadData, +} = require('./network'); const { getFuncTestData } = require('../../test/testHelper'); jest.mock('../util/stats', () => ({ timing: jest.fn(), @@ -20,14 +33,28 @@ jest.mock('@rudderstack/integrations-lib', () => { }; }); -jest.mock('axios', () => jest.fn()); +// Mock the axios module +jest.mock('axios', () => { + const mockAxios = jest.fn(); // Mock the default axios function + mockAxios.get = jest.fn(); // Mock axios.get + mockAxios.post = jest.fn(); // Mock axios.post + mockAxios.put = jest.fn(); // Mock axios.put + mockAxios.patch = jest.fn(); // Mock axios.patch + mockAxios.delete = jest.fn(); // Mock axios.delete + + // Mock the axios.create method if needed + mockAxios.create = jest.fn(() => mockAxios); + + return mockAxios; // Return the mocked axios +}); + +const axios = require('axios'); jest.mock('../util/logger', () => ({ ...jest.requireActual('../util/logger'), getMatchedMetadata: jest.fn(), })); -const axios = require('axios'); const loggerUtil = require('../util/logger'); axios.post = jest.fn(); @@ -635,3 +662,338 @@ describe('logging in http methods', () => { expect(mockLoggerInstance.info).toHaveBeenCalledTimes(0); }); }); + +describe('httpDELETE tests', () => { + beforeEach(() => { + mockLoggerInstance.info.mockClear(); + loggerUtil.getMatchedMetadata.mockClear(); + axios.delete.mockClear(); + }); + + test('should call axios.delete with correct parameters and log request/response', async () => { + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/m/n/o', + requestMethod: 'delete', + }; + loggerUtil.getMatchedMetadata.mockReturnValue([statTags.metadata]); + + axios.delete.mockResolvedValueOnce({ + status: 200, + data: { a: 1, b: 2, c: 'abc' }, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + + await expect(httpDELETE('https://some.web.com/m/n/o', {}, statTags)).resolves.not.toThrow( + Error, + ); + expect(loggerUtil.getMatchedMetadata).toHaveBeenCalledTimes(2); + expect(mockLoggerInstance.info).toHaveBeenCalledTimes(2); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(1, ' [DT] /m/n/o request', { + body: undefined, + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + url: 'https://some.web.com/m/n/o', + method: 'delete', + }); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(2, ' [DT] /m/n/o response', { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + body: { a: 1, b: 2, c: 'abc' }, + status: 200, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + }); +}); + +describe('httpPUT tests', () => { + beforeEach(() => { + mockLoggerInstance.info.mockClear(); + loggerUtil.getMatchedMetadata.mockClear(); + axios.put.mockClear(); + }); + + test('should call axios.put with correct parameters and log request/response', async () => { + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/m/n/o', + requestMethod: 'put', + }; + loggerUtil.getMatchedMetadata.mockReturnValue([statTags.metadata]); + + axios.put.mockResolvedValueOnce({ + status: 200, + data: { a: 1, b: 2, c: 'abc' }, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + + await expect(httpPUT('https://some.web.com/m/n/o', {}, {}, statTags)).resolves.not.toThrow( + Error, + ); + expect(loggerUtil.getMatchedMetadata).toHaveBeenCalledTimes(2); + expect(mockLoggerInstance.info).toHaveBeenCalledTimes(2); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(1, ' [DT] /m/n/o request', { + body: {}, + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + url: 'https://some.web.com/m/n/o', + method: 'put', + }); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(2, ' [DT] /m/n/o response', { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + body: { a: 1, b: 2, c: 'abc' }, + status: 200, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + }); +}); + +describe('httpPATCH tests', () => { + beforeEach(() => { + mockLoggerInstance.info.mockClear(); + loggerUtil.getMatchedMetadata.mockClear(); + axios.patch.mockClear(); + }); + + test('should call axios.patch with correct parameters and log request/response', async () => { + const statTags = { + metadata: { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + }, + destType: 'DT', + feature: 'feat', + endpointPath: '/m/n/o', + requestMethod: 'patch', + }; + loggerUtil.getMatchedMetadata.mockReturnValue([statTags.metadata]); + + axios.patch.mockResolvedValueOnce({ + status: 200, + data: { a: 1, b: 2, c: 'abc' }, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + + await expect(httpPATCH('https://some.web.com/m/n/o', {}, {}, statTags)).resolves.not.toThrow( + Error, + ); + expect(loggerUtil.getMatchedMetadata).toHaveBeenCalledTimes(2); + expect(mockLoggerInstance.info).toHaveBeenCalledTimes(2); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(1, ' [DT] /m/n/o request', { + body: {}, + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + url: 'https://some.web.com/m/n/o', + method: 'patch', + }); + + expect(mockLoggerInstance.info).toHaveBeenNthCalledWith(2, ' [DT] /m/n/o response', { + destType: 'DT', + destinationId: 'd1', + workspaceId: 'w1', + sourceId: 's1', + body: { a: 1, b: 2, c: 'abc' }, + status: 200, + headers: { + 'Content-Type': 'application/json', + 'X-Some-Header': 'headsome', + }, + }); + }); +}); + +describe('getPayloadData tests', () => { + test('should return payload and payloadFormat for non-empty body', () => { + const body = { + JSON: { key: 'value' }, + XML: null, + FORM: null, + }; + const result = getPayloadData(body); + expect(result).toEqual({ payload: { key: 'value' }, payloadFormat: 'JSON' }); + }); + + test('should return undefined payload and payloadFormat for empty body', () => { + const body = {}; + const result = getPayloadData(body); + expect(result).toEqual({ payload: undefined, payloadFormat: undefined }); + }); +}); + +describe('prepareProxyRequest tests', () => { + test('should prepare proxy request with correct headers and payload', () => { + const request = { + body: { JSON: { key: 'value' } }, + method: 'POST', + params: { param1: 'value1' }, + endpoint: 'https://example.com', + headers: { 'Content-Type': 'application/json' }, + destinationConfig: { key: 'value' }, + }; + const result = prepareProxyRequest(request); + expect(result).toEqual({ + endpoint: 'https://example.com', + data: { key: 'value' }, + params: { param1: 'value1' }, + headers: { 'Content-Type': 'application/json', 'User-Agent': 'RudderLabs' }, + method: 'POST', + config: { key: 'value' }, + }); + }); +}); + +describe('handleHttpRequest tests', () => { + beforeEach(() => { + axios.post.mockClear(); + axios.get.mockClear(); + axios.put.mockClear(); + axios.patch.mockClear(); + axios.delete.mockClear(); + }); + + test('should handle POST request correctly', async () => { + axios.post.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('post', 'https://example.com', { key: 'value' }, {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle GET request correctly', async () => { + axios.get.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('get', 'https://example.com', {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle PUT request correctly', async () => { + axios.put.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('put', 'https://example.com', { key: 'value' }, {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle PATCH request correctly', async () => { + axios.patch.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('patch', 'https://example.com', { key: 'value' }, {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); + + test('should handle DELETE request correctly', async () => { + axios.delete.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const result = await handleHttpRequest('delete', 'https://example.com', {}); + expect(result.httpResponse).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + expect(result.processedResponse).toBeDefined(); + }); +}); + +describe('proxyRequest tests', () => { + beforeEach(() => { + axios.mockClear(); + }); + + test('should proxy request correctly', async () => { + axios.mockResolvedValueOnce({ + status: 200, + data: { key: 'value' }, + }); + + const request = { + body: { JSON: { key: 'value' } }, + method: 'POST', + params: { param1: 'value1' }, + endpoint: 'https://example.com', + headers: { 'Content-Type': 'application/json' }, + destinationConfig: { key: 'value' }, + metadata: { destType: 'DT' }, + }; + + const result = await proxyRequest(request, 'DT'); + expect(result).toEqual({ + success: true, + response: { status: 200, data: { key: 'value' } }, + }); + }); +}); diff --git a/src/cdk/v2/destinations/bluecore/utils.js b/src/cdk/v2/destinations/bluecore/utils.js index 543b6de745f..e82a1355b51 100644 --- a/src/cdk/v2/destinations/bluecore/utils.js +++ b/src/cdk/v2/destinations/bluecore/utils.js @@ -19,74 +19,83 @@ const { EVENT_NAME_MAPPING, IDENTIFY_EXCLUSION_LIST, TRACK_EXCLUSION_LIST } = re const { EventType } = require('../../../../constants'); const { MAPPING_CONFIG, CONFIG_CATEGORIES } = require('./config'); -/** - * Verifies the correctness of payload for different events. - * - * @param {Object} payload - The payload object containing event information. - * @param {Object} message - The message object containing additional information. - * @throws {InstrumentationError} - Throws an error if required properties are missing. - * @returns {void} - */ -const verifyPayload = (payload, message) => { +const validateCustomerProperties = (payload, eventName) => { + if ( + !isDefinedAndNotNull(payload?.properties?.customer) || + Object.keys(payload.properties.customer).length === 0 + ) { + throw new InstrumentationError( + `[Bluecore] property:: No relevant trait to populate customer information, which is required for ${eventName} action`, + ); + } +}; + +const validateIdentifyAction = (message) => { if ( message.type === EventType.IDENTIFY && isDefinedNotNullNotEmpty(message.traits?.action) && message.traits?.action !== 'identify' ) { throw new InstrumentationError( - "[Bluecore] traits.action must be 'identify' for identify action", + "[Bluecore] traits.action must be 'identify' for identify action", ); } - switch (payload.event) { - case 'search': - if (!payload?.properties?.search_term) { - throw new InstrumentationError( - '[Bluecore] property:: search_query is required for search event', - ); - } - break; - case 'purchase': - if (!isDefinedAndNotNull(payload?.properties?.order_id)) { - throw new InstrumentationError( - '[Bluecore] property:: order_id is required for purchase event', - ); - } - if (!isDefinedAndNotNull(payload?.properties?.total)) { - throw new InstrumentationError( - '[Bluecore] property:: total is required for purchase event', - ); - } - if ( - !isDefinedAndNotNull(payload?.properties?.customer) || - Object.keys(payload.properties.customer).length === 0 - ) { - throw new InstrumentationError( - `[Bluecore] property:: No relevant trait to populate customer information, which is required for ${payload.event} event`, - ); - } - break; - case 'identify': - case 'optin': - case 'unsubscribe': - if (!isDefinedAndNotNullAndNotEmpty(getFieldValueFromMessage(message, 'email'))) { - throw new InstrumentationError( - `[Bluecore] property:: email is required for ${payload.event} action`, - ); - } - if ( - !isDefinedAndNotNull(payload?.properties?.customer) || - Object.keys(payload.properties.customer).length === 0 - ) { - throw new InstrumentationError( - `[Bluecore] property:: No relevant trait to populate customer information, which is required for ${payload.event} action`, - ); - } - break; - default: - break; +}; +const validateSearchEvent = (payload) => { + if (!payload?.properties?.search_term) { + throw new InstrumentationError( + '[Bluecore] property:: search_query is required for search event', + ); } }; +const validatePurchaseEvent = (payload) => { + if (!isDefinedAndNotNull(payload?.properties?.order_id)) { + throw new InstrumentationError('[Bluecore] property:: order_id is required for purchase event'); + } + if (!isDefinedAndNotNull(payload?.properties?.total)) { + throw new InstrumentationError('[Bluecore] property:: total is required for purchase event'); + } + validateCustomerProperties(payload, 'purchase'); +}; + +const validateCustomerEvent = (payload, message) => { + if (!isDefinedAndNotNullAndNotEmpty(getFieldValueFromMessage(message, 'email'))) { + throw new InstrumentationError( + `[Bluecore] property:: email is required for ${payload.event} action`, + ); + } + validateCustomerProperties(payload, payload.event); +}; + +const validateEventSpecificPayload = (payload, message) => { + const eventValidators = { + search: validateSearchEvent, + purchase: validatePurchaseEvent, + identify: validateCustomerEvent, + optin: validateCustomerEvent, + unsubscribe: validateCustomerEvent, + }; + + const validator = eventValidators[payload.event]; + if (validator) { + validator(payload, message); + } +}; + +/** + * Verifies the correctness of payload for different events. + * + * @param {Object} payload - The payload object containing event information. + * @param {Object} message - The message object containing additional information. + * @throws {InstrumentationError} - Throws an error if required properties are missing. + * @returns {void} + */ +const verifyPayload = (payload, message) => { + validateIdentifyAction(message); + validateEventSpecificPayload(payload, message); +}; + /** * Deduces the track event name based on the provided track event name and configuration. * diff --git a/src/cdk/v2/destinations/clicksend/utils.js b/src/cdk/v2/destinations/clicksend/utils.js index 797ea12025e..c6a8c281e3b 100644 --- a/src/cdk/v2/destinations/clicksend/utils.js +++ b/src/cdk/v2/destinations/clicksend/utils.js @@ -4,8 +4,11 @@ const { BatchUtils } = require('@rudderstack/workflow-engine'); const { SMS_SEND_ENDPOINT, MAX_BATCH_SIZE, COMMON_CONTACT_DOMAIN } = require('./config'); const { isDefinedAndNotNullAndNotEmpty, isDefinedAndNotNull } = require('../../../../v0/util'); -const getEndIdentifyPoint = (contactId, contactListId) => - `${COMMON_CONTACT_DOMAIN}/${contactListId}/contacts${isDefinedAndNotNullAndNotEmpty(contactId) ? `/${contactId}` : ''}`; +const getEndIdentifyPoint = (contactId, contactListId) => { + const basePath = `${COMMON_CONTACT_DOMAIN}/${contactListId}/contacts`; + const contactSuffix = isDefinedAndNotNullAndNotEmpty(contactId) ? `/${contactId}` : ''; + return basePath + contactSuffix; +}; const validateIdentifyPayload = (payload) => { if ( diff --git a/src/cdk/v2/destinations/http/procWorkflow.yaml b/src/cdk/v2/destinations/http/procWorkflow.yaml index 080dcdd80af..1db0fce1d72 100644 --- a/src/cdk/v2/destinations/http/procWorkflow.yaml +++ b/src/cdk/v2/destinations/http/procWorkflow.yaml @@ -30,35 +30,38 @@ steps: - name: deduceBodyFormat template: | - $.context.format = .destination.Config.format ?? 'JSON'; + const format = .destination.Config.format ?? 'JSON'; + $.context.format = $.CONTENT_TYPES_MAP[format]; - name: buildHeaders template: | const configAuthHeaders = $.getAuthHeaders(.destination.Config); const additionalConfigHeaders = $.getCustomMappings(.message, .destination.Config.headers); + const metadataHeaders = $.metadataHeaders($.context.format); $.context.headers = { ...configAuthHeaders, - ...additionalConfigHeaders + ...additionalConfigHeaders, + ...metadataHeaders, } - name: prepareParams template: | - $.context.params = $.getCustomMappings(.message, .destination.Config.queryParams) + const params = $.getCustomMappings(.message, .destination.Config.queryParams); + $.context.params = $.encodeParamsObject(params); - name: deduceEndPoint template: | - $.context.endpoint = $.addPathParams(.message, .destination.Config.apiUrl); + $.context.endpoint = $.prepareEndpoint(.message, .destination.Config.apiUrl, .destination.Config.pathParams); - name: prepareBody template: | const payload = $.getCustomMappings(.message, .destination.Config.propertiesMapping); - $.context.payload = $.removeUndefinedAndNullValues($.excludeMappedFields(payload, .destination.Config.propertiesMapping)) - $.context.format === "XML" && !$.isEmptyObject($.context.payload) ? $.context.payload = {payload: $.getXMLPayload($.context.payload)}; + $.context.payload = $.prepareBody(payload, $.context.format, .destination.Config.xmlRootKey); - name: buildResponseForProcessTransformation template: | const response = $.defaultRequestConfig(); - $.context.format === "JSON" ? response.body.JSON = $.context.payload: response.body.XML = $.context.payload; + response.body[$.context.format] = $.context.payload; response.endpoint = $.context.endpoint; response.headers = $.context.headers; response.method = $.context.method; diff --git a/src/cdk/v2/destinations/http/utils.js b/src/cdk/v2/destinations/http/utils.js index 355eb034870..7657a1cd60a 100644 --- a/src/cdk/v2/destinations/http/utils.js +++ b/src/cdk/v2/destinations/http/utils.js @@ -1,15 +1,22 @@ -const { toXML } = require('jstoxml'); +const { XMLBuilder } = require('fast-xml-parser'); const { groupBy } = require('lodash'); const { createHash } = require('crypto'); const { ConfigurationError } = require('@rudderstack/integrations-lib'); const { BatchUtils } = require('@rudderstack/workflow-engine'); +const jsonpath = require('rs-jsonpath'); const { base64Convertor, applyCustomMappings, isEmptyObject, - applyJSONStringTemplate, + removeUndefinedAndNullRecurse, } = require('../../../../v0/util'); +const CONTENT_TYPES_MAP = { + JSON: 'JSON', + XML: 'XML', + FORM: 'FORM', +}; + const getAuthHeaders = (config) => { let headers; switch (config.auth) { @@ -33,56 +40,119 @@ const getAuthHeaders = (config) => { return headers; }; +const enhanceMappings = (mappings) => { + let enhancedMappings = mappings; + if (Array.isArray(mappings)) { + enhancedMappings = mappings.map((mapping) => { + const enhancedMapping = { ...mapping }; + if ( + mapping.hasOwnProperty('from') && + !mapping.from.includes('$') && + !(mapping.from.startsWith("'") && mapping.from.endsWith("'")) + ) { + enhancedMapping.from = `'${mapping.from}'`; + } + return enhancedMapping; + }); + } + return enhancedMappings; +}; + const getCustomMappings = (message, mapping) => { + const enhancedMappings = enhanceMappings(mapping); try { - return applyCustomMappings(message, mapping); + return applyCustomMappings(message, enhancedMappings); } catch (e) { throw new ConfigurationError(`Error in custom mappings: ${e.message}`); } }; -const addPathParams = (message, apiUrl) => { - try { - return applyJSONStringTemplate(message, `\`${apiUrl}\``); - } catch (e) { - throw new ConfigurationError(`Error in api url template: ${e.message}`); +const encodeParamsObject = (params) => { + if (!params || typeof params !== 'object') { + return {}; // Return an empty object if input is null, undefined, or not an object } + return Object.keys(params) + .filter((key) => params[key] !== undefined) + .reduce((acc, key) => { + acc[encodeURIComponent(key)] = encodeURIComponent(params[key]); + return acc; + }, {}); }; -const excludeMappedFields = (payload, mapping) => { - const rawPayload = { ...payload }; - if (mapping) { - mapping.forEach(({ from, to }) => { - // continue when from === to - if (from === to) return; - - // Remove the '$.' prefix and split the remaining string by '.' - const keys = from.replace(/^\$\./, '').split('.'); - let current = rawPayload; - - // Traverse to the parent of the key to be removed - keys.slice(0, -1).forEach((key) => { - if (current?.[key]) { - current = current[key]; - } else { - current = null; - } - }); - - if (current) { - // Remove the 'from' field from input payload - delete current[keys[keys.length - 1]]; - } - }); +const getPathValueFromJsonpath = (message, path) => { + let finalPath = path; + if (path.includes('/')) { + throw new ConfigurationError('Path value cannot contain "/"'); } + if (path.includes('$')) { + try { + [finalPath = null] = jsonpath.query(message, path); + } catch (error) { + throw new ConfigurationError( + `An error occurred while querying the JSON path: ${error.message}`, + ); + } + if (finalPath === null) { + throw new ConfigurationError('Path not found in the object.'); + } + } + return finalPath; +}; - return rawPayload; +const getPathParamsSubString = (message, pathParamsArray) => { + if (pathParamsArray.length === 0) { + return ''; + } + const pathParamsValuesArray = pathParamsArray.map((pathParam) => + encodeURIComponent(getPathValueFromJsonpath(message, pathParam.path)), + ); + return `/${pathParamsValuesArray.join('/')}`; }; -const getXMLPayload = (payload) => - toXML(payload, { - header: true, - }); +const prepareEndpoint = (message, apiUrl, pathParams) => { + if (!Array.isArray(pathParams)) { + return apiUrl; + } + const requestUrl = apiUrl.replace(/\/{1,10}$/, ''); + const pathParamsSubString = getPathParamsSubString(message, pathParams); + return `${requestUrl}${pathParamsSubString}`; +}; + +const sanitizeKey = (key) => + key + .replace(/[^\w.-]/g, '_') // Replace invalid characters with underscores + .replace(/^[^A-Z_a-z]/, '_'); // Ensure key starts with a letter or underscore + +const preprocessJson = (obj) => { + if (typeof obj !== 'object' || obj === null) { + return obj; // Return primitive values as is + } + + if (Array.isArray(obj)) { + return obj.map(preprocessJson); + } + + return Object.entries(obj).reduce((acc, [key, value]) => { + const sanitizedKey = sanitizeKey(key); + acc[sanitizedKey] = preprocessJson(value); + return acc; + }, {}); +}; + +const getXMLPayload = (payload, rootKey = 'root') => { + const builderOptions = { + ignoreAttributes: false, + suppressEmptyNode: true, + }; + + const builder = new XMLBuilder(builderOptions); + const processesPayload = { + [rootKey]: { + ...preprocessJson(payload), + }, + }; + return `${builder.build(processesPayload)}`; +}; const getMergedEvents = (batch) => { const events = []; @@ -94,6 +164,39 @@ const getMergedEvents = (batch) => { return events; }; +const metadataHeaders = (contentType) => { + switch (contentType) { + case CONTENT_TYPES_MAP.XML: + return { 'Content-Type': 'application/xml' }; + case CONTENT_TYPES_MAP.FORM: + return { 'Content-Type': 'application/x-www-form-urlencoded' }; + default: + return { 'Content-Type': 'application/json' }; + } +}; + +function stringifyFirstLevelValues(obj) { + return Object.entries(obj).reduce((acc, [key, value]) => { + acc[key] = typeof value === 'string' ? value : JSON.stringify(value); + return acc; + }, {}); +} + +const prepareBody = (payload, contentType, xmlRootKey) => { + let responseBody; + removeUndefinedAndNullRecurse(payload); + if (contentType === CONTENT_TYPES_MAP.XML && !isEmptyObject(payload)) { + responseBody = { + payload: getXMLPayload(payload, xmlRootKey), + }; + } else if (contentType === CONTENT_TYPES_MAP.FORM && !isEmptyObject(payload)) { + responseBody = stringifyFirstLevelValues(payload); + } else { + responseBody = payload || {}; + } + return responseBody; +}; + const mergeMetadata = (batch) => batch.map((event) => event.metadata[0]); const createHashKey = (endpoint, headers, params) => { @@ -147,10 +250,14 @@ const batchSuccessfulEvents = (events, batchSize) => { }; module.exports = { + CONTENT_TYPES_MAP, getAuthHeaders, + enhanceMappings, getCustomMappings, - addPathParams, - excludeMappedFields, - getXMLPayload, + encodeParamsObject, + prepareEndpoint, + metadataHeaders, + prepareBody, batchSuccessfulEvents, + stringifyFirstLevelValues, }; diff --git a/src/cdk/v2/destinations/http/utils.test.js b/src/cdk/v2/destinations/http/utils.test.js new file mode 100644 index 00000000000..5b6eca90a4c --- /dev/null +++ b/src/cdk/v2/destinations/http/utils.test.js @@ -0,0 +1,169 @@ +const { + enhanceMappings, + encodeParamsObject, + prepareEndpoint, + prepareBody, + stringifyFirstLevelValues, +} = require('./utils'); + +const { XMLBuilder } = require('fast-xml-parser'); +const jsonpath = require('rs-jsonpath'); + +describe('Utils Functions', () => { + describe('encodeParamsObject', () => { + test('should return empty object for invalid inputs', () => { + expect(encodeParamsObject(null)).toEqual({}); + expect(encodeParamsObject(undefined)).toEqual({}); + expect(encodeParamsObject('string')).toEqual({}); + }); + + test('should encode object keys and values', () => { + const params = { key1: 'value1', key2: 'value2 3 4' }; + const expected = { key1: 'value1', key2: 'value2%203%204' }; + expect(encodeParamsObject(params)).toEqual(expected); + }); + }); + + describe('prepareEndpoint', () => { + test('should replace template variables in API URL', () => { + const message = { id: 123 }; + const apiUrl = 'https://api.example.com/resource/'; + expect(prepareEndpoint(message, apiUrl, [])).toBe('https://api.example.com/resource'); + }); + test('should replace template variables in API URL and add path params', () => { + const message = { id: 123, p2: 'P2' }; + const apiUrl = 'https://api.example.com/resource/'; + const pathParams = [ + { + path: 'p1', + }, + { + path: '$.p2', + }, + ]; + expect(prepareEndpoint(message, apiUrl, pathParams)).toBe( + 'https://api.example.com/resource/p1/P2', + ); + }); + test('should add path params after uri encoding', () => { + const message = { id: 123, p2: 'P2%&' }; + const apiUrl = 'https://api.example.com/resource/'; + const pathParams = [ + { + path: 'p1', + }, + { + path: '$.p2', + }, + ]; + expect(prepareEndpoint(message, apiUrl, pathParams)).toBe( + 'https://api.example.com/resource/p1/P2%25%26', + ); + }); + test('should throw error as path contains slash', () => { + const message = { id: 123, p2: 'P2%&' }; + const apiUrl = 'https://api.example.com/resource/${$.id}'; + const pathParams = [ + { + path: 'p1/', + }, + { + path: '$.p2', + }, + ]; + expect(() => prepareEndpoint(message, apiUrl, pathParams)).toThrowError( + 'Path value cannot contain "/"', + ); + }); + }); + + describe('prepareBody', () => { + test('should prepare XML payload when content type is XML', () => { + const payload = { key: 'value' }; + const expectedXML = 'value'; + const result = prepareBody(payload, 'XML', 'root'); + expect(result).toEqual({ payload: expectedXML }); + }); + + test('should prepare FORM-URLENCODED payload when content type is FORM-URLENCODED', () => { + const payload = { + key1: 'value1', + key2: 'value2', + key3: { subKey: 'value3', subkey2: undefined }, + }; + const expectedFORM = { key1: 'value1', key2: 'value2', key3: '{"subKey":"value3"}' }; + const result = prepareBody(payload, 'FORM'); + expect(result).toEqual(expectedFORM); + }); + + test('should return original payload without null or undefined values for other content types', () => { + const payload = { + key1: 'value1', + key2: null, + key3: undefined, + key4: 'value4', + key5: { subKey1: undefined, subKey2: 'value5' }, + }; + const expected = { key1: 'value1', key4: 'value4', key5: { subKey2: 'value5' } }; + const result = prepareBody(payload, 'JSON'); + expect(result).toEqual(expected); + }); + }); + + describe('stringifyFirstLevelValues', () => { + test('converts non-string first-level values to strings', () => { + const input = { a: 1, b: true, c: { d: 42 } }; + const expected = { a: '1', b: 'true', c: '{"d":42}' }; + expect(stringifyFirstLevelValues(input)).toEqual(expected); + }); + + test('keeps string values unchanged', () => { + const input = { a: 'hello', b: 'world' }; + expect(stringifyFirstLevelValues(input)).toEqual(input); + }); + + test('handles empty objects', () => { + expect(stringifyFirstLevelValues({})).toEqual({}); + }); + }); + + describe('enhanceMappings function', () => { + test("should wrap 'from' property in single quotes if it is not already wrapped and does not contain '$'", () => { + const input = [{ to: 'a', from: 'b' }]; + const output = enhanceMappings(input); + expect(output).toEqual([{ to: 'a', from: "'b'" }]); + }); + + test("should not modify 'from' property if it is already wrapped in single quotes", () => { + const input = [{ to: 'a', from: "'b'" }]; + const output = enhanceMappings(input); + expect(output).toEqual([{ to: 'a', from: "'b'" }]); + }); + + test("should not modify 'from' property if it contains '$'", () => { + const input = [{ to: 'a', from: '$.b' }]; + const output = enhanceMappings(input); + expect(output).toEqual([{ to: 'a', from: '$.b' }]); + }); + + test('should return an empty array if input is an empty array', () => { + const input = []; + const output = enhanceMappings(input); + expect(output).toEqual([]); + }); + + test('should correctly handle multiple mappings in an array', () => { + const input = [ + { to: 'a', from: 'b' }, + { to: 'x', from: "'y'" }, + { to: 'p', from: '$.q' }, + ]; + const output = enhanceMappings(input); + expect(output).toEqual([ + { to: 'a', from: "'b'" }, + { to: 'x', from: "'y'" }, + { to: 'p', from: '$.q' }, + ]); + }); + }); +}); diff --git a/src/constants/destinationCanonicalNames.js b/src/constants/destinationCanonicalNames.js index e9b7dc136b9..819ada2aa1b 100644 --- a/src/constants/destinationCanonicalNames.js +++ b/src/constants/destinationCanonicalNames.js @@ -187,6 +187,7 @@ const DestCanonicalNames = { wunderkind: ['wunderkind', 'Wunderkind', 'WUNDERKIND'], cordial: ['cordial', 'Cordial', 'CORDIAL'], clevertap: ['clevertap', 'Clevertap', 'CleverTap', 'CLEVERTAP'], + airship: ['airship', 'Airship', 'AIRSHIP'], }; module.exports = { DestHandlerMap, DestCanonicalNames }; diff --git a/src/features.ts b/src/features.ts index fb91ae28833..460f2720aff 100644 --- a/src/features.ts +++ b/src/features.ts @@ -94,6 +94,7 @@ const defaultFeaturesConfig: FeaturesConfig = { INTERCOM_V2: true, LINKEDIN_AUDIENCE: true, TOPSORT: true, + CUSTOMERIO_AUDIENCE: true, }, regulations: [ 'BRAZE', diff --git a/src/middleware.test.js b/src/middleware.test.js new file mode 100644 index 00000000000..397dedfbd07 --- /dev/null +++ b/src/middleware.test.js @@ -0,0 +1,107 @@ +const Koa = require('koa'); // Import Koa +const { + addStatMiddleware, + addRequestSizeMiddleware, + getHeapProfile, + getCPUProfile, + initPyroscope, +} = require('./middleware'); + +const Pyroscope = require('@pyroscope/nodejs'); +const stats = require('./util/stats'); +const { getDestTypeFromContext } = require('@rudderstack/integrations-lib'); + +// Mock dependencies +jest.mock('@pyroscope/nodejs'); +jest.mock('./util/stats', () => ({ + timing: jest.fn(), + histogram: jest.fn(), +})); +jest.mock('@rudderstack/integrations-lib', () => ({ + getDestTypeFromContext: jest.fn(), +})); + +describe('Pyroscope Initialization', () => { + it('should initialize Pyroscope with the correct app name', () => { + initPyroscope(); + expect(Pyroscope.init).toHaveBeenCalledWith({ appName: 'rudder-transformer' }); + expect(Pyroscope.startHeapCollecting).toHaveBeenCalled(); + }); +}); + +describe('getCPUProfile', () => { + it('should call Pyroscope.collectCpu with the specified seconds', () => { + const seconds = 5; + getCPUProfile(seconds); + expect(Pyroscope.collectCpu).toHaveBeenCalledWith(seconds); + }); +}); + +describe('getHeapProfile', () => { + it('should call Pyroscope.collectHeap', () => { + getHeapProfile(); + expect(Pyroscope.collectHeap).toHaveBeenCalled(); + }); +}); + +describe('durationMiddleware', () => { + it('should record the duration of the request', async () => { + // Mock getDestTypeFromContext to return a fixed value + getDestTypeFromContext.mockReturnValue('mock-destination-type'); + + const app = new Koa(); // Create a Koa app instance + addStatMiddleware(app); // Pass the app instance to the middleware + + const ctx = { + method: 'GET', + status: 200, + request: { url: '/test' }, + }; + const next = jest.fn().mockResolvedValue(null); + + // Simulate the middleware execution + await app.middleware[0](ctx, next); + + expect(stats.timing).toHaveBeenCalledWith('http_request_duration', expect.any(Date), { + method: 'GET', + code: 200, + route: '/test', + destType: 'mock-destination-type', // Mocked value + }); + }); +}); + +describe('requestSizeMiddleware', () => { + it('should record the size of the request and response', async () => { + const app = new Koa(); // Create a Koa app instance + addRequestSizeMiddleware(app); // Pass the app instance to the middleware + + const ctx = { + method: 'POST', + status: 200, + request: { + url: '/test', + body: { key: 'value' }, + }, + response: { + body: { success: true }, + }, + }; + const next = jest.fn().mockResolvedValue(null); + + // Simulate the middleware execution + await app.middleware[0](ctx, next); + + expect(stats.histogram).toHaveBeenCalledWith('http_request_size', expect.any(Number), { + method: 'POST', + code: 200, + route: '/test', + }); + + expect(stats.histogram).toHaveBeenCalledWith('http_response_size', expect.any(Number), { + method: 'POST', + code: 200, + route: '/test', + }); + }); +}); diff --git a/src/middlewares/stats.ts b/src/middlewares/stats.ts new file mode 100644 index 00000000000..6fe0cc1c2cd --- /dev/null +++ b/src/middlewares/stats.ts @@ -0,0 +1,15 @@ +import { Context, Next } from 'koa'; + +export class StatsMiddleware { + private static instanceID: string = process.env.INSTANCE_ID || 'default'; + + private static workerID: string = process.env.WORKER_ID || 'master'; + + public static async executionStats(ctx: Context, next: Next) { + const start = Date.now(); + await next(); + const ms = Date.now() - start; + ctx.set('X-Response-Time', `${ms}ms`); + ctx.set('X-Instance-ID', `${StatsMiddleware.instanceID}/${StatsMiddleware.workerID}`); + } +} diff --git a/src/routerUtils.js b/src/routerUtils.js index 081070d78ac..67b6e0d31c4 100644 --- a/src/routerUtils.js +++ b/src/routerUtils.js @@ -4,13 +4,7 @@ const logger = require('./logger'); const { proxyRequest } = require('./adapters/network'); const { nodeSysErrorToStatus } = require('./adapters/utils/networkUtils'); -let areFunctionsEnabled = -1; -const functionsEnabled = () => { - if (areFunctionsEnabled === -1) { - areFunctionsEnabled = process.env.ENABLE_FUNCTIONS === 'false' ? 0 : 1; - } - return areFunctionsEnabled === 1; -}; +const functionsEnabled = () => process.env.ENABLE_FUNCTIONS !== 'false'; const userTransformHandler = () => { if (functionsEnabled()) { diff --git a/src/routerUtils.test.js b/src/routerUtils.test.js new file mode 100644 index 00000000000..81c8ed49919 --- /dev/null +++ b/src/routerUtils.test.js @@ -0,0 +1,126 @@ +const { sendToDestination, userTransformHandler } = require('./routerUtils'); // Update the path accordingly + +const logger = require('./logger'); +const { proxyRequest } = require('./adapters/network'); +const { nodeSysErrorToStatus } = require('./adapters/utils/networkUtils'); + +// Mock dependencies +jest.mock('./logger'); +jest.mock('./adapters/network'); +jest.mock('./adapters/utils/networkUtils'); + +describe('sendToDestination', () => { + beforeEach(() => { + jest.clearAllMocks(); // Clear mocks before each test + }); + + it('should send a request to the destination and return a successful response', async () => { + // Mock proxyRequest to return a successful response + proxyRequest.mockResolvedValue({ + success: true, + response: { + headers: { 'content-type': 'application/json' }, + data: { message: 'Success' }, + status: 200, + }, + }); + + const destination = 'mock-destination'; + const payload = { key: 'value' }; + + const result = await sendToDestination(destination, payload); + + expect(logger.info).toHaveBeenCalledWith('Request recieved for destination', destination); + expect(proxyRequest).toHaveBeenCalledWith(payload); + expect(result).toEqual({ + headers: { 'content-type': 'application/json' }, + response: { message: 'Success' }, + status: 200, + }); + }); + + it('should handle network failure and return a parsed response', async () => { + // Mock proxyRequest to return a network failure + proxyRequest.mockResolvedValue({ + success: false, + response: { + code: 'ENOTFOUND', // Simulate a network error + }, + }); + + // Mock nodeSysErrorToStatus to return a specific error message and status + nodeSysErrorToStatus.mockReturnValue({ + message: 'Network error', + status: 500, + }); + + const destination = 'mock-destination'; + const payload = { key: 'value' }; + + const result = await sendToDestination(destination, payload); + + expect(logger.info).toHaveBeenCalledWith('Request recieved for destination', destination); + expect(proxyRequest).toHaveBeenCalledWith(payload); + expect(nodeSysErrorToStatus).toHaveBeenCalledWith('ENOTFOUND'); + expect(result).toEqual({ + headers: null, + networkFailure: true, + response: 'Network error', + status: 500, + }); + }); + + it('should handle axios error with response and return a parsed response', async () => { + // Mock proxyRequest to return an axios error with response + proxyRequest.mockResolvedValue({ + success: false, + response: { + response: { + headers: { 'content-type': 'application/json' }, + status: 400, + data: 'Bad Request', + }, + }, + }); + + const destination = 'mock-destination'; + const payload = { key: 'value' }; + + const result = await sendToDestination(destination, payload); + + expect(logger.info).toHaveBeenCalledWith('Request recieved for destination', destination); + expect(proxyRequest).toHaveBeenCalledWith(payload); + expect(result).toEqual({ + headers: { 'content-type': 'application/json' }, + status: 400, + response: 'Bad Request', + }); + }); +}); + +describe('userTransformHandler', () => { + beforeEach(() => { + jest.clearAllMocks(); // Clear mocks before each test + jest.resetModules(); // Reset modules to reset process.env + }); + + it('should return userTransformHandler when functions are enabled', () => { + // Mock process.env to enable functions + process.env.ENABLE_FUNCTIONS = 'true'; + + const mockUserTransformHandler = jest.fn(); + jest.mock('./util/customTransformer', () => ({ + userTransformHandler: mockUserTransformHandler, + })); + + const result = userTransformHandler(); + expect(result).toBe(mockUserTransformHandler); + }); + + it('should throw an error when functions are not enabled', () => { + // Mock process.env to disable functions + process.env.ENABLE_FUNCTIONS = 'false'; + + expect(() => userTransformHandler()).toThrow('Functions are not enabled'); + }); +}); diff --git a/src/routes/userTransform.ts b/src/routes/userTransform.ts index fc61ab7b941..e2883bdc22f 100644 --- a/src/routes/userTransform.ts +++ b/src/routes/userTransform.ts @@ -1,7 +1,8 @@ import Router from '@koa/router'; -import { RouteActivationMiddleware } from '../middlewares/routeActivation'; -import { FeatureFlagMiddleware } from '../middlewares/featureFlag'; import { UserTransformController } from '../controllers/userTransform'; +import { FeatureFlagMiddleware } from '../middlewares/featureFlag'; +import { RouteActivationMiddleware } from '../middlewares/routeActivation'; +import { StatsMiddleware } from '../middlewares/stats'; const router = new Router(); @@ -15,6 +16,7 @@ router.post( '/customTransform', RouteActivationMiddleware.isUserTransformRouteActive, FeatureFlagMiddleware.handle, + StatsMiddleware.executionStats, UserTransformController.transform, ); router.post( diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts index 2afad88c56a..345985d5179 100644 --- a/src/services/userTransform.ts +++ b/src/services/userTransform.ts @@ -38,7 +38,6 @@ export class UserTransformService { `${event.metadata.destinationId}_${event.metadata.sourceId}`, ); stats.counter('user_transform_function_group_size', Object.entries(groupedEvents).length, {}); - stats.histogram('user_transform_input_events', events.length, {}); const transformedEvents: FixMe[] = []; let librariesVersionIDs: FixMe[] = []; @@ -63,11 +62,12 @@ export class UserTransformService { const messageIdsInOutputSet = new Set(); + const workspaceId = eventsToProcess[0]?.metadata.workspaceId; const commonMetadata = { sourceId: eventsToProcess[0]?.metadata?.sourceId, destinationId: eventsToProcess[0]?.metadata.destinationId, destinationType: eventsToProcess[0]?.metadata.destinationType, - workspaceId: eventsToProcess[0]?.metadata.workspaceId, + workspaceId, transformationId: eventsToProcess[0]?.metadata.transformationId, messageIds, }; @@ -76,6 +76,7 @@ export class UserTransformService { eventsToProcess.length > 0 && eventsToProcess[0].metadata ? getMetadata(eventsToProcess[0].metadata) : {}; + const transformationTags = getTransformationMetadata(eventsToProcess[0]?.metadata); if (!transformationVersionId) { const errorMessage = 'Transformation VersionID not found'; @@ -87,6 +88,11 @@ export class UserTransformService { } as ProcessorTransformationResponse); return transformedEvents; } + stats.counter('user_transform_input_events', events.length, { workspaceId }); + logger.info('user_transform_input_events', { + inCount: events.length, + ...transformationTags, + }); const userFuncStartTime = new Date(); try { const destTransformedEvents: UserTransformationResponse[] = await userTransformHandler()( @@ -167,22 +173,28 @@ export class UserTransformService { stats.counter('user_transform_errors', eventsToProcess.length, { status, ...metaTags, - ...getTransformationMetadata(eventsToProcess[0]?.metadata), + ...transformationTags, }); } finally { stats.timingSummary('user_transform_request_latency_summary', userFuncStartTime, { ...metaTags, - ...getTransformationMetadata(eventsToProcess[0]?.metadata), + ...transformationTags, }); stats.summary('user_transform_batch_size_summary', requestSize, { ...metaTags, - ...getTransformationMetadata(eventsToProcess[0]?.metadata), + ...transformationTags, }); } stats.counter('user_transform_requests', 1, {}); - stats.histogram('user_transform_output_events', transformedEvents.length, {}); + stats.counter('user_transform_output_events', transformedEvents.length, { + workspaceId, + }); + logger.info('user_transform_output_events', { + outCount: transformedEvents.length, + ...transformationTags, + }); return transformedEvents; }), ); diff --git a/src/types/index.ts b/src/types/index.ts index 7c07f659df3..a8e05d3ed62 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -122,11 +122,11 @@ type DestinationDefinition = { Config: FixMe; }; -type Destination = { +type Destination = { ID: string; Name: string; DestinationDefinition: DestinationDefinition; - Config: FixMe; + Config: DestinationConfig; Enabled: boolean; WorkspaceID: string; Transformations: UserTransformationInput[]; @@ -164,12 +164,16 @@ type ProcessorTransformationRequest = { credentials?: Credential[]; }; -type RouterTransformationRequestData = { +type RouterTransformationRequestData< + Message = object, + DestinationType = Destination, + ConnectionType = Connection, +> = { request?: object; - message: object; + message: Message; metadata: Metadata; - destination: Destination; - connection?: Connection; + destination: DestinationType; + connection?: ConnectionType; }; type RouterTransformationRequest = { diff --git a/src/util/cluster.js b/src/util/cluster.js index b9b86cd3c61..6be25018f1d 100644 --- a/src/util/cluster.js +++ b/src/util/cluster.js @@ -28,6 +28,7 @@ async function shutdownWorkers() { } function start(port, app, metricsApp) { + if (cluster.isMaster) { logger.info(`Master (pid: ${process.pid}) has started`); @@ -44,7 +45,9 @@ function start(port, app, metricsApp) { // Fork workers. for (let i = 0; i < numWorkers; i += 1) { - cluster.fork(); + cluster.fork({ + WORKER_ID: `worker-${i + 1}`, + }); } cluster.on('online', (worker) => { diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index 0f59f5db60f..d8bf556d893 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -34,7 +34,6 @@ function generateFunctionName(userTransformation, libraryVersionIds, testMode, h ids = ids.concat([hashSecret]); } - // FIXME: Why the id's are sorted ?! const hash = crypto.createHash('md5').update(`${ids}`).digest('hex'); return `fn-${userTransformation.workspaceId}-${hash}`.substring(0, 63).toLowerCase(); } diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js index 98f3340cad8..ec3604f066f 100644 --- a/src/util/openfaas/index.js +++ b/src/util/openfaas/index.js @@ -401,7 +401,7 @@ const executeFaasFunction = async ( throw new RetryRequestError(`Rate limit exceeded for ${name}`); } - if (error.statusCode === 500 || error.statusCode === 503) { + if (error.statusCode === 500 || error.statusCode === 503 || error.statusCode === 502) { throw new RetryRequestError(error.message); } diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 2a3a1fb22a8..af3f1c5fc19 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -445,6 +445,12 @@ class Prometheus { type: 'counter', labelNames: ['event', 'writeKey'], }, + { + name: 'shopify_pixel_cart_token_not_found_server_side', + help: 'shopify_pixel_cart_token_not_found_server_side', + type: 'counter', + labelNames: ['event', 'writeKey'], + }, { name: 'shopify_pixel_cart_token_set', help: 'shopify_pixel_cart_token_set', @@ -858,6 +864,18 @@ class Prometheus { { name: 'get_tracking_plan', help: 'get_tracking_plan', type: 'histogram', labelNames: [] }, // User transform metrics // counter + { + name: 'user_transform_input_events', + help: 'Number of input events to user transform', + type: 'counter', + labelNames: ['workspaceId'], + }, + { + name: 'user_transform_output_events', + help: 'user_transform_output_events', + type: 'counter', + labelNames: ['workspaceId'], + }, { name: 'user_transform_function_group_size', help: 'user_transform_function_group_size', @@ -948,20 +966,6 @@ class Prometheus { type: 'histogram', labelNames: ['identifier', 'transformationId', 'workspaceId'], }, - { - name: 'user_transform_input_events', - help: 'Number of input events to user transform', - type: 'histogram', - labelNames: [], - buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], - }, - { - name: 'user_transform_output_events', - help: 'user_transform_output_events', - type: 'histogram', - labelNames: [], - buckets: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 150, 200], - }, // summary { name: 'user_transform_request_latency_summary', diff --git a/src/util/utils.js b/src/util/utils.js index 82c85b41a58..36554ff5d0a 100644 --- a/src/util/utils.js +++ b/src/util/utils.js @@ -44,35 +44,35 @@ const fetchAddressFromHostName = async (hostname) => { return { address, cacheHit: false }; }; -const staticLookup = (transformationTags) => async (hostname, _, cb) => { - let ip; - const resolveStartTime = new Date(); - try { - const { address, cacheHit } = await fetchAddressFromHostName(hostname); - ip = address; - stats.timing('fetch_dns_resolve_time', resolveStartTime, { ...transformationTags, cacheHit }); - } catch (error) { - logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); - stats.timing('fetch_dns_resolve_time', resolveStartTime, { - ...transformationTags, - error: 'true', - }); - cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); - return; - } - - if (!ip) { - cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); - return; - } - - if (ip.startsWith(LOCALHOST_OCTET)) { - cb(null, `cannot use ${ip} as IP address`, RECORD_TYPE_A); - return; - } - - cb(null, ip, RECORD_TYPE_A); -}; +const staticLookup = + (transformationTags, fetchAddress = fetchAddressFromHostName) => + (hostname, _, cb) => { + const resolveStartTime = new Date(); + + fetchAddress(hostname) + .then(({ address, cacheHit }) => { + stats.timing('fetch_dns_resolve_time', resolveStartTime, { + ...transformationTags, + cacheHit, + }); + + if (!address) { + cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); + } else if (address.startsWith(LOCALHOST_OCTET)) { + cb(null, `cannot use ${address} as IP address`, RECORD_TYPE_A); + } else { + cb(null, address, RECORD_TYPE_A); + } + }) + .catch((error) => { + logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); + stats.timing('fetch_dns_resolve_time', resolveStartTime, { + ...transformationTags, + error: 'true', + }); + cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); + }); + }; const httpAgentWithDnsLookup = (scheme, transformationTags) => { const httpModule = scheme === 'http' ? http : https; @@ -226,4 +226,5 @@ module.exports = { logProcessInfo, extractStackTraceUptoLastSubstringMatch, fetchWithDnsWrapper, + staticLookup, }; diff --git a/src/util/utils.test.js b/src/util/utils.test.js new file mode 100644 index 00000000000..d7974e3ff61 --- /dev/null +++ b/src/util/utils.test.js @@ -0,0 +1,71 @@ +const { staticLookup } = require('./utils'); + +describe('staticLookup', () => { + const transformationTags = { tag: 'value' }; + const RECORD_TYPE_A = 4; + const HOST_NAME = 'example.com'; + const fetchAddressFromHostName = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should resolve the hostname and return the IP address', async () => { + const mockAddress = '192.168.1.1'; + fetchAddressFromHostName.mockResolvedValueOnce({ address: mockAddress, cacheHit: true }); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual(null, mockAddress, RECORD_TYPE_A); + }; + resolve(HOST_NAME, null, callback); + }); + + it('should handle errors from fetchAddressFromHostName', async () => { + const error = new Error('DNS error'); + error.code = 'ENOTFOUND'; + fetchAddressFromHostName.mockRejectedValueOnce(error); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual(null, `unable to resolve IP address for ${HOST_NAME}`, RECORD_TYPE_A); + }; + resolve(HOST_NAME, null, callback); + }); + + it('should handle empty address', async () => { + fetchAddressFromHostName.mockResolvedValueOnce({ address: '', cacheHit: true }); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual( + null, + `resolved empty list of IP address for ${HOST_NAME}`, + RECORD_TYPE_A, + ); + }; + resolve(HOST_NAME, null, callback); + }); + + it('should handle localhost address', async () => { + const LOCALHOST_OCTET = '127'; + fetchAddressFromHostName.mockResolvedValueOnce({ + address: `${LOCALHOST_OCTET}.0.0.1`, + cacheHit: true, + }); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual( + null, + `cannot use ${LOCALHOST_OCTET}.0.0.1 as IP address`, + RECORD_TYPE_A, + ); + }; + resolve(HOST_NAME, null, callback); + }); +}); diff --git a/src/util/worker.js b/src/util/worker.js index d1b7b24d969..df6cbf4984d 100644 --- a/src/util/worker.js +++ b/src/util/worker.js @@ -4,7 +4,7 @@ const { MESSAGE_TYPES } = require('./metricsAggregator'); const { AggregatorRegistry } = require('prom-client'); parentPort.on('message', async (message) => { - if ((message.type = MESSAGE_TYPES.AGGREGATE_METRICS_REQ)) { + if (message.type === MESSAGE_TYPES.AGGREGATE_METRICS_REQ) { try { const promString = await AggregatorRegistry.aggregate(message.metrics).metrics(); parentPort.postMessage({ type: MESSAGE_TYPES.AGGREGATE_METRICS_RES, metrics: promString }); diff --git a/src/v0/destinations/active_campaign/transform.js b/src/v0/destinations/active_campaign/transform.js index f21bb1a70d4..3fd5229c371 100644 --- a/src/v0/destinations/active_campaign/transform.js +++ b/src/v0/destinations/active_campaign/transform.js @@ -95,7 +95,7 @@ const customTagProcessor = async ({ message, destination, metadata }, category, // Step - 1 // Fetch already created tags from dest, so that we avoid duplicate tag creation request // Ref - https://developers.activecampaign.com/reference/retrieve-all-tags - endpoint = `${destination.Config.apiUrl}${`${tagEndPoint}?limit=100`}`; + endpoint = `${destination.Config.apiUrl}${tagEndPoint}?limit=100`; requestOptions = { headers: getHeader(destination), }; diff --git a/src/v0/destinations/adobe_analytics/utils.js b/src/v0/destinations/adobe_analytics/utils.js index ceba177ff14..e5196a30d48 100644 --- a/src/v0/destinations/adobe_analytics/utils.js +++ b/src/v0/destinations/adobe_analytics/utils.js @@ -97,6 +97,25 @@ function escapeToHTML(inputString) { ); } +/** + * Tries to find a value from alternative sources based on the source key. + * @param {Object} message - The input message object. + * @param {string} sourceKey - The key to look for. + * @returns {any} - The found value or null. + */ +function findValueFromSources(message, sourceKey) { + // Try alternative sources defined in SOURCE_KEYS + for (const source of SOURCE_KEYS) { + const value = getMappingFieldValueFormMessage(message, source, sourceKey); + if (isDefinedAndNotNull(value)) { + return value; + } + } + + // Fallback to value retrieval by path + return getValueByPath(message, sourceKey); +} + /** * This function is used for populating the eVars and hVars in the payload * @param {*} destVarMapping @@ -106,31 +125,28 @@ function escapeToHTML(inputString) { * @returns updated paylaod with eVars and hVars added */ function rudderPropToDestMap(destVarMapping, message, payload, destVarStrPrefix) { - const mappedVar = {}; - // pass the Rudder Property mapped in the ui whose evar you want to map - Object.keys(destVarMapping).forEach((key) => { - let val = get(message, `properties.${key}`); - if (isDefinedAndNotNull(val)) { - const destVarKey = destVarStrPrefix + destVarMapping[key]; - mappedVar[destVarKey] = escapeToHTML(val); - } else { - SOURCE_KEYS.some((sourceKey) => { - val = getMappingFieldValueFormMessage(message, sourceKey, key); - if (isDefinedAndNotNull(val)) { - mappedVar[`${destVarStrPrefix}${[destVarMapping[key]]}`] = escapeToHTML(val); - } else { - val = getValueByPath(message, key); - if (isDefinedAndNotNull(val)) { - mappedVar[`${destVarStrPrefix}${[destVarMapping[key]]}`] = escapeToHTML(val); - } - } - }); + const mappedVariables = {}; + + // Iterate over each key in the destination variable mapping + Object.keys(destVarMapping).forEach((sourceKey) => { + let value = message?.properties?.[sourceKey]; + + if (!isDefinedAndNotNull(value)) { + // Try getting the value from alternative sources + value = findValueFromSources(message, sourceKey); + } + + if (isDefinedAndNotNull(value)) { + const destinationKey = `${destVarStrPrefix}${destVarMapping[sourceKey]}`; + mappedVariables[destinationKey] = escapeToHTML(value); } }); - if (Object.keys(mappedVar).length > 0) { - // non-empty object - Object.assign(payload, mappedVar); + + // Add non-empty mapped variables to the payload + if (Object.keys(mappedVariables).length > 0) { + Object.assign(payload, mappedVariables); } + return payload; } diff --git a/src/v0/destinations/adobe_analytics/utils.test.js b/src/v0/destinations/adobe_analytics/utils.test.js new file mode 100644 index 00000000000..df40bf2ff31 --- /dev/null +++ b/src/v0/destinations/adobe_analytics/utils.test.js @@ -0,0 +1,271 @@ +const { + handleContextData, + handleEvar, + handleHier, + handleList, + handleCustomProperties, + stringifyValueAndJoinWithDelimiter, + escapeToHTML, +} = require('./utils'); // Update the path accordingly + +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + +describe('handleContextData', () => { + it('should add context data to the payload when values are found', () => { + const payload = {}; + const destinationConfig = { + contextDataPrefix: 'c_', + contextDataMapping: { + 'user.id': 'userId', + 'user.email': 'userEmail', + }, + }; + const message = { + user: { + id: '123', + email: 'test@example.com', + }, + }; + + const result = handleContextData(payload, destinationConfig, message); + + expect(result.contextData).toEqual({ + c_userId: '123', + c_userEmail: 'test@example.com', + }); + }); + + it('should not add context data to the payload when no values are found', () => { + const payload = {}; + const destinationConfig = { + contextDataPrefix: 'c_', + contextDataMapping: { + 'user.id': 'userId', + 'user.email': 'userEmail', + }, + }; + const message = { + user: { + name: 'John Doe', + }, + }; + + const result = handleContextData(payload, destinationConfig, message); + + expect(result.contextData).toBeUndefined(); + }); +}); + +describe('handleEvar', () => { + it('should map properties to eVars in the payload', () => { + const payload = {}; + const destinationConfig = { + eVarMapping: { + productId: '1', + category: '2', + }, + }; + const message = { + properties: { + productId: 'p123', + category: 'electronics', + }, + }; + + const result = handleEvar(payload, destinationConfig, message); + + expect(result).toEqual({ + eVar1: 'p123', + eVar2: 'electronics', + }); + }); + + it('should not add eVars to the payload when no values are found', () => { + const payload = {}; + const destinationConfig = { + eVarMapping: { + productId: '1', + category: '2', + }, + }; + const message = { + properties: { + name: 'Product Name', + }, + }; + + const result = handleEvar(payload, destinationConfig, message); + + expect(result).toEqual({}); + }); +}); + +describe('handleHier', () => { + it('should map properties to hVars in the payload', () => { + const payload = {}; + const destinationConfig = { + hierMapping: { + section: '1', + subsection: '2', + }, + }; + const message = { + properties: { + section: 'home', + subsection: 'kitchen', + }, + }; + + const result = handleHier(payload, destinationConfig, message); + + expect(result).toEqual({ + hier1: 'home', + hier2: 'kitchen', + }); + }); + + it('should not add hVars to the payload when no values are found', () => { + const payload = {}; + const destinationConfig = { + hierMapping: { + section: '1', + subsection: '2', + }, + }; + const message = { + properties: { + name: 'Section Name', + }, + }; + + const result = handleHier(payload, destinationConfig, message); + + expect(result).toEqual({}); + }); +}); + +describe('handleList', () => { + it('should map properties to list variables in the payload', () => { + const payload = {}; + const destinationConfig = { + listMapping: { + products: '1', + }, + listDelimiter: { + products: ',', + }, + }; + const message = { + properties: { + products: ['p1', 'p2', 'p3'], + }, + }; + + const result = handleList(payload, destinationConfig, message); + + expect(result).toEqual({ + list1: 'p1,p2,p3', + }); + }); + + it('should throw an error when list properties are not strings or arrays', () => { + const payload = {}; + const destinationConfig = { + listMapping: { + products: '1', + }, + listDelimiter: { + products: ',', + }, + }; + const message = { + properties: { + products: 123, // Invalid type + }, + }; + + expect(() => handleList(payload, destinationConfig, message)).toThrow(InstrumentationError); + }); +}); + +describe('handleCustomProperties', () => { + it('should map properties to custom properties in the payload', () => { + const payload = {}; + const destinationConfig = { + customPropsMapping: { + color: '1', + size: '2', + }, + propsDelimiter: { + color: ',', + size: ';', + }, + }; + const message = { + properties: { + color: 'red,green,blue', + size: ['S', 'M', 'L'], + }, + }; + + const result = handleCustomProperties(payload, destinationConfig, message); + + expect(result).toEqual({ + prop1: 'red,green,blue', + prop2: 'S;M;L', + }); + }); + + it('should throw an error when custom properties are not strings or arrays', () => { + const payload = {}; + const destinationConfig = { + customPropsMapping: { + color: '1', + }, + propsDelimiter: { + color: ',', + }, + }; + const message = { + properties: { + color: 123, // Invalid type + }, + }; + + expect(() => handleCustomProperties(payload, destinationConfig, message)).toThrow( + InstrumentationError, + ); + }); +}); + +describe('stringifyValueAndJoinWithDelimiter', () => { + it('should join values with a delimiter after stringifying them', () => { + const values = [1, null, 'test', true]; + const result = stringifyValueAndJoinWithDelimiter(values, '|'); + + expect(result).toBe('1|null|test|true'); + }); + + it('should use the default delimiter if none is provided', () => { + const values = [1, 2, 3]; + const result = stringifyValueAndJoinWithDelimiter(values); + + expect(result).toBe('1;2;3'); + }); +}); + +describe('escapeToHTML', () => { + it('should escape HTML entities in a string', () => { + const input = '
&
'; + const result = escapeToHTML(input); + + expect(result).toBe('<div>&</div>'); + }); + + it('should return non-string values unchanged', () => { + const input = 123; + const result = escapeToHTML(input); + + expect(result).toBe(123); + }); +}); diff --git a/src/v0/destinations/airship/transform.js b/src/v0/destinations/airship/transform.js index 50b5f309553..d2244b542a3 100644 --- a/src/v0/destinations/airship/transform.js +++ b/src/v0/destinations/airship/transform.js @@ -7,7 +7,6 @@ const { groupMapping, BASE_URL_EU, BASE_URL_US, - RESERVED_TRAITS_MAPPING, AIRSHIP_TRACK_EXCLUSION, } = require('./config'); @@ -25,6 +24,7 @@ const { convertToUuid, } = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); +const { prepareAttributePayload, prepareTagPayload } = require('./utils'); const DEFAULT_ACCEPT_HEADER = 'application/vnd.urbanairship+json; version=3'; @@ -37,7 +37,7 @@ const transformSessionId = (rawSessionId) => { }; const identifyResponseBuilder = (message, { Config }) => { - const tagPayload = constructPayload(message, identifyMapping); + const initialTagPayload = constructPayload(message, identifyMapping); const { apiKey, dataCenter } = Config; if (!apiKey) @@ -55,38 +55,10 @@ const identifyResponseBuilder = (message, { Config }) => { } // Creating tags and attribute payload - tagPayload.add = { rudderstack_integration: [] }; - tagPayload.remove = { rudderstack_integration: [] }; - let timestamp = getFieldValueFromMessage(message, 'timestamp'); - timestamp = new Date(timestamp).toISOString().replace(/\.\d{3}/, ''); + const tagPayload = prepareTagPayload(traits, initialTagPayload); // Creating attribute payload - const attributePayload = { attributes: [] }; - Object.keys(traits).forEach((key) => { - // tags - if (typeof traits[key] === 'boolean') { - const tag = key.toLowerCase().replace(/\./g, '_'); - if (traits[key] === true) { - tagPayload.add.rudderstack_integration.push(tag); - } - if (traits[key] === false) { - tagPayload.remove.rudderstack_integration.push(tag); - } - } - // attribute - if (typeof traits[key] !== 'boolean') { - const attribute = { action: 'set' }; - const keyMapped = RESERVED_TRAITS_MAPPING[key] || RESERVED_TRAITS_MAPPING[key.toLowerCase()]; - if (keyMapped) { - attribute.key = keyMapped; - } else { - attribute.key = key.replace(/\./g, '_'); - } - attribute.value = traits[key]; - attribute.timestamp = timestamp; - attributePayload.attributes.push(attribute); - } - }); + const attributePayload = prepareAttributePayload(traits, message); let tagResponse; let attributeResponse; @@ -183,7 +155,7 @@ const trackResponseBuilder = async (message, { Config }) => { }; const groupResponseBuilder = (message, { Config }) => { - const tagPayload = constructPayload(message, groupMapping); + const initTagPayload = constructPayload(message, groupMapping); const { apiKey, dataCenter } = Config; if (!apiKey) @@ -200,37 +172,9 @@ const groupResponseBuilder = (message, { Config }) => { ); } - tagPayload.add = { rudderstack_integration_group: [] }; - tagPayload.remove = { rudderstack_integration_group: [] }; - let timestamp = getFieldValueFromMessage(message, 'timestamp'); - timestamp = new Date(timestamp).toISOString().replace(/\.\d{3}/, ''); - - const attributePayload = { attributes: [] }; - Object.keys(traits).forEach((key) => { - // tags - if (typeof traits[key] === 'boolean') { - const tag = key.toLowerCase().replace(/\./g, '_'); - if (traits[key] === true) { - tagPayload.add.rudderstack_integration_group.push(tag); - } - if (traits[key] === false) { - tagPayload.remove.rudderstack_integration_group.push(tag); - } - } - // attribute - if (typeof traits[key] !== 'boolean') { - const attribute = { action: 'set' }; - const keyMapped = RESERVED_TRAITS_MAPPING[key.toLowerCase()]; - if (keyMapped) { - attribute.key = keyMapped; - } else { - attribute.key = key.replace(/\./g, '_'); - } - attribute.value = traits[key]; - attribute.timestamp = timestamp; - attributePayload.attributes.push(attribute); - } - }); + const tagPayload = prepareTagPayload(traits, initTagPayload, 'group'); + + const attributePayload = prepareAttributePayload(traits, message); let tagResponse; let attributeResponse; diff --git a/src/v0/destinations/airship/utils.test.ts b/src/v0/destinations/airship/utils.test.ts new file mode 100644 index 00000000000..cf637abb73c --- /dev/null +++ b/src/v0/destinations/airship/utils.test.ts @@ -0,0 +1,433 @@ +import { RudderMessage } from '../../../types'; +import { convertToUuid, flattenJson, getFieldValueFromMessage } from '../../util'; +import { getAirshipTimestamp, isValidTimestamp, prepareAttributePayload } from './utils'; + +type timestampTc = { + description: string; + input: string; + output?: string; + error?: string; +}; + +describe('Airship utils - getAirshipTimestamp', () => { + const timestampCases: timestampTc[] = [ + { + description: 'should return the same timestamp', + input: '2025-01-23T12:00:00Z', + output: '2025-01-23T12:00:00Z', + }, + { + description: 'should remove milliseconds', + input: '2025-01-23T12:00:00.123Z', + output: '2025-01-23T12:00:00Z', + }, + { + description: 'should remove milliseconds - 2', + input: '2025-01-23T12:00:00.123456Z', + output: '2025-01-23T12:00:00Z', + }, + { + description: 'should return with correct format when T is present but Z is not present', + input: '2025-01-23T12:00:00', + output: '2025-01-23T12:00:00Z', + }, + { + description: 'should return with correct format when T & Z is not present', + input: '2025-01-23 12:00:00', + output: '2025-01-23T12:00:00Z', + }, + { + description: 'should throw error when timestamp is not supported', + input: 'abcd', + error: 'timestamp is not supported: abcd', + }, + { + description: + 'should return with correct format when timestamp contains microseconds without Z', + input: '2025-01-23T12:00:00.123456', + output: '2025-01-23T12:00:00Z', + }, + ]; + + test.each(timestampCases)('getAirshipTimestamp - $description', ({ input, output, error }) => { + const message = { + timestamp: input, + } as unknown as RudderMessage; + if (error) { + expect(() => getAirshipTimestamp(message)).toThrow(error); + } else { + const timestamp = getAirshipTimestamp(message); + expect(timestamp).toBe(output); + } + }); +}); + +describe('Airship utils - prepareAttributePayload', () => { + const commonContextProps = { + app: { + build: '1', + name: 'Polarsteps', + namespace: 'com.polarsteps.Polarsteps', + version: '8.2.11', + }, + device: { + attTrackingStatus: 0, + id: '1d89c859-76c6-4374-ac0a-e32dee541e12', + manufacturer: 'Apple', + model: 'arm64', + name: 'iPhone 14 Pro Max', + type: 'iOS', + }, + library: { + name: 'rudder-ios-library', + version: '1.31.0', + }, + locale: 'en-US', + network: { + cellular: false, + wifi: true, + }, + os: { + name: 'iOS', + version: '17.0', + }, + screen: { + density: 3, + height: 932, + width: 430, + }, + sessionId: 1736246350, + timezone: 'Europe/Amsterdam', + }; + const commonEventProps = { + anonymousId: 'd00de6f3-2ea3-44bd-8fc5-0c318cb0b9d9', + channel: 'mobile', + messageId: 'b95d29ee-f9c8-486c-b9ef-acf231759612', + originalTimestamp: '2025-01-07T10:57:38.768Z', + receivedAt: '2025-01-07T10:57:49.882Z', + request_ip: '77.248.183.43', + rudderId: 'b931e94b-3b22-462c-8b58-243cb4b37366', + sentAt: '2025-01-07T10:57:47.707Z', + userId: '1c5577e3-8d2d-4ecd-9361-88c2bfb254c5', + }; + it('should return the correct attribute payload when jsonAttributes is not present in integrations object ', () => { + const message = { + ...commonEventProps, + context: { + ...commonContextProps, + traits: { + af_install_time: '2024-12-09 12:26:29.643', + af_status: 'Organic', + firstName: 'Orcun', + lastName: 'Test', + widgets_installed: ['no_widgets_installed', 'widgets_installed'], + }, + }, + event: 'identify', + integrations: { + All: true, + }, + type: 'identify', + } as unknown as RudderMessage; + + const expectedAttributePayload = { + attributes: [ + { + action: 'set', + key: 'af_install_time', + value: '2024-12-09T12:26:29Z', + timestamp: '2025-01-07T10:57:38Z', + }, + { action: 'set', key: 'af_status', value: 'Organic', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'first_name', value: 'Orcun', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'last_name', value: 'Test', timestamp: '2025-01-07T10:57:38Z' }, + { + action: 'set', + key: 'widgets_installed[0]', + value: 'no_widgets_installed', + timestamp: '2025-01-07T10:57:38Z', + }, + { + action: 'set', + key: 'widgets_installed[1]', + value: 'widgets_installed', + timestamp: '2025-01-07T10:57:38Z', + }, + ], + }; + + const traits = getFieldValueFromMessage(message, 'traits'); + const flattenedTraits = flattenJson(traits); + // @ts-expect-error error with type + const attributePayload = prepareAttributePayload(flattenedTraits, message); + expect(attributePayload).toEqual(expectedAttributePayload); + }); + + it('should throw error when jsonAttributes is present in integrations object and jsonAttribute(widgets_installed#123) is array', () => { + const message = { + ...commonEventProps, + context: { + ...commonContextProps, + traits: { + af_install_time: '2024-12-09 12:26:29.643', + af_status: 'Organic', + firstName: 'Orcun', + lastName: 'Test', + widgets_installed: ['no_widgets_installed', 'widgets_installed'], + }, + }, + event: 'identify', + integrations: { + All: true, + AIRSHIP: { + JSONAttributes: { + 'widgets_installed#123': ['no_widgets_installed', 'widgets_installed'], + }, + }, + }, + type: 'identify', + } as unknown as RudderMessage; + + const traits = getFieldValueFromMessage(message, 'traits'); + const flattenedTraits = flattenJson(traits); + // @ts-expect-error error with type + expect(() => prepareAttributePayload(flattenedTraits, message)).toThrow( + 'JsonAttribute as array is not supported for widgets_installed#123 in Airship', + ); + }); + + it('should return the correct attribute payload when jsonAttributes is present in integrations object and jsonAttribute(widgets_installed#123) is object', () => { + const message = { + ...commonEventProps, + context: { + ...commonContextProps, + traits: { + af_install_time: '2024-12-09 12:26:29.643', + af_status: 'Organic', + firstName: 'Orcun', + lastName: 'Test', + }, + }, + event: 'identify', + integrations: { + All: true, + AIRSHIP: { + JSONAttributes: { + 'widgets_installed#123': { + widgets: ['no_widgets_installed', 'widgets_installed'], + }, + }, + }, + }, + type: 'identify', + } as unknown as RudderMessage; + + const expectedAttributePayload = { + attributes: [ + { + action: 'set', + key: 'af_install_time', + value: '2024-12-09T12:26:29Z', + timestamp: '2025-01-07T10:57:38Z', + }, + { action: 'set', key: 'af_status', value: 'Organic', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'first_name', value: 'Orcun', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'last_name', value: 'Test', timestamp: '2025-01-07T10:57:38Z' }, + { + action: 'set', + key: 'widgets_installed#123', + value: { + widgets: ['no_widgets_installed', 'widgets_installed'], + }, + timestamp: '2025-01-07T10:57:38Z', + }, + ], + }; + + const traits = getFieldValueFromMessage(message, 'traits'); + const flattenedTraits = flattenJson(traits); + // @ts-expect-error error with type + const attributePayload = prepareAttributePayload(flattenedTraits, message); + expect(attributePayload).toEqual(expectedAttributePayload); + }); + + it('should return the correct attribute payload when jsonAttributes is present in integrations object and jsonAttribute(data) is object & traits include an object', () => { + const message = { + ...commonEventProps, + context: { + ...commonContextProps, + traits: { + af_install_time: '2024-12-09 12:26:29.643', + af_status: 'Organic', + firstName: 'Orcun', + lastName: 'Test', + data: { + recordId: '123', + recordType: 'user', + }, + }, + }, + event: 'identify', + integrations: { + All: true, + AIRSHIP: { + JSONAttributes: { + 'widgets_installed#123': { + widgets: ['no_widgets_installed', 'widgets_installed'], + }, + }, + }, + }, + type: 'identify', + } as unknown as RudderMessage; + + const expectedAttributePayload = { + attributes: [ + { + action: 'set', + key: 'af_install_time', + value: '2024-12-09T12:26:29Z', + timestamp: '2025-01-07T10:57:38Z', + }, + { action: 'set', key: 'af_status', value: 'Organic', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'first_name', value: 'Orcun', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'last_name', value: 'Test', timestamp: '2025-01-07T10:57:38Z' }, + { + action: 'set', + key: 'data_recordId', + value: '123', + timestamp: '2025-01-07T10:57:38Z', + }, + { + action: 'set', + key: 'data_recordType', + value: 'user', + timestamp: '2025-01-07T10:57:38Z', + }, + { + action: 'set', + key: 'widgets_installed#123', + value: { + widgets: ['no_widgets_installed', 'widgets_installed'], + }, + timestamp: '2025-01-07T10:57:38Z', + }, + ], + }; + + const traits = getFieldValueFromMessage(message, 'traits'); + const flattenedTraits = flattenJson(traits); + // @ts-expect-error error with type + const attributePayload = prepareAttributePayload(flattenedTraits, message); + expect(attributePayload).toEqual(expectedAttributePayload); + }); + + it('should return the correct attribute payload when jsonAttributes is present in integrations object and jsonAttribute(widgets_installed#123) is object & traits include an object', () => { + const message = { + ...commonEventProps, + context: { + ...commonContextProps, + traits: { + af_install_time: '2024-12-09 12:26:29.643', + af_status: 'Organic', + firstName: 'Orcun', + lastName: 'Test', + data: { + recordId: '123', + recordType: 'user', + }, + widgets_installed: ['no_widgets_installed', 'widgets_installed'], + }, + }, + event: 'identify', + integrations: { + All: true, + AIRSHIP: { + JSONAttributes: { + 'widgets_installed#123': { + widgets: ['no_widgets_installed', 'widgets_installed'], + }, + }, + }, + }, + type: 'identify', + } as unknown as RudderMessage; + + const expectedAttributePayload = { + attributes: [ + { + action: 'set', + key: 'af_install_time', + value: '2024-12-09T12:26:29Z', + timestamp: '2025-01-07T10:57:38Z', + }, + { action: 'set', key: 'af_status', value: 'Organic', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'first_name', value: 'Orcun', timestamp: '2025-01-07T10:57:38Z' }, + { action: 'set', key: 'last_name', value: 'Test', timestamp: '2025-01-07T10:57:38Z' }, + { + action: 'set', + key: 'data_recordId', + value: '123', + timestamp: '2025-01-07T10:57:38Z', + }, + { + action: 'set', + key: 'data_recordType', + value: 'user', + timestamp: '2025-01-07T10:57:38Z', + }, + { + action: 'set', + key: 'widgets_installed#123', + value: { + widgets: ['no_widgets_installed', 'widgets_installed'], + }, + timestamp: '2025-01-07T10:57:38Z', + }, + ], + }; + + const traits = getFieldValueFromMessage(message, 'traits'); + const flattenedTraits = flattenJson(traits); + // @ts-expect-error error with type + const attributePayload = prepareAttributePayload(flattenedTraits, message); + expect(attributePayload).toEqual(expectedAttributePayload); + }); +}); + +describe('Airship utils - isValidTimestamp', () => { + it('should return true when timestamp is a valid Unix timestamp', () => { + const timestamp = 1736246350; + expect(isValidTimestamp(timestamp)).toBe(true); + }); + + it('should return true when timestamp is a valid Unix timestamp with milliseconds', () => { + const timestamp = 1736246350 * 1000; + expect(isValidTimestamp(timestamp)).toBe(true); + }); + + it('should return true when timestamp is a valid date string', () => { + const timestamp = '2025-01-23T12:00:00Z'; + expect(isValidTimestamp(timestamp)).toBe(true); + }); + + it('should return false when timestamp is not a valid Unix timestamp or date string(invalid_timestamp)', () => { + const timestamp = 'invalid_timestamp'; + expect(isValidTimestamp(timestamp)).toBe(false); + }); + + it('should return false when timestamp is not a valid Unix timestamp or date string(uuid)', () => { + const timestamp = convertToUuid('invalid_timestamp'); + expect(isValidTimestamp(timestamp)).toBe(false); + }); + + it('should return false when timestamp is not a valid Unix timestamp or date string(91504)', () => { + const timestamp = 91504; + expect(isValidTimestamp(timestamp)).toBe(false); + }); + + it('should return false when timestamp is not a valid Unix timestamp or date string("91504")', () => { + const timestamp = '91504'; + expect(isValidTimestamp(timestamp)).toBe(false); + }); +}); diff --git a/src/v0/destinations/airship/utils.ts b/src/v0/destinations/airship/utils.ts new file mode 100644 index 00000000000..d8c48fa2d04 --- /dev/null +++ b/src/v0/destinations/airship/utils.ts @@ -0,0 +1,209 @@ +import moment from 'moment'; +import { InstrumentationError } from '@rudderstack/integrations-lib'; +import { RudderMessage } from '../../../types'; +import { getFieldValueFromMessage, getIntegrationsObj } from '../../util'; +import { RESERVED_TRAITS_MAPPING } from './config'; + +export type TagPayloadEventType = 'identify' | 'group'; + +type AirshipTagProperties = { + identify: 'rudderstack_integration'; + group: 'rudderstack_integration_group'; +}; + +const AIRSHIP_TAG_PROPERTIES: Record = { + identify: 'rudderstack_integration', + group: 'rudderstack_integration_group', +}; + +type AirshipTag = { + [K in keyof Pick]: string[]; +}; + +type TagPayload = { + add: AirshipTag[T]; + remove: AirshipTag[T]; + [key: string]: unknown; +}; + +type AttributeValue = string | number | object; + +type Attribute = { + action: 'set' | 'remove'; + key: string; + value?: AttributeValue; + timestamp: string; +}; + +type AttributePayload = { + attributes: Attribute[]; +}; + +type AirshipIntegrationsObj = { + JSONAttributes: Record; + removeAttributes: string[]; +}; + +type AirshipObjectAttributes = Partial<{ + jsonAttributes: Attribute[]; + removeAttributes: Omit[]; +}>; + +const getDigitCount = (num: number): number => Math.floor(Math.log10(Math.abs(num))) + 1; + +export const isValidTimestamp = (timestamp: string | number): boolean => { + // Check if timestamp is a valid Unix timestamp (10 digits) + if (typeof timestamp === 'number' || !Number.isNaN(Number(timestamp))) { + return getDigitCount(Number(timestamp)) >= 10; + } + + // Check if timestamp is a valid date string + const date = moment.utc(timestamp); + return date.isValid() && date.year() >= 1970; +}; + +// Airship timestamp format: https://docs.airship.com/api/ua/#api-request-format +const AIRSHIP_TIMESTAMP_FORMAT = 'YYYY-MM-DD[T]HH:mm:ss[Z]'; + +const convertToAirshipTimestamp = (timestamp: string) => { + if (!timestamp || !moment(timestamp).isValid()) { + throw new InstrumentationError(`timestamp is not supported: ${timestamp}`); + } + return moment.utc(timestamp).format(AIRSHIP_TIMESTAMP_FORMAT); +}; + +export const getAirshipTimestamp = (message: RudderMessage) => { + const timestamp = getFieldValueFromMessage(message, 'timestamp'); + return convertToAirshipTimestamp(timestamp); +}; + +export const prepareTagPayload = ( + flattenedTraits: Record, + initTagPayload: TagPayload, + eventType: TagPayloadEventType = 'identify', +): TagPayload => { + const property = AIRSHIP_TAG_PROPERTIES[eventType]; + const initialTagPayload: TagPayload = { + ...initTagPayload, + add: { + [property]: [], + } as unknown as AirshipTag[TagPayloadEventType], + remove: { + [property]: [], + } as unknown as AirshipTag[TagPayloadEventType], + }; + + const tagPayload = Object.entries(flattenedTraits).reduce((acc, [key, value]) => { + // tags + if (typeof value === 'boolean') { + const tag = key.toLowerCase().replace(/\./g, '_'); + if (value === true) { + acc.add[property].push(tag); + } + if (value === false) { + acc.remove[property].push(tag); + } + } + return acc; + }, initialTagPayload); + return tagPayload; +}; + +const getJsonAttributesFromIntegrationsObj = (message: RudderMessage): AirshipObjectAttributes => { + const integrationsObj = getIntegrationsObj( + message, + 'airship' as any, + ) as Partial; + const timestamp = getAirshipTimestamp(message); + const airshipObjectAttributes: AirshipObjectAttributes = {}; + if (integrationsObj?.JSONAttributes) { + airshipObjectAttributes.jsonAttributes = Object.entries(integrationsObj.JSONAttributes).map( + ([key, value]) => { + // object attribute type in Airship: https://docs.airship.com/api/ua/#schemas-setattributeobject + if (Array.isArray(value)) { + throw new InstrumentationError( + `JsonAttribute as array is not supported for ${key} in Airship`, + ); + } + return { + action: 'set', + key, + value: value as AttributeValue, + timestamp, + }; + }, + ); + } + if (integrationsObj?.removeAttributes) { + // Remove Attributes in Airship: https://docs.airship.com/api/ua/#schemas-removeattributeobject + airshipObjectAttributes.removeAttributes = integrationsObj.removeAttributes.map((key) => ({ + action: 'remove', + key, + timestamp, + })); + } + return airshipObjectAttributes; +}; + +export const getAttributeValue = (value: string | number | object): AttributeValue => { + if (isValidTimestamp(value as string)) { + return convertToAirshipTimestamp(value as string); + } + return value as AttributeValue; +}; + +export const prepareAttributePayload = ( + flattenedTraits: Record, + message: RudderMessage, +): AttributePayload => { + const timestamp = getAirshipTimestamp(message); + const initialAttributePayload: AttributePayload = { attributes: [] }; + const airshipObjectAttributes: AirshipObjectAttributes = + getJsonAttributesFromIntegrationsObj(message); + + const isJsonAttributesPresent = + Array.isArray(airshipObjectAttributes?.jsonAttributes) && + airshipObjectAttributes?.jsonAttributes.length > 0; + + const attributePayload = Object.entries(flattenedTraits).reduce((acc, [key, value]) => { + // attribute + if (typeof value !== 'boolean') { + const attribute: Attribute = { action: 'set', key: '', value: '', timestamp }; + const keyMapped = RESERVED_TRAITS_MAPPING[key] || RESERVED_TRAITS_MAPPING[key.toLowerCase()]; + const isKeyObjectType = key.includes('.') || (key.includes('[') && key.includes(']')); + if (keyMapped) { + attribute.key = keyMapped; + } else { + attribute.key = key.replace(/\./g, '_'); + } + if (isJsonAttributesPresent && isKeyObjectType) { + // Skip these keys + // they can be in the form of an array or object + const keyParts = key?.split(/[[\]_]+/g) || []; + if (keyParts.length === 0) { + // If key doesn't include any of the delimiters like '[' or ']' or '_' , skip it + return acc; + } + // Skip keys that exist in both traits and integrations object to avoid duplication + const isKeyPresentInJsonAttributes = airshipObjectAttributes.jsonAttributes?.some((attr) => + attr.key.includes(keyParts[0]), + ); + if (isKeyPresentInJsonAttributes) { + // Skip this key + return acc; + } + } + attribute.value = getAttributeValue(value as AttributeValue); + acc.attributes.push(attribute); + } + return acc; + }, initialAttributePayload); + + attributePayload.attributes = [ + ...attributePayload.attributes, + ...(airshipObjectAttributes?.jsonAttributes || []), + ...(airshipObjectAttributes?.removeAttributes || []), + ]; + + return attributePayload; +}; diff --git a/src/v0/destinations/braze/util.js b/src/v0/destinations/braze/util.js index 74cb7fb9533..87119b91171 100644 --- a/src/v0/destinations/braze/util.js +++ b/src/v0/destinations/braze/util.js @@ -771,7 +771,7 @@ const collectStatsForAliasFailure = (brazeResponse, destinationId) => { if (!isDefinedAndNotNull(brazeResponse)) { return; } - const { aliases_processed: aliasesProcessed, errors } = brazeResponse; + const { aliases_processed: aliasesProcessed } = brazeResponse; if (aliasesProcessed === 0) { stats.increment('braze_alias_failure_count', { destination_id: destinationId }); } diff --git a/src/v0/destinations/clevertap/transform.js b/src/v0/destinations/clevertap/transform.js index e558b119f15..ee190246de5 100644 --- a/src/v0/destinations/clevertap/transform.js +++ b/src/v0/destinations/clevertap/transform.js @@ -404,7 +404,6 @@ const process = (event) => processEvent(event.message, event.destination); const processRouterDest = (inputs, reqMetadata) => { const eventsChunk = []; const errorRespList = []; - // const { destination } = inputs[0]; inputs.forEach((event) => { try { diff --git a/src/v0/destinations/clickup/util.js b/src/v0/destinations/clickup/util.js index 9930954ecbf..55923b29538 100644 --- a/src/v0/destinations/clickup/util.js +++ b/src/v0/destinations/clickup/util.js @@ -1,4 +1,5 @@ const { NetworkError, InstrumentationError } = require('@rudderstack/integrations-lib'); +const validator = require('validator'); const { httpGET } = require('../../../adapters/network'); const { processAxiosResponse, @@ -9,6 +10,7 @@ const { getHashFromArrayWithValueAsObject, formatTimeStamp, } = require('../../util'); + const { getCustomFieldsEndPoint } = require('./config'); const tags = require('../../util/tags'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -44,9 +46,7 @@ const validatePhoneWithCountryCode = (phone) => { * @param {*} email */ const validateEmail = (email) => { - const regex = - /^(([^\s"(),.:;<>@[\\\]]+(\.[^\s"(),.:;<>@[\\\]]+)*)|(".+"))@((\[(?:\d{1,3}\.){3}\d{1,3}])|(([\dA-Za-z-]+\.)+[A-Za-z]{2,}))$/; - if (!regex.test(email)) { + if (!validator.isEmail(email)) { throw new InstrumentationError('The provided email is invalid'); } }; diff --git a/src/v0/destinations/customerio/util.js b/src/v0/destinations/customerio/util.js index cadad5620ce..d5696123b04 100644 --- a/src/v0/destinations/customerio/util.js +++ b/src/v0/destinations/customerio/util.js @@ -1,6 +1,7 @@ const get = require('get-value'); const set = require('set-value'); const truncate = require('truncate-utf8-bytes'); +const validator = require('validator'); const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { MAX_BATCH_SIZE, configFieldsToCheck } = require('./config'); const { @@ -10,7 +11,6 @@ const { getFieldValueFromMessage, defaultDeleteRequestConfig, isAppleFamily, - validateEmail, } = require('../../util'); const { EventType, SpecedTraits, TraitsMapping } = require('../../../constants'); @@ -166,26 +166,19 @@ const identifyResponseBuilder = (userId, message) => { const aliasResponseBuilder = (message, userId) => { // ref : https://customer.io/docs/api/#operation/merge - if (!userId && !message.previousId) { - throw new InstrumentationError('Both userId and previousId is mandatory for merge operation'); + if (!userId || !message.previousId) { + throw new InstrumentationError('Both userId and previousId are mandatory for merge operation'); } const endpoint = MERGE_USER_ENDPOINT; const requestConfig = defaultPostRequestConfig; - let cioProperty = 'id'; - if (validateEmail(userId)) { - cioProperty = 'email'; - } - // eslint-disable-next-line @typescript-eslint/naming-convention - let prev_cioProperty = 'id'; - if (validateEmail(message.previousId)) { - prev_cioProperty = 'email'; - } + const cioProperty = validator.isEmail(userId) ? 'email' : 'id'; + const prevCioProperty = validator.isEmail(message.previousId) ? 'email' : 'id'; const rawPayload = { primary: { [cioProperty]: userId, }, secondary: { - [prev_cioProperty]: message.previousId, + [prevCioProperty]: message.previousId, }, }; @@ -208,11 +201,8 @@ const groupResponseBuilder = (message) => { cio_relationships: [], }; const id = payload?.userId || payload?.email; - let cioProperty = 'id'; - if (validateEmail(id)) { - cioProperty = 'email'; - } if (id) { + const cioProperty = validator.isEmail(id) ? 'email' : 'id'; rawPayload.cio_relationships.push({ identifiers: { [cioProperty]: id } }); } const requestConfig = defaultPostRequestConfig; diff --git a/src/v0/destinations/customerio/util.test.js b/src/v0/destinations/customerio/util.test.js index 2c308cd262d..ee36bd3f1be 100644 --- a/src/v0/destinations/customerio/util.test.js +++ b/src/v0/destinations/customerio/util.test.js @@ -8,6 +8,7 @@ const { const getTestMessage = () => { let message = { anonymousId: 'anonId', + previousId: 'user0', traits: { email: 'abc@test.com', name: 'rudder', @@ -133,7 +134,7 @@ describe('Unit test cases for customerio aliasResponseBuilder', () => { it('Device token name does not match with event name as well as allowed list', async () => { let expectedOutput = { endpoint: 'https://track.customer.io/api/v1/merge_customers', - rawPayload: { primary: { id: 'user1' }, secondary: { id: undefined } }, + rawPayload: { primary: { id: 'user1' }, secondary: { id: 'user0' } }, requestConfig: { requestFormat: 'JSON', requestMethod: 'POST' }, }; expect(aliasResponseBuilder(getTestMessage(), 'user1')).toEqual(expectedOutput); diff --git a/src/v0/destinations/customerio_audience/config.ts b/src/v0/destinations/customerio_audience/config.ts new file mode 100644 index 00000000000..d9b223c59b8 --- /dev/null +++ b/src/v0/destinations/customerio_audience/config.ts @@ -0,0 +1,11 @@ +export const MAX_ITEMS = 1000; + +export const DEFAULT_ID_TYPE = 'id'; + +export const BASE_ENDPOINT = 'https://track.customer.io/api/v1/segments'; + +export const SegmentAction = { + INSERT: 'insert', + UPDATE: 'update', + DELETE: 'delete', +}; diff --git a/src/v0/destinations/customerio_audience/transform.ts b/src/v0/destinations/customerio_audience/transform.ts new file mode 100644 index 00000000000..c8d9f8a89c0 --- /dev/null +++ b/src/v0/destinations/customerio_audience/transform.ts @@ -0,0 +1,113 @@ +import { ConfigurationError } from '@rudderstack/integrations-lib'; +import { SegmentAction } from './config'; +import { CustomerIORouterRequestType, RespList } from './type'; + +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { batchResponseBuilder, getEventAction } = require('./utils'); +const { handleRtTfSingleEventError, getEventType } = require('../../util'); +const { EventType } = require('../../../constants'); + +interface ProcessedEvent extends RespList { + eventAction: keyof typeof SegmentAction; +} + +const createEventChunk = (event: CustomerIORouterRequestType): ProcessedEvent => { + const eventAction = getEventAction(event); + const { identifiers } = event?.message || {}; + const id: string | number = Object.values(identifiers)[0]; + + return { + payload: { ids: [id] }, + metadata: event.metadata, + eventAction, + }; +}; + +const validateEvent = (event: CustomerIORouterRequestType): boolean => { + const eventType = getEventType(event?.message); + if (eventType !== EventType.RECORD) { + throw new InstrumentationError(`message type ${eventType} is not supported`); + } + + const eventAction = getEventAction(event); + if (!Object.values(SegmentAction).includes(eventAction)) { + throw new InstrumentationError(`action ${eventAction} is not supported`); + } + + const identifiers = event?.message?.identifiers; + if (!identifiers || Object.keys(identifiers).length === 0) { + throw new InstrumentationError(`identifiers cannot be empty`); + } + + if (Object.keys(identifiers).length > 1) { + throw new InstrumentationError(`only one identifier is supported`); + } + + const id = Object.values(identifiers)[0]; + if (typeof id !== 'string' && typeof id !== 'number') { + throw new ConfigurationError(`identifier type should be a string or integer`); + } + + const audienceId = event?.connection?.config?.destination?.audienceId; + if (!audienceId) { + throw new InstrumentationError('audienceId is required, aborting.'); + } + + const identifierMappings = event?.connection?.config?.destination?.identifierMappings; + if (!identifierMappings || Object.keys(identifierMappings).length === 0) { + throw new InstrumentationError('identifierMappings cannot be empty'); + } + + return true; +}; + +const processRouterDest = async (inputs: CustomerIORouterRequestType[], reqMetadata: any) => { + if (!inputs?.length) return []; + + const { destination, connection } = inputs[0]; + + // Process events and separate valid and error cases + const processedEvents = inputs.map((event) => { + try { + validateEvent(event); + return { + success: true, + data: createEventChunk(event), + }; + } catch (error) { + return { + success: false, + error: handleRtTfSingleEventError(event, error, reqMetadata), + }; + } + }); + + // Separate successful and failed events + const successfulEvents = processedEvents + .filter((result) => result.success) + .map((result) => result.data as ProcessedEvent); + + const errorEvents = processedEvents + .filter((result) => !result.success) + .map((result) => result.error); + + // Split successful events into delete and insert/update lists + const deleteRespList = successfulEvents + .filter((event) => event.eventAction === SegmentAction.DELETE) + .map(({ payload, metadata }) => ({ payload, metadata })); + + const insertOrUpdateRespList = successfulEvents + .filter((event) => event.eventAction !== SegmentAction.DELETE) + .map(({ payload, metadata }) => ({ payload, metadata })); + + const batchSuccessfulRespList = batchResponseBuilder( + insertOrUpdateRespList, + deleteRespList, + destination, + connection, + ); + + return [...batchSuccessfulRespList, ...errorEvents]; +}; + +export { processRouterDest }; diff --git a/src/v0/destinations/customerio_audience/type.ts b/src/v0/destinations/customerio_audience/type.ts new file mode 100644 index 00000000000..5c5335d6c8e --- /dev/null +++ b/src/v0/destinations/customerio_audience/type.ts @@ -0,0 +1,60 @@ +import { Connection, Destination, Metadata, RouterTransformationRequestData } from '../../../types'; + +// Basic response type for audience list operations +export type RespList = { + payload: { + ids: (string | number)[]; + }; + metadata: Metadata; +}; + +// Types for API request components +export type SegmentationPayloadType = { + ids: (string | number)[]; +}; + +export type SegmentationParamType = { + id_type: string; +}; + +export type SegmentationHeadersType = { + 'Content-Type': string; + Authorization: string; +}; + +// CustomerIO specific configuration types +type CustomerIODestinationConfig = { + apiKey: string; + appApiKey: string; + siteId: string; + [key: string]: any; +}; + +type CustomerIOConnectionConfig = { + destination: { + audienceId: string | number; + identifierMappings: { + from: string; + to: string; + }[]; + }; +}; + +// Message type specific to CustomerIO +export type CustomerIOMessageType = { + action: string; + identifiers: Record; + [key: string]: any; +}; + +// Final exported types using generics from base types +export type CustomerIODestinationType = Destination; +export type CustomerIOConnectionType = Connection & { + config: CustomerIOConnectionConfig; +}; + +export type CustomerIORouterRequestType = RouterTransformationRequestData< + CustomerIOMessageType, + CustomerIODestinationType, + CustomerIOConnectionType +>; diff --git a/src/v0/destinations/customerio_audience/utils.ts b/src/v0/destinations/customerio_audience/utils.ts new file mode 100644 index 00000000000..c642618963b --- /dev/null +++ b/src/v0/destinations/customerio_audience/utils.ts @@ -0,0 +1,121 @@ +import { base64Convertor } from '@rudderstack/integrations-lib'; +import { BatchUtils } from '@rudderstack/workflow-engine'; +import { BASE_ENDPOINT, DEFAULT_ID_TYPE, MAX_ITEMS } from './config'; +import { + CustomerIOConnectionType, + CustomerIODestinationType, + CustomerIORouterRequestType, + RespList, + SegmentationHeadersType, + SegmentationParamType, + SegmentationPayloadType, +} from './type'; +import { Metadata } from '../../../types'; + +const getIdType = (connection: CustomerIOConnectionType): string => + connection.config.destination.identifierMappings[0]?.to || DEFAULT_ID_TYPE; + +const getSegmentId = (connection: CustomerIOConnectionType): string | number => + connection.config.destination.audienceId; + +const getHeaders = (destination: CustomerIODestinationType): SegmentationHeadersType => ({ + 'Content-Type': 'application/json', + Authorization: `Basic ${base64Convertor(`${destination.Config.siteId}:${destination.Config.apiKey}`)}`, +}); + +const getParams = (connection: CustomerIOConnectionType): SegmentationParamType => ({ + id_type: getIdType(connection), +}); + +const getMergedPayload = (batch: RespList[]): SegmentationPayloadType => ({ + ids: batch.flatMap((input) => input.payload.ids), +}); + +const getMergedMetadata = (batch: RespList[]): Metadata[] => batch.map((input) => input.metadata); + +const buildBatchedResponse = ( + payload: SegmentationPayloadType, + endpoint: string, + headers: SegmentationHeadersType, + params: SegmentationParamType, + metadata: Metadata[], + destination: CustomerIODestinationType, +) => ({ + batchedRequest: { + body: { + JSON: payload, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version: '1', + type: 'REST', + method: 'POST', + endpoint, + headers, + params, + files: {}, + }, + metadata, + batched: true, + statusCode: 200, + destination, +}); + +const processBatch = ( + respList: RespList[], + endpoint: string, + destination: CustomerIODestinationType, + connection: CustomerIOConnectionType, +): any[] => { + if (!respList?.length) { + return []; + } + + const headers = getHeaders(destination); + const params = getParams(connection); + const batches = BatchUtils.chunkArrayBySizeAndLength(respList, { maxItems: MAX_ITEMS }); + + return batches.items.map((batch) => { + const mergedPayload = getMergedPayload(batch); + const mergedMetadata = getMergedMetadata(batch); + return buildBatchedResponse( + mergedPayload, + endpoint, + headers, + params, + mergedMetadata, + destination, + ); + }); +}; + +const batchResponseBuilder = ( + insertOrUpdateRespList: RespList[], + deleteRespList: RespList[], + destination: CustomerIODestinationType, + connection: CustomerIOConnectionType, +): any[] => { + const segmentId = getSegmentId(connection); + + const insertResponses = processBatch( + insertOrUpdateRespList, + `${BASE_ENDPOINT}/${segmentId}/add_customers`, + destination, + connection, + ); + + const deleteResponses = processBatch( + deleteRespList, + `${BASE_ENDPOINT}/${segmentId}/remove_customers`, + destination, + connection, + ); + + return [...insertResponses, ...deleteResponses]; +}; + +const getEventAction = (event: CustomerIORouterRequestType): string => + event?.message?.action?.toLowerCase() || ''; + +export { batchResponseBuilder, getEventAction }; diff --git a/src/v0/destinations/fb_custom_audience/recordTransform.js b/src/v0/destinations/fb_custom_audience/recordTransform.js index db1fbeec595..719a53619e3 100644 --- a/src/v0/destinations/fb_custom_audience/recordTransform.js +++ b/src/v0/destinations/fb_custom_audience/recordTransform.js @@ -23,54 +23,68 @@ const { generateAppSecretProof, } = require('./util'); -const processRecordEventArray = ( - recordChunksArray, - userSchema, - isHashRequired, - disableFormat, - paramsPayload, - prepareParams, - destination, - operation, - audienceId, -) => { +/** + * Processes a single record and updates the data element. + * @param {Object} record - The record to process. + * @param {Array} userSchema - The schema defining user properties. + * @param {boolean} isHashRequired - Whether hashing is required. + * @param {boolean} disableFormat - Whether formatting is disabled. + * @returns {Object} - The processed data element and metadata. + */ +const processRecord = (record, userSchema, isHashRequired, disableFormat) => { + const { fields } = record.message; + let dataElement = []; + let nullUserData = true; + + userSchema.forEach((eachProperty) => { + const userProperty = fields[eachProperty]; + let updatedProperty = userProperty; + + if (isHashRequired && !disableFormat) { + updatedProperty = ensureApplicableFormat(eachProperty, userProperty); + } + + dataElement = getUpdatedDataElement(dataElement, isHashRequired, eachProperty, updatedProperty); + + if (dataElement[dataElement.length - 1]) { + nullUserData = false; + } + }); + + if (nullUserData) { + stats.increment('fb_custom_audience_event_having_all_null_field_values_for_a_user', { + destinationId: record.destination.ID, + nullFields: userSchema, + }); + } + + return { dataElement, metadata: record.metadata }; +}; + +/** + * Processes an array of record chunks and prepares the payload for sending. + * @param {Array} recordChunksArray - The array of record chunks. + * @param {Object} config - Configuration object containing userSchema, isHashRequired, disableFormat, etc. + * @param {Object} destination - The destination configuration. + * @param {string} operation - The operation to perform (e.g., 'add', 'remove'). + * @param {string} audienceId - The audience ID. + * @returns {Array} - The response events to send. + */ +const processRecordEventArray = (recordChunksArray, config, destination, operation, audienceId) => { + const { userSchema, isHashRequired, disableFormat, paramsPayload, prepareParams } = config; const toSendEvents = []; const metadata = []; + recordChunksArray.forEach((recordArray) => { - const data = []; - recordArray.forEach((input) => { - const { fields } = input.message; - let dataElement = []; - let nullUserData = true; - - userSchema.forEach((eachProperty) => { - const userProperty = fields[eachProperty]; - let updatedProperty = userProperty; - - if (isHashRequired && !disableFormat) { - updatedProperty = ensureApplicableFormat(eachProperty, userProperty); - } - - dataElement = getUpdatedDataElement( - dataElement, - isHashRequired, - eachProperty, - updatedProperty, - ); - - if (dataElement[dataElement.length - 1]) { - nullUserData = false; - } - }); - - if (nullUserData) { - stats.increment('fb_custom_audience_event_having_all_null_field_values_for_a_user', { - destinationId: destination.ID, - nullFields: userSchema, - }); - } - data.push(dataElement); - metadata.push(input.metadata); + const data = recordArray.map((input) => { + const { dataElement, metadata: recordMetadata } = processRecord( + input, + userSchema, + isHashRequired, + disableFormat, + ); + metadata.push(recordMetadata); + return dataElement; }); const prepareFinalPayload = lodash.cloneDeep(paramsPayload); @@ -90,16 +104,19 @@ const processRecordEventArray = ( }; const builtResponse = responseBuilderSimple(wrappedResponse, audienceId); - toSendEvents.push(builtResponse); }); }); - const response = getSuccessRespEvents(toSendEvents, metadata, destination, true); - - return response; + return getSuccessRespEvents(toSendEvents, metadata, destination, true); }; +/** + * Prepares the payload for the given events and configuration. + * @param {Array} events - The events to process. + * @param {Object} config - The configuration object. + * @returns {Array} - The final response payload. + */ function preparePayload(events, config) { const { audienceId, userSchema, isRaw, type, subType, isHashRequired, disableFormat } = config; const { destination } = events[0]; @@ -138,64 +155,32 @@ function preparePayload(events, config) { record.message.action?.toLowerCase(), ); - let insertResponse; - let deleteResponse; - let updateResponse; - - if (groupedRecordsByAction.delete) { - const deleteRecordChunksArray = returnArrayOfSubarrays( - groupedRecordsByAction.delete, - MAX_USER_COUNT, - ); - deleteResponse = processRecordEventArray( - deleteRecordChunksArray, - cleanUserSchema, - isHashRequired, - disableFormat, - paramsPayload, - prepareParams, - destination, - 'remove', - audienceId, - ); - } - - if (groupedRecordsByAction.insert) { - const insertRecordChunksArray = returnArrayOfSubarrays( - groupedRecordsByAction.insert, - MAX_USER_COUNT, - ); - - insertResponse = processRecordEventArray( - insertRecordChunksArray, - cleanUserSchema, - isHashRequired, - disableFormat, - paramsPayload, - prepareParams, - destination, - 'add', - audienceId, - ); - } + const processAction = (action, operation) => { + if (groupedRecordsByAction[action]) { + const recordChunksArray = returnArrayOfSubarrays( + groupedRecordsByAction[action], + MAX_USER_COUNT, + ); + return processRecordEventArray( + recordChunksArray, + { + userSchema: cleanUserSchema, + isHashRequired, + disableFormat, + paramsPayload, + prepareParams, + }, + destination, + operation, + audienceId, + ); + } + return null; + }; - if (groupedRecordsByAction.update) { - const updateRecordChunksArray = returnArrayOfSubarrays( - groupedRecordsByAction.update, - MAX_USER_COUNT, - ); - updateResponse = processRecordEventArray( - updateRecordChunksArray, - cleanUserSchema, - isHashRequired, - disableFormat, - paramsPayload, - prepareParams, - destination, - 'add', - audienceId, - ); - } + const deleteResponse = processAction('delete', 'remove'); + const insertResponse = processAction('insert', 'add'); + const updateResponse = processAction('update', 'add'); const errorResponse = getErrorResponse(groupedRecordsByAction); @@ -203,7 +188,6 @@ function preparePayload(events, config) { deleteResponse, insertResponse, updateResponse, - errorResponse, ); if (finalResponse.length === 0) { @@ -214,6 +198,11 @@ function preparePayload(events, config) { return finalResponse; } +/** + * Processes record inputs for V1 flow. + * @param {Array} groupedRecordInputs - The grouped record inputs. + * @returns {Array} - The processed payload. + */ function processRecordInputsV1(groupedRecordInputs) { const { destination } = groupedRecordInputs[0]; const { message } = groupedRecordInputs[0]; @@ -239,11 +228,15 @@ function processRecordInputsV1(groupedRecordInputs) { }); } +/** + * Processes record inputs for V2 flow. + * @param {Array} groupedRecordInputs - The grouped record inputs. + * @returns {Array} - The processed payload. + */ const processRecordInputsV2 = (groupedRecordInputs) => { const { connection, message } = groupedRecordInputs[0]; const { isHashRequired, disableFormat, type, subType, isRaw, audienceId } = connection.config.destination; - // Ref: https://www.notion.so/rudderstacks/VDM-V2-Final-Config-and-Record-EventPayload-8cc80f3d88ad46c7bc43df4b87a0bbff const identifiers = message?.identifiers; let userSchema; if (identifiers) { @@ -267,6 +260,11 @@ const processRecordInputsV2 = (groupedRecordInputs) => { }); }; +/** + * Processes record inputs based on the flow type. + * @param {Array} groupedRecordInputs - The grouped record inputs. + * @returns {Array} - The processed payload. + */ function processRecordInputs(groupedRecordInputs) { const event = groupedRecordInputs[0]; // First check for rETL flow and second check for ES flow diff --git a/src/v0/destinations/ga4_v2/customMappingsHandler.js b/src/v0/destinations/ga4_v2/customMappingsHandler.js index b5818d6fff1..2bdfd375e25 100644 --- a/src/v0/destinations/ga4_v2/customMappingsHandler.js +++ b/src/v0/destinations/ga4_v2/customMappingsHandler.js @@ -23,7 +23,6 @@ const { isEmptyObject, removeUndefinedAndNullValues, isHybridModeEnabled, - getIntegrationsObj, applyCustomMappings, } = require('../../util'); const { trackCommonConfig, ConfigCategory, mappingConfig } = require('../ga4/config'); @@ -145,7 +144,6 @@ const handleCustomMappings = (message, Config) => { const boilerplateOperations = (ga4Payload, message, Config, eventName) => { removeReservedParameterPrefixNames(ga4Payload.events[0].params); ga4Payload.events[0].name = eventName; - const integrationsObj = getIntegrationsObj(message, 'ga4_v2'); if (ga4Payload.events[0].params) { ga4Payload.events[0].params = removeInvalidParams( diff --git a/src/v0/destinations/google_adwords_offline_conversions/config.js b/src/v0/destinations/google_adwords_offline_conversions/config.js index 6eec1068a6b..ce0fc45e47e 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/config.js +++ b/src/v0/destinations/google_adwords_offline_conversions/config.js @@ -1,6 +1,6 @@ const { getMappingConfig } = require('../../util'); -const API_VERSION = 'v16'; +const API_VERSION = 'v17'; const BASE_ENDPOINT = `https://googleads.googleapis.com/${API_VERSION}/customers/:customerId`; diff --git a/src/v0/destinations/google_adwords_offline_conversions/data/TrackAddStoreConversionsConfig.json b/src/v0/destinations/google_adwords_offline_conversions/data/TrackAddStoreConversionsConfig.json index 9c88e59ddb3..9bfd0bc179f 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/data/TrackAddStoreConversionsConfig.json +++ b/src/v0/destinations/google_adwords_offline_conversions/data/TrackAddStoreConversionsConfig.json @@ -17,7 +17,8 @@ ], "required": true, "metadata": { - "type": "toNumber" + "type": "toNumber", + "regex": "^([1-9]\\d*(\\.\\d+)?|0\\.\\d+)$" } }, { diff --git a/src/v0/destinations/google_adwords_offline_conversions/transform.js b/src/v0/destinations/google_adwords_offline_conversions/transform.js index 2648f03e8a3..76b12587cdc 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/transform.js +++ b/src/v0/destinations/google_adwords_offline_conversions/transform.js @@ -18,7 +18,6 @@ const { getClickConversionPayloadAndEndpoint, getConsentsDataFromIntegrationObj, getCallConversionPayload, - updateConversion, } = require('./utils'); const helper = require('./helper'); @@ -49,9 +48,6 @@ const getConversions = (message, metadata, { Config }, event, conversionType) => filteredCustomerId, eventLevelConsentsData, ); - convertedPayload.payload.conversions[0] = updateConversion( - convertedPayload.payload.conversions[0], - ); payload = convertedPayload.payload; endpoint = convertedPayload.endpoint; } else if (conversionType === 'store') { diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 2d47095eea7..eeb4d2c3a45 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -346,6 +346,24 @@ const populateUserIdentifier = ({ email, phone, properties, payload, UserIdentif } return copiedPayload; }; + +/** + * remove redundant ids + * @param {*} conversionCopy + */ +const updateConversion = (conversion) => { + const conversionCopy = cloneDeep(conversion); + if (conversionCopy.gclid) { + delete conversionCopy.wbraid; + delete conversionCopy.gbraid; + } else if (conversionCopy.wbraid && conversionCopy.gbraid) { + throw new InstrumentationError(`You can't use both wbraid and gbraid.`); + } else if (conversionCopy.wbraid || conversionCopy.gbraid) { + delete conversionCopy.userIdentifiers; + } + return conversionCopy; +}; + const getClickConversionPayloadAndEndpoint = ( message, Config, @@ -423,6 +441,7 @@ const getClickConversionPayloadAndEndpoint = ( const consentObject = finaliseConsent(consentConfigMap, eventLevelConsent, Config); // here conversions[0] is expected to be present there are some mandatory properties mapped in the mapping json. set(payload, 'conversions[0].consent', consentObject); + payload.conversions[0] = updateConversion(payload.conversions[0]); return { payload, endpoint }; }; @@ -431,25 +450,6 @@ const getConsentsDataFromIntegrationObj = (message) => { return integrationObj?.consents || {}; }; -/** - * remove redundant ids - * @param {*} conversionCopy - */ -const updateConversion = (conversion) => { - const conversionCopy = cloneDeep(conversion); - if (conversionCopy.gclid) { - delete conversionCopy.wbraid; - delete conversionCopy.gbraid; - } - if (conversionCopy.wbraid && conversionCopy.gbraid) { - throw new InstrumentationError(`You can't use both wbraid and gbraid.`); - } - if (conversionCopy.wbraid || conversionCopy.gbraid) { - delete conversionCopy.userIdentifiers; - } - return conversionCopy; -}; - module.exports = { validateDestinationConfig, generateItemListFromProducts, diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js index b6c66537829..59141fce3bc 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.test.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.test.js @@ -163,7 +163,7 @@ describe('getExisitingUserIdentifier util tests', () => { describe('getClickConversionPayloadAndEndpoint util tests', () => { it('getClickConversionPayloadAndEndpoint flow check when default field identifier is present', () => { let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', payload: { conversions: [ { @@ -193,7 +193,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { delete fittingPayload.traits.email; delete fittingPayload.properties.email; let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', payload: { conversions: [ { @@ -225,7 +225,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { delete fittingPayload.traits.phone; delete fittingPayload.properties.email; let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', payload: { conversions: [ { @@ -263,7 +263,7 @@ describe('getClickConversionPayloadAndEndpoint util tests', () => { }, ]; let expectedOutput = { - endpoint: 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + endpoint: 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', payload: { conversions: [ { diff --git a/src/v0/destinations/google_cloud_function/util.js b/src/v0/destinations/google_cloud_function/util.js index 8f85460902c..a960f02ab04 100644 --- a/src/v0/destinations/google_cloud_function/util.js +++ b/src/v0/destinations/google_cloud_function/util.js @@ -30,7 +30,6 @@ function generateBatchedPayload(events) { let batchEventResponse = events.map((event) => event.message); // Batch event into dest batch structure events.forEach((ev) => { - // batchResponseList.push(ev.message.body.JSON); metadata.push(ev.metadata); }); batchEventResponse = { diff --git a/src/v0/destinations/intercom/config.js b/src/v0/destinations/intercom/config.js deleted file mode 100644 index ae29eebc1eb..00000000000 --- a/src/v0/destinations/intercom/config.js +++ /dev/null @@ -1,53 +0,0 @@ -const { getMappingConfig } = require('../../util'); - -const BASE_ENDPOINT = 'https://api.intercom.io'; - -// track events | Track -const TRACK_ENDPOINT = `${BASE_ENDPOINT}/events`; -// Create, Update a user with a company | Identify -const IDENTIFY_ENDPOINT = `${BASE_ENDPOINT}/users`; -// create, update, delete a company | Group -const GROUP_ENDPOINT = `${BASE_ENDPOINT}/companies`; - -const ConfigCategory = { - TRACK: { - endpoint: TRACK_ENDPOINT, - name: 'INTERCOMTrackConfig', - }, - IDENTIFY: { - endpoint: IDENTIFY_ENDPOINT, - name: 'INTERCOMIdentifyConfig', - }, - GROUP: { - endpoint: GROUP_ENDPOINT, - name: 'INTERCOMGroupConfig', - }, -}; - -const MappingConfig = getMappingConfig(ConfigCategory, __dirname); - -const ReservedTraitsProperties = [ - 'userId', - 'email', - 'phone', - 'name', - 'createdAt', - 'firstName', - 'lastName', - 'firstname', - 'lastname', - 'company', -]; - -const ReservedCompanyProperties = ['id', 'name', 'industry']; - -// ref:- https://developers.intercom.com/intercom-api-reference/v1.4/reference/event-metadata-types -const MetadataTypes = { richLink: ['url', 'value'], monetaryAmount: ['amount', 'currency'] }; - -module.exports = { - ConfigCategory, - MappingConfig, - ReservedCompanyProperties, - ReservedTraitsProperties, - MetadataTypes, -}; diff --git a/src/v0/destinations/intercom/data/INTERCOMGroupConfig.json b/src/v0/destinations/intercom/data/INTERCOMGroupConfig.json deleted file mode 100644 index 6857c4e1045..00000000000 --- a/src/v0/destinations/intercom/data/INTERCOMGroupConfig.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "destKey": "company_id", - "sourceKeys": "groupId", - "required": true - }, - { - "destKey": "name", - "sourceKeys": "name", - "sourceFromGenericMap": true, - "required": false - }, - { - "destKey": "plan", - "sourceKeys": ["traits.plan", "context.traits.plan"], - "required": false - }, - { - "destKey": "size", - "sourceKeys": ["traits.size", "context.traits.size"], - "metadata": { - "type": "toNumber" - }, - "required": false - }, - { - "destKey": "website", - "sourceKeys": "website", - "sourceFromGenericMap": true, - "required": false - }, - { - "destKey": "industry", - "sourceKeys": ["traits.industry", "context.traits.industry"], - "required": false - }, - { - "destKey": "monthly_spend", - "sourceKeys": ["traits.monthlySpend", "context.traits.monthlySpend"], - "metadata": { - "type": "toNumber" - }, - "required": false - }, - { - "destKey": "remote_created_at", - "sourceKeys": ["traits.remoteCreatedAt", "context.traits.remoteCreatedAt"], - "metadata": { - "type": "toNumber" - }, - "required": false - } -] diff --git a/src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json b/src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json deleted file mode 100644 index 726a741161e..00000000000 --- a/src/v0/destinations/intercom/data/INTERCOMIdentifyConfig.json +++ /dev/null @@ -1,46 +0,0 @@ -[ - { - "destKey": "user_id", - "sourceKeys": [ - "userId", - "traits.userId", - "traits.id", - "context.traits.userId", - "context.traits.id" - ], - "required": false - }, - { - "destKey": "email", - "sourceKeys": ["traits.email", "context.traits.email"], - "required": false - }, - { - "destKey": "phone", - "sourceKeys": ["traits.phone", "context.traits.phone"], - "required": false - }, - { - "destKey": "name", - "sourceKeys": ["traits.name", "context.traits.name"], - "required": false - }, - { - "destKey": "signed_up_at", - "sourceKeys": ["traits.createdAt", "context.traits.createdAt"], - "required": false, - "metadata": { - "type": "secondTimestamp" - } - }, - { - "destKey": "last_seen_user_agent", - "sourceKeys": "context.userAgent", - "required": false - }, - { - "destKey": "custom_attributes", - "sourceKeys": ["traits", "context.traits"], - "required": false - } -] diff --git a/src/v0/destinations/intercom/data/INTERCOMTrackConfig.json b/src/v0/destinations/intercom/data/INTERCOMTrackConfig.json deleted file mode 100644 index f33c9a8a982..00000000000 --- a/src/v0/destinations/intercom/data/INTERCOMTrackConfig.json +++ /dev/null @@ -1,36 +0,0 @@ -[ - { - "destKey": "user_id", - "sourceKeys": [ - "userId", - "traits.userId", - "traits.id", - "context.traits.userId", - "context.traits.id" - ], - "required": false - }, - { - "destKey": "email", - "sourceKeys": ["traits.email", "context.traits.email"], - "required": false - }, - { - "destKey": "event_name", - "sourceKeys": "event", - "required": true - }, - { - "destKey": "created", - "sourceKeys": "timestamp", - "sourceFromGenericMap": true, - "required": true, - "metadata": { - "type": "secondTimestamp" - } - }, - { - "destKey": "metadata", - "sourceKeys": "properties" - } -] diff --git a/src/v0/destinations/intercom/transform.js b/src/v0/destinations/intercom/transform.js deleted file mode 100644 index 212eaba13b5..00000000000 --- a/src/v0/destinations/intercom/transform.js +++ /dev/null @@ -1,252 +0,0 @@ -const md5 = require('md5'); -const get = require('get-value'); -const { InstrumentationError } = require('@rudderstack/integrations-lib'); -const { EventType, MappedToDestinationKey } = require('../../../constants'); -const { - ConfigCategory, - MappingConfig, - ReservedTraitsProperties, - ReservedCompanyProperties, -} = require('./config'); -const { - constructPayload, - removeUndefinedAndNullValues, - defaultRequestConfig, - defaultPostRequestConfig, - getFieldValueFromMessage, - addExternalIdToTraits, - simpleProcessRouterDest, - flattenJson, -} = require('../../util'); -const { separateReservedAndRestMetadata } = require('./util'); -const { JSON_MIME_TYPE } = require('../../util/constant'); - -function getCompanyAttribute(company) { - const companiesList = []; - if (company.name || company.id) { - const customAttributes = {}; - Object.keys(company).forEach((key) => { - // the key is not in ReservedCompanyProperties - if (!ReservedCompanyProperties.includes(key)) { - const val = company[key]; - if (val !== Object(val)) { - customAttributes[key] = val; - } else { - customAttributes[key] = JSON.stringify(val); - } - } - }); - - companiesList.push({ - company_id: company.id || md5(company.name), - custom_attributes: removeUndefinedAndNullValues(customAttributes), - name: company.name, - industry: company.industry, - }); - } - return companiesList; -} - -function validateIdentify(message, payload, config) { - const finalPayload = payload; - - finalPayload.update_last_request_at = - config.updateLastRequestAt !== undefined ? config.updateLastRequestAt : true; - if (payload.user_id || payload.email) { - if (payload.name === undefined || payload.name === '') { - const firstName = getFieldValueFromMessage(message, 'firstName'); - const lastName = getFieldValueFromMessage(message, 'lastName'); - if (firstName && lastName) { - finalPayload.name = `${firstName} ${lastName}`; - } else { - finalPayload.name = firstName || lastName; - } - } - - if (get(finalPayload, 'custom_attributes.company')) { - finalPayload.companies = getCompanyAttribute(finalPayload.custom_attributes.company); - } - - if (finalPayload.custom_attributes) { - ReservedTraitsProperties.forEach((trait) => { - delete finalPayload.custom_attributes[trait]; - }); - finalPayload.custom_attributes = flattenJson(finalPayload.custom_attributes); - } - - return finalPayload; - } - throw new InstrumentationError('Either of `email` or `userId` is required for Identify call'); -} - -function validateTrack(payload) { - if (!payload.user_id && !payload.email) { - throw new InstrumentationError('Either of `email` or `userId` is required for Track call'); - } - // pass only string, number, boolean properties - if (payload.metadata) { - // reserved metadata contains JSON objects that does not requires flattening - const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(payload.metadata); - return { ...payload, metadata: { ...reservedMetadata, ...flattenJson(restMetadata) } }; - } - - return payload; -} - -const checkIfEmailOrUserIdPresent = (message, Config) => { - const { context, anonymousId } = message; - let { userId } = message; - if (Config.sendAnonymousId && !userId) { - userId = anonymousId; - } - return !!(userId || context.traits?.email); -}; - -function attachUserAndCompany(message, Config) { - const email = message.context?.traits?.email; - const { userId, anonymousId, traits, groupId } = message; - const requestBody = {}; - if (userId) { - requestBody.user_id = userId; - } - if (Config.sendAnonymousId && !userId) { - requestBody.user_id = anonymousId; - } - if (email) { - requestBody.email = email; - } - const companyObj = { - company_id: groupId, - }; - if (traits?.name) { - companyObj.name = traits.name; - } - requestBody.companies = [companyObj]; - const response = defaultRequestConfig(); - response.method = defaultPostRequestConfig.requestMethod; - response.endpoint = ConfigCategory.IDENTIFY.endpoint; - response.headers = { - 'Content-Type': JSON_MIME_TYPE, - Authorization: `Bearer ${Config.apiKey}`, - Accept: JSON_MIME_TYPE, - 'Intercom-Version': '1.4', - }; - response.body.JSON = requestBody; - return response; -} - -function buildCustomAttributes(message, payload) { - const finalPayload = payload; - const { traits } = message; - const customAttributes = {}; - const companyReservedKeys = [ - 'remoteCreatedAt', - 'monthlySpend', - 'industry', - 'website', - 'size', - 'plan', - 'name', - ]; - - if (traits) { - Object.keys(traits).forEach((key) => { - if (!companyReservedKeys.includes(key) && key !== 'userId') { - customAttributes[key] = traits[key]; - } - }); - } - - if (Object.keys(customAttributes).length > 0) { - finalPayload.custom_attributes = flattenJson(customAttributes); - } - - return finalPayload; -} - -function validateAndBuildResponse(message, payload, category, destination) { - const respList = []; - const response = defaultRequestConfig(); - response.method = defaultPostRequestConfig.requestMethod; - response.endpoint = category.endpoint; - response.headers = { - 'Content-Type': JSON_MIME_TYPE, - Authorization: `Bearer ${destination.Config.apiKey}`, - Accept: JSON_MIME_TYPE, - 'Intercom-Version': '1.4', - }; - response.userId = message.anonymousId; - const messageType = message.type.toLowerCase(); - switch (messageType) { - case EventType.IDENTIFY: - response.body.JSON = removeUndefinedAndNullValues( - validateIdentify(message, payload, destination.Config), - ); - break; - case EventType.TRACK: - response.body.JSON = removeUndefinedAndNullValues(validateTrack(payload)); - break; - case EventType.GROUP: { - response.body.JSON = removeUndefinedAndNullValues(buildCustomAttributes(message, payload)); - respList.push(response); - if (checkIfEmailOrUserIdPresent(message, destination.Config)) { - const attachUserAndCompanyResponse = attachUserAndCompany(message, destination.Config); - attachUserAndCompanyResponse.userId = message.anonymousId; - respList.push(attachUserAndCompanyResponse); - } - break; - } - default: - throw new InstrumentationError(`Message type ${messageType} not supported`); - } - - return messageType === EventType.GROUP ? respList : response; -} - -function processSingleMessage(message, destination) { - if (!message.type) { - throw new InstrumentationError('Message Type is not present. Aborting message.'); - } - const { sendAnonymousId } = destination.Config; - const messageType = message.type.toLowerCase(); - let category; - - switch (messageType) { - case EventType.IDENTIFY: - category = ConfigCategory.IDENTIFY; - break; - case EventType.TRACK: - category = ConfigCategory.TRACK; - break; - case EventType.GROUP: - category = ConfigCategory.GROUP; - break; - default: - throw new InstrumentationError(`Message type ${messageType} not supported`); - } - - // build the response and return - let payload; - if (get(message, MappedToDestinationKey)) { - addExternalIdToTraits(message); - payload = getFieldValueFromMessage(message, 'traits'); - } else { - payload = constructPayload(message, MappingConfig[category.name]); - } - if (category !== ConfigCategory.GROUP && sendAnonymousId && !payload.user_id) { - payload.user_id = message.anonymousId; - } - return validateAndBuildResponse(message, payload, category, destination); -} - -function process(event) { - const response = processSingleMessage(event.message, event.destination); - return response; -} - -const processRouterDest = async (inputs, reqMetadata) => { - const respList = await simpleProcessRouterDest(inputs, process, reqMetadata); - return respList; -}; - -module.exports = { process, processRouterDest }; diff --git a/src/v0/destinations/intercom/util.js b/src/v0/destinations/intercom/util.js deleted file mode 100644 index 24a2934f7e0..00000000000 --- a/src/v0/destinations/intercom/util.js +++ /dev/null @@ -1,32 +0,0 @@ -const { MetadataTypes } = require('./config'); - -/** - * Separates reserved metadata from rest of the metadata based on the metadata types - * ref:- https://developers.intercom.com/intercom-api-reference/v1.4/reference/event-metadata-types - * @param {*} metadata - * @returns - */ -function separateReservedAndRestMetadata(metadata) { - const reservedMetadata = {}; - const restMetadata = {}; - if (metadata) { - Object.entries(metadata).forEach(([key, value]) => { - if (value && typeof value === 'object') { - const hasMonetaryAmountKeys = MetadataTypes.monetaryAmount.every((type) => type in value); - const hasRichLinkKeys = MetadataTypes.richLink.every((type) => type in value); - if (hasMonetaryAmountKeys || hasRichLinkKeys) { - reservedMetadata[key] = value; - } else { - restMetadata[key] = value; - } - } else { - restMetadata[key] = value; - } - }); - } - - // Return the separated metadata objects - return { reservedMetadata, restMetadata }; -} - -module.exports = { separateReservedAndRestMetadata }; diff --git a/src/v0/destinations/intercom/util.test.js b/src/v0/destinations/intercom/util.test.js deleted file mode 100644 index 99dbdd1f7ec..00000000000 --- a/src/v0/destinations/intercom/util.test.js +++ /dev/null @@ -1,176 +0,0 @@ -const { separateReservedAndRestMetadata } = require('./util'); - -describe('separateReservedAndRestMetadata utility test', () => { - it('separate reserved and rest metadata', () => { - const metadata = { - property1: 1, - property2: 'test', - property3: true, - property4: { - property1: 1, - property2: 'test', - property3: { - subProp1: { - a: 'a', - b: 'b', - }, - subProp2: ['a', 'b'], - }, - }, - property5: {}, - property6: [], - property7: null, - property8: undefined, - revenue: { - amount: 1232, - currency: 'inr', - test: 123, - }, - price: { - amount: 3000, - currency: 'USD', - }, - article: { - url: 'https://example.org/ab1de.html', - value: 'the dude abides', - }, - }; - const expectedReservedMetadata = { - revenue: { - amount: 1232, - currency: 'inr', - test: 123, - }, - price: { - amount: 3000, - currency: 'USD', - }, - article: { - url: 'https://example.org/ab1de.html', - value: 'the dude abides', - }, - }; - const expectedRestMetadata = { - property1: 1, - property2: 'test', - property3: true, - property4: { - property1: 1, - property2: 'test', - property3: { - subProp1: { - a: 'a', - b: 'b', - }, - subProp2: ['a', 'b'], - }, - }, - property5: {}, - property6: [], - property7: null, - property8: undefined, - }; - const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); - - expect(expectedReservedMetadata).toEqual(reservedMetadata); - expect(expectedRestMetadata).toEqual(restMetadata); - }); - - it('reserved metadata types not present in input metadata', () => { - const metadata = { - property1: 1, - property2: 'test', - property3: true, - property4: { - property1: 1, - property2: 'test', - property3: { - subProp1: { - a: 'a', - b: 'b', - }, - subProp2: ['a', 'b'], - }, - }, - property5: {}, - property6: [], - property7: null, - property8: undefined, - }; - const expectedRestMetadata = { - property1: 1, - property2: 'test', - property3: true, - property4: { - property1: 1, - property2: 'test', - property3: { - subProp1: { - a: 'a', - b: 'b', - }, - subProp2: ['a', 'b'], - }, - }, - property5: {}, - property6: [], - property7: null, - property8: undefined, - }; - const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); - - expect({}).toEqual(reservedMetadata); - expect(expectedRestMetadata).toEqual(restMetadata); - }); - - it('metadata input contains only reserved metadata types', () => { - const metadata = { - revenue: { - amount: 1232, - currency: 'inr', - test: 123, - }, - price: { - amount: 3000, - currency: 'USD', - }, - article: { - url: 'https://example.org/ab1de.html', - value: 'the dude abides', - }, - }; - const expectedReservedMetadata = { - revenue: { - amount: 1232, - currency: 'inr', - test: 123, - }, - price: { - amount: 3000, - currency: 'USD', - }, - article: { - url: 'https://example.org/ab1de.html', - value: 'the dude abides', - }, - }; - const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); - - expect(expectedReservedMetadata).toEqual(reservedMetadata); - expect({}).toEqual(restMetadata); - }); - - it('empty metadata object', () => { - const metadata = {}; - const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); - expect({}).toEqual(reservedMetadata); - expect({}).toEqual(restMetadata); - }); - - it('null/undefined metadata', () => { - const metadata = null; - const { reservedMetadata, restMetadata } = separateReservedAndRestMetadata(metadata); - expect({}).toEqual(reservedMetadata); - expect({}).toEqual(restMetadata); - }); -}); diff --git a/src/v0/destinations/iterable/transform.js b/src/v0/destinations/iterable/transform.js index dd67deef69e..c676097c96a 100644 --- a/src/v0/destinations/iterable/transform.js +++ b/src/v0/destinations/iterable/transform.js @@ -1,6 +1,7 @@ const lodash = require('lodash'); const get = require('get-value'); const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const logger = require('../../../logger'); const { getCatalogEndpoint, hasMultipleResponses, @@ -28,6 +29,7 @@ const { const { JSON_MIME_TYPE } = require('../../util/constant'); const { mappingConfig, ConfigCategory } = require('./config'); const { EventType, MappedToDestinationKey } = require('../../../constants'); +const { MD5 } = require('../../../cdk/v2/bindings/default'); /** * Common payload builder function for all events @@ -108,10 +110,16 @@ const responseBuilder = (message, category, destination) => { * @returns */ const responseBuilderForRegisterDeviceOrBrowserTokenEvents = (message, destination) => { - const { device } = message.context; + const { device, os } = message.context; const category = device?.token ? ConfigCategory.IDENTIFY_DEVICE : ConfigCategory.IDENTIFY_BROWSER; - const response = responseBuilder(message, category, destination); + + const categoryWithEndpoint = getCategoryWithEndpoint(category, destination.Config.dataCenter); + const response = responseBuilder(message, categoryWithEndpoint, destination); response.headers.api_key = destination.Config.registerDeviceOrBrowserApiKey; + logger.info('{{ITERABLE::}} registerDeviceApiCalled', { + destinationId: destination.ID, + token: MD5(device?.token || os?.token || 'no token'), + }); return response; }; diff --git a/src/v0/destinations/iterable/util.js b/src/v0/destinations/iterable/util.js index 764d76f8825..34ee1237598 100644 --- a/src/v0/destinations/iterable/util.js +++ b/src/v0/destinations/iterable/util.js @@ -69,6 +69,11 @@ const validateMandatoryField = (payload) => { } }; +const getCategoryWithEndpoint = (categoryConfig, dataCenter) => ({ + ...categoryConfig, + endpoint: constructEndpoint(dataCenter, categoryConfig), +}); + /** * Check for register device and register browser events * @param {*} message @@ -80,18 +85,13 @@ const hasMultipleResponses = (message, category, config) => { const { context } = message; const isIdentifyEvent = message.type === EventType.IDENTIFY; - const isIdentifyCategory = category === ConfigCategory.IDENTIFY; - const hasToken = context && (context.device?.token || context.os?.token); + const isIdentifyCategory = category.action === ConfigCategory.IDENTIFY.action; + const hasToken = Boolean(context && (context.device?.token || context.os?.token)); const hasRegisterDeviceOrBrowserKey = Boolean(config.registerDeviceOrBrowserApiKey); return isIdentifyEvent && isIdentifyCategory && hasToken && hasRegisterDeviceOrBrowserKey; }; -const getCategoryWithEndpoint = (categoryConfig, dataCenter) => ({ - ...categoryConfig, - endpoint: constructEndpoint(dataCenter, categoryConfig), -}); - /** * Returns category value * @param {*} message diff --git a/src/v0/destinations/iterable/util.test.js b/src/v0/destinations/iterable/util.test.js index 6bbf00ba085..8189a5b1f29 100644 --- a/src/v0/destinations/iterable/util.test.js +++ b/src/v0/destinations/iterable/util.test.js @@ -7,6 +7,8 @@ const { updateUserEventPayloadBuilder, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + hasMultipleResponses, + getCategoryWithEndpoint, } = require('./util'); const { ConfigCategory } = require('./config'); @@ -798,4 +800,93 @@ describe('iterable utils test', () => { ); }); }); + describe('Unit test cases for iterable hasMultipleResponses', () => { + it('should return false when message type is not identify', () => { + const category = getCategoryWithEndpoint(ConfigCategory.IDENTIFY, 'USDC'); + const message = { + type: 'track', + context: { + device: { token: '123' }, + os: { token: '456' }, + }, + }; + const config = { registerDeviceOrBrowserApiKey: 'test-key' }; + + expect(hasMultipleResponses(message, category, config)).toBe(false); + }); + + it('should return false when category is not identify', () => { + const category = getCategoryWithEndpoint(ConfigCategory.PAGE, 'USDC'); + const message = { + type: 'identify', + context: { + device: { token: '123' }, + os: { token: '456' }, + }, + }; + const config = { registerDeviceOrBrowserApiKey: 'test-key' }; + + expect(hasMultipleResponses(message, category, config)).toBe(false); + }); + + it('should return false when no device/os token present', () => { + const category = getCategoryWithEndpoint(ConfigCategory.IDENTIFY, 'USDC'); + const message = { + type: 'identify', + context: { + device: {}, + os: {}, + }, + }; + const config = { registerDeviceOrBrowserApiKey: 'test-key' }; + + expect(hasMultipleResponses(message, category, config)).toBe(false); + }); + + it('should return false when registerDeviceOrBrowserApiKey not present in config', () => { + const category = getCategoryWithEndpoint(ConfigCategory.IDENTIFY, 'USDC'); + const message = { + type: 'identify', + context: { + device: { token: '123' }, + os: { token: '456' }, + }, + }; + const config = {}; + + expect(hasMultipleResponses(message, category, config)).toBe(false); + }); + + it('should return true when all conditions are met with device token', () => { + const category = getCategoryWithEndpoint(ConfigCategory.IDENTIFY, 'USDC'); + const message = { + type: 'identify', + context: { + device: { token: '123' }, + }, + }; + const config = { + dataCenter: '123', + registerDeviceOrBrowserApiKey: 'test-key', + }; + + expect(hasMultipleResponses(message, category, config)).toBe(true); + }); + + it('should return true when all conditions are met with os token', () => { + const category = getCategoryWithEndpoint(ConfigCategory.IDENTIFY, 'USDC'); + const message = { + type: 'identify', + context: { + os: { token: '456' }, + }, + }; + const config = { + dataCenter: '123', + registerDeviceOrBrowserApiKey: 'test-key', + }; + + expect(hasMultipleResponses(message, category, config)).toBe(true); + }); + }); }); diff --git a/src/v0/destinations/klaviyo/util.js b/src/v0/destinations/klaviyo/util.js index 4421764d953..42e1d8a8012 100644 --- a/src/v0/destinations/klaviyo/util.js +++ b/src/v0/destinations/klaviyo/util.js @@ -1,5 +1,6 @@ const set = require('set-value'); const lodash = require('lodash'); +const { parsePhoneNumberFromString } = require('libphonenumber-js'); const { NetworkError, InstrumentationError } = require('@rudderstack/integrations-lib'); const { WhiteListedTraits } = require('../../../constants'); const { @@ -32,6 +33,24 @@ const { const REVISION_CONSTANT = '2023-02-22'; +/** + * This function is used to check if the phone number is in E.164 format. It uses libphonenumber-js library to parse the phone number + * @param {*} phoneNumber + * @returns boolean + */ +function isValidE164PhoneNumber(phoneNumber) { + try { + // Remove all non-numeric characters from the phone number like spaces, hyphens, etc. + const sanitizedPhoneNumber = phoneNumber.replace(/[^\d+]/g, ''); + const parsedNumber = parsePhoneNumberFromString(sanitizedPhoneNumber); + // Check if the number is valid and properly formatted in E.164. + return parsedNumber && parsedNumber.format('E.164') === sanitizedPhoneNumber; + } catch (error) { + // If parsing fails, it's not a valid E.164 number, i.e doesn't start with '+' and country code + return false; + } +} + /** * This function calls the create user endpoint ref: https://developers.klaviyo.com/en/reference/create_profile * If the user doesn't exist, it creates a profile for the user and return 201 status code and the response which contains all the profile data @@ -414,6 +433,12 @@ const constructProfile = (message, destination, isIdentifyCall) => { message, MAPPING_CONFIG[CONFIG_CATEGORIES.PROFILEV2.name], ); + if ( + isDefinedAndNotNull(profileAttributes.phone_number) && + !isValidE164PhoneNumber(profileAttributes.phone_number) + ) { + throw new InstrumentationError('Phone number is not in E.164 format.'); + } const { enforceEmailAsPrimary, flattenProperties } = destination.Config; let customPropertyPayload = {}; const { meta, metadataFields } = getProfileMetadataAndMetadataFields(message); diff --git a/src/v0/destinations/mautic/utils.js b/src/v0/destinations/mautic/utils.js index fc9654d2e3a..061c91f3578 100644 --- a/src/v0/destinations/mautic/utils.js +++ b/src/v0/destinations/mautic/utils.js @@ -1,5 +1,6 @@ /* eslint-disable no-return-assign, no-param-reassign, no-restricted-syntax */ const get = require('get-value'); +const validator = require('validator'); const { NetworkError, InstrumentationError, @@ -24,15 +25,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); function createAxiosUrl(propertyName, value, baseUrl) { return `${baseUrl}/contacts?where%5B0%5D%5Bcol%5D=${propertyName}&where%5B0%5D%5Bexpr%5D=eq&where%5B0%5D%5Bval%5D=${value}`; } -/** - * @param {*} inputText - * @returns Boolean Value - * for validating email match - */ -function validateEmail(inputText) { - const mailformat = /^[\d%+._a-z-]+@[\d.a-z-]+\.[a-z]{2,3}$/; - return mailformat.test(inputText); -} + /** * @param {*} message * @param {*} lookUpField @@ -126,7 +119,7 @@ const validatePayload = (payload) => { throw new InstrumentationError('The provided phone number is invalid'); } - if (payload.email && !validateEmail(payload.email)) { + if (payload.email && !validator.isEmail(payload.email)) { throw new InstrumentationError('The provided email is invalid'); } return true; @@ -220,7 +213,6 @@ const getEndpoint = (Config) => { }; module.exports = { deduceStateField, - validateEmail, validatePhone, deduceAddressFields, validateGroupCall, diff --git a/src/v0/destinations/mp/data/MPEventPropertiesConfig.json b/src/v0/destinations/mp/data/MPEventPropertiesConfig.json index 956b60248eb..bed246bc7be 100644 --- a/src/v0/destinations/mp/data/MPEventPropertiesConfig.json +++ b/src/v0/destinations/mp/data/MPEventPropertiesConfig.json @@ -76,11 +76,11 @@ "destKey": "mp_lib" }, { - "sourceKeys": "context.page.initialReferrer", + "sourceKeys": ["context.page.initialReferrer", "context.page.initial_referrer"], "destKey": "$initial_referrer" }, { - "sourceKeys": "context.page.initialReferringDomain", + "sourceKeys": ["context.page.initialReferringDomain", "context.page.initial_referring_domain"], "destKey": "$initial_referring_domain" }, { diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index b2807d6e119..50a8f9ac2b8 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -17,6 +17,7 @@ const { isObject, isDefinedAndNotNullAndNotEmpty, isDefinedAndNotNull, + removeUndefinedValues, } = require('../../util'); const { ConfigCategory, @@ -32,6 +33,30 @@ const mPProfileAndroidConfigJson = mappingConfig[ConfigCategory.PROFILE_ANDROID. const mPProfileIosConfigJson = mappingConfig[ConfigCategory.PROFILE_IOS.name]; const mPSetOnceConfigJson = mappingConfig[ConfigCategory.SET_ONCE.name]; +/** + * This method populates the payload with device fields based on mp mapping + * @param message + * @param rawPayload + * @returns + */ +const populateDeviceFieldsInPayload = (message, rawPayload) => { + const device = get(message, 'context.device'); + let payload = {}; + let updatedRawPayload = { ...rawPayload }; + if (device) { + const deviceTokenArray = isDefinedAndNotNull(device.token) ? [device.token] : undefined; + if (isAppleFamily(device.type)) { + payload = constructPayload(message, mPProfileIosConfigJson); + updatedRawPayload.$ios_devices = deviceTokenArray; + } else if (device.type?.toLowerCase() === 'android') { + payload = constructPayload(message, mPProfileAndroidConfigJson); + updatedRawPayload.$android_devices = deviceTokenArray; + } + updatedRawPayload = removeUndefinedValues(updatedRawPayload); + } + return { ...updatedRawPayload, ...payload }; +}; + /** * this function has been used to create * @param {*} message rudderstack identify payload @@ -75,18 +100,8 @@ const getTransformedJSON = (message, mappingJson, useNewMapping) => { } } - const device = get(message, 'context.device'); - if (device && device.token) { - let payload; - if (isAppleFamily(device.type)) { - payload = constructPayload(message, mPProfileIosConfigJson); - rawPayload.$ios_devices = [device.token]; - } else if (device.type.toLowerCase() === 'android') { - payload = constructPayload(message, mPProfileAndroidConfigJson); - rawPayload.$android_devices = [device.token]; - } - rawPayload = { ...rawPayload, ...payload }; - } + rawPayload = populateDeviceFieldsInPayload(message, rawPayload); + if (message.channel === 'web' && message.context?.userAgent) { const browser = getBrowserInfo(message.context.userAgent); rawPayload.$browser = browser.name; @@ -373,4 +388,5 @@ module.exports = { trimTraits, generatePageOrScreenCustomEventName, recordBatchSizeMetrics, + getTransformedJSON, }; diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index 3666081f593..4813b1fe1ff 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -5,9 +5,11 @@ const { buildUtmParams, trimTraits, generatePageOrScreenCustomEventName, + getTransformedJSON, } = require('./util'); const { FEATURE_GZIP_SUPPORT } = require('../../util/constant'); const { ConfigurationError } = require('@rudderstack/integrations-lib'); +const { mappingConfig, ConfigCategory } = require('./config'); const maxBatchSizeMock = 2; @@ -489,3 +491,199 @@ describe('generatePageOrScreenCustomEventName', () => { expect(result).toBe(expected); }); }); + +describe('Unit test cases for getTransformedJSON', () => { + it('should transform the message payload to appropriate payload if device.token is present', () => { + const message = { + context: { + app: { + build: '1', + name: 'LeanPlumIntegrationAndroid', + namespace: 'com.android.SampleLeanPlum', + version: '1.0', + }, + device: { + id: '5094f5704b9cf2b3', + manufacturer: 'Google', + model: 'Android SDK built for x86', + name: 'generic_x86', + type: 'ios', + token: 'test_device_token', + }, + network: { carrier: 'Android', bluetooth: false, cellular: true, wifi: true }, + os: { name: 'iOS', version: '8.1.0' }, + timezone: 'Asia/Kolkata', + traits: { userId: 'test_user_id' }, + }, + }; + const result = getTransformedJSON(message, mappingConfig[ConfigCategory.IDENTIFY.name], true); + + const expectedResult = { + $carrier: 'Android', + $manufacturer: 'Google', + $model: 'Android SDK built for x86', + $wifi: true, + userId: 'test_user_id', + $ios_devices: ['test_device_token'], + $os: 'iOS', + $ios_device_model: 'Android SDK built for x86', + $ios_version: '8.1.0', + $ios_app_release: '1', + $ios_app_version: '1.0', + }; + + expect(result).toEqual(expectedResult); + }); + + it('should transform the message payload to appropriate payload if device.token is present and device.token is null', () => { + const message = { + context: { + app: { + build: '1', + name: 'LeanPlumIntegrationAndroid', + namespace: 'com.android.SampleLeanPlum', + version: '1.0', + }, + device: { + id: '5094f5704b9cf2b3', + manufacturer: 'Google', + model: 'Android SDK built for x86', + name: 'generic_x86', + type: 'android', + token: null, + }, + network: { carrier: 'Android', bluetooth: false, cellular: true, wifi: true }, + os: { name: 'Android', version: '8.1.0' }, + timezone: 'Asia/Kolkata', + traits: { userId: 'test_user_id' }, + }, + }; + const result = getTransformedJSON(message, mappingConfig[ConfigCategory.IDENTIFY.name], true); + + const expectedResult = { + $carrier: 'Android', + $manufacturer: 'Google', + $model: 'Android SDK built for x86', + $wifi: true, + userId: 'test_user_id', + $os: 'Android', + $android_model: 'Android SDK built for x86', + $android_os_version: '8.1.0', + $android_manufacturer: 'Google', + $android_app_version: '1.0', + $android_app_version_code: '1.0', + $android_brand: 'Google', + }; + + expect(result).toEqual(expectedResult); + }); + + it('should transform the message payload to appropriate payload if device.token is not present for apple device', () => { + const message = { + context: { + app: { + build: '1', + name: 'LeanPlumIntegrationAndroid', + namespace: 'com.android.SampleLeanPlum', + version: '1.0', + }, + device: { + id: '5094f5704b9cf2b3', + manufacturer: 'Google', + model: 'Android SDK built for x86', + name: 'generic_x86', + type: 'ios', + }, + network: { carrier: 'Android', bluetooth: false, cellular: true, wifi: true }, + os: { name: 'iOS', version: '8.1.0' }, + timezone: 'Asia/Kolkata', + traits: { userId: 'test_user_id' }, + }, + }; + const result = getTransformedJSON(message, mappingConfig[ConfigCategory.IDENTIFY.name], true); + + const expectedResult = { + $carrier: 'Android', + $manufacturer: 'Google', + $model: 'Android SDK built for x86', + $wifi: true, + userId: 'test_user_id', + $os: 'iOS', + $ios_device_model: 'Android SDK built for x86', + $ios_version: '8.1.0', + $ios_app_release: '1', + $ios_app_version: '1.0', + }; + + expect(result).toEqual(expectedResult); + }); + + it('should transform the message payload to appropriate payload if device.token is not present for android device', () => { + const message = { + context: { + app: { + build: '1', + name: 'LeanPlumIntegrationAndroid', + namespace: 'com.android.SampleLeanPlum', + version: '1.0', + }, + device: { + id: '5094f5704b9cf2b3', + manufacturer: 'Google', + model: 'Android SDK built for x86', + name: 'generic_x86', + type: 'android', + token: undefined, + }, + network: { carrier: 'Android', bluetooth: false, cellular: true, wifi: true }, + os: { name: 'Android', version: '8.1.0' }, + timezone: 'Asia/Kolkata', + traits: { userId: 'test_user_id' }, + }, + }; + const result = getTransformedJSON(message, mappingConfig[ConfigCategory.IDENTIFY.name], true); + + const expectedResult = { + $carrier: 'Android', + $manufacturer: 'Google', + $model: 'Android SDK built for x86', + $wifi: true, + userId: 'test_user_id', + $os: 'Android', + $android_model: 'Android SDK built for x86', + $android_os_version: '8.1.0', + $android_manufacturer: 'Google', + $android_app_version: '1.0', + $android_app_version_code: '1.0', + $android_brand: 'Google', + }; + + expect(result).toEqual(expectedResult); + }); + + it('should transform the message payload to appropriate payload if device is not present', () => { + const message = { + context: { + app: { + build: '1', + name: 'LeanPlumIntegrationAndroid', + namespace: 'com.android.SampleLeanPlum', + version: '1.0', + }, + network: { carrier: 'Android', bluetooth: false, cellular: true, wifi: true }, + os: { name: 'iOS', version: '8.1.0' }, + timezone: 'Asia/Kolkata', + traits: { userId: 'test_user_id' }, + }, + }; + const result = getTransformedJSON(message, mappingConfig[ConfigCategory.IDENTIFY.name], true); + + const expectedResult = { + $carrier: 'Android', + $wifi: true, + userId: 'test_user_id', + }; + + expect(result).toEqual(expectedResult); + }); +}); diff --git a/src/v0/destinations/sendinblue/util.js b/src/v0/destinations/sendinblue/util.js index 3af35cd7b99..813d5c4150b 100644 --- a/src/v0/destinations/sendinblue/util.js +++ b/src/v0/destinations/sendinblue/util.js @@ -1,10 +1,10 @@ const { NetworkError, InstrumentationError } = require('@rudderstack/integrations-lib'); +const validator = require('validator'); const { EMAIL_SUFFIX, getContactDetailsEndpoint } = require('./config'); const { getHashFromArray, getIntegrationsObj, isNotEmpty, - validateEmail, validatePhoneWithCountryCode, getDestinationExternalID, } = require('../../util'); @@ -36,7 +36,7 @@ const checkIfEmailOrPhoneExists = (email, phone) => { }; const validateEmailAndPhone = (email, phone = null) => { - if (email && !validateEmail(email)) { + if (email && !validator.isEmail(email)) { throw new InstrumentationError('The provided email is invalid'); } diff --git a/src/v0/sources/shopify/util.js b/src/v0/sources/shopify/util.js index b7e79e35a16..3665dfff203 100644 --- a/src/v0/sources/shopify/util.js +++ b/src/v0/sources/shopify/util.js @@ -272,6 +272,7 @@ module.exports = { createPropertiesForEcomEvent, extractEmailFromPayload, getAnonymousIdAndSessionId, + getCartToken, checkAndUpdateCartItems, getHashLineItems, getDataFromRedis, diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 0f9abfb0403..36e04e53950 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -971,6 +971,7 @@ const handleMetadataForValue = (value, metadata, destKey, integrationsObj = null validateTimestamp, allowedKeyCheck, toArray, + regex, } = metadata; // if value is null and defaultValue is supplied - use that @@ -1044,7 +1045,14 @@ const handleMetadataForValue = (value, metadata, destKey, integrationsObj = null } return [formattedVal]; } - + if (regex) { + const regexPattern = new RegExp(regex); + if (!regexPattern.test(formattedVal)) { + throw new InstrumentationError( + `The value '${formattedVal}' does not match the regex pattern, ${regex}`, + ); + } + } return formattedVal; }; @@ -1924,12 +1932,6 @@ const refinePayload = (obj) => { return refinedPayload; }; -const validateEmail = (email) => { - const regex = - /^(([^\s"(),.:;<>@[\\\]]+(\.[^\s"(),.:;<>@[\\\]]+)*)|(".+"))@((\[(?:\d{1,3}\.){3}\d{1,3}])|(([\dA-Za-z-]+\.)+[A-Za-z]{2,}))$/; - return !!regex.test(email); -}; - const validatePhoneWithCountryCode = (phone) => { const regex = /^\+(?:[\d{] ?){6,14}\d$/; return !!regex.test(phone); @@ -2464,7 +2466,6 @@ module.exports = { getErrorStatusCode, getDestAuthCacheInstance, refinePayload, - validateEmail, validateEventName, validatePhoneWithCountryCode, getEventReqMetadata, @@ -2489,4 +2490,5 @@ module.exports = { removeEmptyKey, isAxiosError, convertToUuid, + handleMetadataForValue, }; diff --git a/src/v0/util/index.test.js b/src/v0/util/index.test.js index cfdfefddee3..0b0b585e4cb 100644 --- a/src/v0/util/index.test.js +++ b/src/v0/util/index.test.js @@ -1049,3 +1049,27 @@ describe('convertToUuid', () => { expect(result).toBe('672ca00c-37f4-5d71-b8c3-6ae0848080ec'); }); }); + +describe('', () => { + it('should return original value when regex pattern is invalid', () => { + const value = 'test value'; + const metadata = { + regex: `\\b(?!1000\\b)\\d{4,}\\b`, + }; + try { + const result = utilities.handleMetadataForValue(value, metadata); + } catch (e) { + expect(e.message).toBe( + `The value 'test value' does not match the regex pattern, \\b(?!1000\\b)\\d{4,}\\b`, + ); + } + }); + it('should return true when the regex matches', () => { + const value = 1003; + const metadata = { + regex: `\\b(?!1000\\b)\\d{4,}\\b`, + }; + const res = utilities.handleMetadataForValue(value, metadata); + expect(res).toBe(1003); + }); +}); diff --git a/src/v1/sources/shopify/config.js b/src/v1/sources/shopify/config.js index 9cb11e471f5..20db7be331e 100644 --- a/src/v1/sources/shopify/config.js +++ b/src/v1/sources/shopify/config.js @@ -35,11 +35,20 @@ const PIXEL_EVENT_MAPPING = { search_submitted: 'Search Submitted', }; +const ECOM_TOPICS = { + CHECKOUTS_CREATE: 'checkouts_create', + CHECKOUTS_UPDATE: 'checkouts_update', + ORDERS_UPDATE: 'orders_updated', + ORDERS_CREATE: 'orders_create', + ORDERS_CANCELLED: 'orders_cancelled', +}; + const RUDDER_ECOM_MAP = { - checkouts_create: 'Checkout Started - Webhook', + checkouts_create: 'Checkout Started Webhook', checkouts_update: 'Checkout Updated', orders_updated: 'Order Updated', orders_create: 'Order Created', + orders_cancelled: 'Order Cancelled', }; const contextualFieldMappingJSON = JSON.parse( @@ -94,6 +103,7 @@ module.exports = { INTEGERATION, PIXEL_EVENT_TOPICS, PIXEL_EVENT_MAPPING, + ECOM_TOPICS, RUDDER_ECOM_MAP, contextualFieldMappingJSON, cartViewedEventMappingJSON, diff --git a/src/v1/sources/shopify/pixelEventsMappings/campaignObjectMappings.json b/src/v1/sources/shopify/pixelEventsMappings/campaignObjectMappings.json new file mode 100644 index 00000000000..319502e377b --- /dev/null +++ b/src/v1/sources/shopify/pixelEventsMappings/campaignObjectMappings.json @@ -0,0 +1,18 @@ +[ + { + "sourceKeys": "utm_campaign", + "destKeys": "name" + }, + { + "sourceKeys": "utm_medium", + "destKeys": "medium" + }, + { + "sourceKeys": "utm_term", + "destKeys": "term" + }, + { + "sourceKeys": "utm_content", + "destKeys": "content" + } +] diff --git a/src/v1/sources/shopify/transform.js b/src/v1/sources/shopify/transform.js index 5ebf4a34fc5..23228ecc10f 100644 --- a/src/v1/sources/shopify/transform.js +++ b/src/v1/sources/shopify/transform.js @@ -1,29 +1,13 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform'); -const { process: processWebhookEvents } = require('../../../v0/sources/shopify/transform'); -const { - process: processPixelWebhookEvents, -} = require('./webhookTransformations/serverSideTransform'); +const { process: processV0 } = require('../../../v0/sources/shopify/transform'); +const { processV1Events } = require('./transformV1'); +const { isShopifyV1Event } = require('./utils'); const process = async (inputEvent) => { const { event } = inputEvent; - const { query_parameters } = event; - // check identify the event is from the web pixel based on the pixelEventLabel property. - const { pixelEventLabel: pixelClientEventLabel } = event; - if (pixelClientEventLabel) { - // this is a event fired from the web pixel loaded on the browser - // by the user interactions with the store. - const pixelWebEventResponse = await processPixelWebEvents(event); - return pixelWebEventResponse; + if (isShopifyV1Event(event)) { + return processV1Events(event); } - if (query_parameters && query_parameters?.version?.[0] === 'pixel') { - // this is a server-side event from the webhook subscription made by the pixel app. - const pixelWebhookEventResponse = await processPixelWebhookEvents(event); - return pixelWebhookEventResponse; - } - // this is a server-side event from the webhook subscription made by the legacy tracker-based app. - const response = await processWebhookEvents(event); - return response; + return processV0(event); }; module.exports = { process }; diff --git a/src/v1/sources/shopify/transformV1.js b/src/v1/sources/shopify/transformV1.js new file mode 100644 index 00000000000..b3e01d95c67 --- /dev/null +++ b/src/v1/sources/shopify/transformV1.js @@ -0,0 +1,36 @@ +const { PlatformError } = require('@rudderstack/integrations-lib'); +const { isIdentifierEvent, processIdentifierEvent } = require('./utils'); +const { processWebhookEvents } = require('./webhookTransformations/serverSideTransform'); +const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform'); + +const processV1Events = async (event) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { query_parameters } = event; + + // these are the events from the front-end tracking, viz. web-pixel or theme-app extension. + const { pixelEventLabel: clientSideEvent } = event; + const isServerSideEvent = query_parameters && query_parameters?.version?.[0] === 'pixel'; + + if (clientSideEvent) { + // check if the event is an identifier event, used to set the anonymousId in the redis for identity stitching. + if (isIdentifierEvent(event)) { + return processIdentifierEvent(event); + } + // handle events from the app pixel. + const pixelWebEventResponse = processPixelWebEvents(event); + return pixelWebEventResponse; + } + if (isServerSideEvent) { + // this is a server-side event from the webhook subscription made by the pixel app. + const pixelWebhookEventResponse = await processWebhookEvents(event); + return pixelWebhookEventResponse; + } + throw new PlatformError( + 'Invalid Event for Shopiyf V1 (not matching client or server side event requirements)', + 500, + ); +}; + +module.exports = { + processV1Events, +}; diff --git a/src/v1/sources/shopify/utils.js b/src/v1/sources/shopify/utils.js new file mode 100644 index 00000000000..9502b2ee595 --- /dev/null +++ b/src/v1/sources/shopify/utils.js @@ -0,0 +1,34 @@ +const { RedisDB } = require('../../../util/redis/redisConnector'); + +const NO_OPERATION_SUCCESS = { + outputToSource: { + body: Buffer.from('OK').toString('base64'), + contentType: 'text/plain', + }, + statusCode: 200, +}; + +const isIdentifierEvent = (payload) => ['rudderIdentifier'].includes(payload?.event); + +const processIdentifierEvent = async (event) => { + const { cartToken, anonymousId } = event; + await RedisDB.setVal(`pixel:${cartToken}`, ['anonymousId', anonymousId]); + return NO_OPERATION_SUCCESS; +}; + +const isShopifyV1Event = (event) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { query_parameters } = event; + const { pixelEventLabel: pixelClientEventLabel } = event; + + return !!( + (query_parameters && query_parameters?.version?.[0] === 'pixel') || + pixelClientEventLabel + ); +}; + +module.exports = { + processIdentifierEvent, + isIdentifierEvent, + isShopifyV1Event, +}; diff --git a/src/v1/sources/shopify/utils.test.js b/src/v1/sources/shopify/utils.test.js new file mode 100644 index 00000000000..558cdec7c52 --- /dev/null +++ b/src/v1/sources/shopify/utils.test.js @@ -0,0 +1,48 @@ +const { isIdentifierEvent, processIdentifierEvent } = require('./utils'); +const { RedisDB } = require('../../../util/redis/redisConnector'); + +describe('Identifier Utils Tests', () => { + describe('test isIdentifierEvent', () => { + it('should return true if the event is rudderIdentifier', () => { + const event = { event: 'rudderIdentifier' }; + expect(isIdentifierEvent(event)).toBe(true); + }); + + it('should return false if the event is not rudderIdentifier', () => { + const event = { event: 'checkout started' }; + expect(isIdentifierEvent(event)).toBe(false); + }); + }); + + describe('test processIdentifierEvent', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should set the anonymousId in redis and return NO_OPERATION_SUCCESS', async () => { + const setValSpy = jest.spyOn(RedisDB, 'setVal').mockResolvedValue('OK'); + const event = { cartToken: 'cartTokenTest1', anonymousId: 'anonymousIdTest1' }; + + const response = await processIdentifierEvent(event); + + expect(setValSpy).toHaveBeenCalledWith('pixel:cartTokenTest1', [ + 'anonymousId', + 'anonymousIdTest1', + ]); + expect(response).toEqual({ + outputToSource: { + body: Buffer.from('OK').toString('base64'), + contentType: 'text/plain', + }, + statusCode: 200, + }); + }); + + it('should handle redis errors', async () => { + jest.spyOn(RedisDB, 'setVal').mockRejectedValue(new Error('Redis connection failed')); + const event = { cartToken: 'cartTokenTest1', anonymousId: 'anonymousIdTest1' }; + + await expect(processIdentifierEvent(event)).rejects.toThrow('Redis connection failed'); + }); + }); +}); diff --git a/src/v1/sources/shopify/webhookEventsMapping/productMapping.json b/src/v1/sources/shopify/webhookEventsMapping/productMapping.json index e78ed50dfc4..7bd20498286 100644 --- a/src/v1/sources/shopify/webhookEventsMapping/productMapping.json +++ b/src/v1/sources/shopify/webhookEventsMapping/productMapping.json @@ -8,7 +8,10 @@ }, { "sourceKeys": "total_price", - "destKey": "value" + "destKey": "value", + "metadata": { + "type": "toNumber" + } }, { "sourceKeys": "total_tax", diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js index d0221f6950a..1fe92bbee0d 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -1,25 +1,24 @@ -/* eslint-disable @typescript-eslint/naming-convention */ const lodash = require('lodash'); const get = require('get-value'); const stats = require('../../../../util/stats'); -const { getShopifyTopic, extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); -const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util'); +const { getShopifyTopic } = require('../../../../v0/sources/shopify/util'); +const { removeUndefinedAndNullValues } = require('../../../../v0/util'); const Message = require('../../../../v0/sources/message'); const { EventType } = require('../../../../constants'); const { INTEGERATION, MAPPING_CATEGORIES, IDENTIFY_TOPICS, - ECOM_TOPICS, SUPPORTED_TRACK_EVENTS, SHOPIFY_TRACK_MAP, lineItemsMappingJSON, } = require('../../../../v0/sources/shopify/config'); -const { RUDDER_ECOM_MAP } = require('../config'); +const { ECOM_TOPICS, RUDDER_ECOM_MAP } = require('../config'); const { createPropertiesForEcomEventFromWebhook, getProductsFromLineItems, - getAnonymousIdFromAttributes, + setAnonymousId, + handleCommonProperties, } = require('./serverSideUtlis'); const NO_OPERATION_SUCCESS = { @@ -65,9 +64,6 @@ const ecomPayloadBuilder = (event, shopifyTopic) => { if (event.billing_address) { message.setProperty('traits.billingAddress', event.billing_address); } - if (!message.userId && event.user_id) { - message.setProperty('userId', event.user_id); - } return message; }; @@ -96,6 +92,7 @@ const processEvent = async (inputEvent, metricMetadata) => { case ECOM_TOPICS.ORDERS_UPDATE: case ECOM_TOPICS.CHECKOUTS_CREATE: case ECOM_TOPICS.CHECKOUTS_UPDATE: + case ECOM_TOPICS.ORDERS_CANCELLED: message = ecomPayloadBuilder(event, shopifyTopic); break; default: @@ -110,42 +107,16 @@ const processEvent = async (inputEvent, metricMetadata) => { message = trackPayloadBuilder(event, shopifyTopic); break; } - - if (message.userId) { - message.userId = String(message.userId); - } - if (!get(message, 'traits.email')) { - const email = extractEmailFromPayload(event); - if (email) { - message.setProperty('traits.email', email); - } - } // attach anonymousId if the event is track event using note_attributes if (message.type !== EventType.IDENTIFY) { - const anonymousId = getAnonymousIdFromAttributes(event); - if (isDefinedAndNotNull(anonymousId)) { - message.setProperty('anonymousId', anonymousId); - } - } - message.setProperty(`integrations.${INTEGERATION}`, true); - message.setProperty('context.library', { - eventOrigin: 'server', - name: 'RudderStack Shopify Cloud', - version: '2.0.0', - }); - message.setProperty('context.topic', shopifyTopic); - // attaching cart, checkout and order tokens in context object - message.setProperty(`context.cart_token`, event.cart_token); - message.setProperty(`context.checkout_token`, event.checkout_token); - // raw shopify payload passed inside context object under shopifyDetails - message.setProperty('context.shopifyDetails', event); - if (shopifyTopic === 'orders_updated') { - message.setProperty(`context.order_token`, event.token); + await setAnonymousId(message, event, metricMetadata); } + // attach userId, email and other contextual properties + message = handleCommonProperties(message, event, shopifyTopic); message = removeUndefinedAndNullValues(message); return message; }; -const process = async (event) => { +const processWebhookEvents = async (event) => { const metricMetadata = { writeKey: event.query_parameters?.writeKey?.[0], source: 'SHOPIFY', @@ -155,7 +126,7 @@ const process = async (event) => { }; module.exports = { - process, + processWebhookEvents, processEvent, identifyPayloadBuilder, ecomPayloadBuilder, diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js index b94ec9c3ddf..070fdafdd71 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -1,15 +1,13 @@ +const { processEvent } = require('./serverSideTransform'); const { getProductsFromLineItems, createPropertiesForEcomEventFromWebhook, getAnonymousIdFromAttributes, + getCartToken, } = require('./serverSideUtlis'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); -const { constructPayload } = require('../../../../v0/util'); - -const { - lineItemsMappingJSON, - productMappingJSON, -} = require('../../../../v0/sources/shopify/config'); +const { lineItemsMappingJSON } = require('../../../../v0/sources/shopify/config'); const Message = require('../../../../v0/sources/message'); jest.mock('../../../../v0/sources/message'); @@ -63,7 +61,6 @@ describe('serverSideUtils.js', () => { }); it('should return array of products', () => { - const mapping = {}; const result = getProductsFromLineItems(LINEITEMS, lineItemsMappingJSON); expect(result).toEqual([ { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 }, @@ -115,7 +112,13 @@ describe('serverSideUtils.js', () => { // Handles empty note_attributes array gracefully it('should return null when note_attributes is an empty array', async () => { const event = { note_attributes: [] }; - const result = await getAnonymousIdFromAttributes(event); + const result = getAnonymousIdFromAttributes(event); + expect(result).toBeNull(); + }); + + it('should return null when note_attributes is not present', async () => { + const event = {}; + const result = getAnonymousIdFromAttributes(event); expect(result).toBeNull(); }); @@ -123,8 +126,48 @@ describe('serverSideUtils.js', () => { const event = { note_attributes: [{ name: 'rudderAnonymousId', value: '123456' }], }; - const result = await getAnonymousIdFromAttributes(event); + const result = getAnonymousIdFromAttributes(event); expect(result).toEqual('123456'); }); }); + + describe('getCartToken', () => { + it('should return null if cart_token is not present', () => { + const event = {}; + const result = getCartToken(event); + expect(result).toBeNull(); + }); + + it('should return cart_token if it is present', () => { + const event = { cart_token: 'cartTokenTest1' }; + const result = getCartToken(event); + expect(result).toEqual('cartTokenTest1'); + }); + }); +}); + +describe('Redis cart token tests', () => { + it('should get anonymousId property from redis', async () => { + const getValSpy = jest + .spyOn(RedisDB, 'getVal') + .mockResolvedValue({ anonymousId: 'anonymousIdTest1' }); + const event = { + cart_token: `cartTokenTest1`, + id: 5778367414385, + line_items: [ + { + id: 14234727743601, + }, + ], + query_parameters: { + topic: ['orders_updated'], + version: ['pixel'], + writeKey: ['dummy-write-key'], + }, + }; + const message = await processEvent(event); + expect(getValSpy).toHaveBeenCalledTimes(1); + expect(getValSpy).toHaveBeenCalledWith('pixel:cartTokenTest1'); + expect(message.anonymousId).toEqual('anonymousIdTest1'); + }); }); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index df33d7a347a..0d81de99ac3 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -1,7 +1,11 @@ +/* eslint-disable no-param-reassign */ +const get = require('get-value'); const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); +const { extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); const { constructPayload } = require('../../../../v0/util'); -const { lineItemsMappingJSON, productMappingJSON } = require('../config'); - +const { INTEGERATION, lineItemsMappingJSON, productMappingJSON } = require('../config'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); +const stats = require('../../../../util/stats'); /** * Returns an array of products from the lineItems array received from the webhook event * @param {Array} lineItems @@ -54,8 +58,84 @@ const getAnonymousIdFromAttributes = (event) => { return rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; }; +/** + * Returns the cart_token from the event message + * @param {Object} event + * @returns {String} cart_token + */ +const getCartToken = (event) => event?.cart_token || null; + +/** + * Handles the anonymousId assignment for the message, based on the event attributes and redis data + * @param {Object} message rudderstack message object + * @param {Object} event raw shopify event payload + * @param {Object} metricMetadata metric metadata object + */ +const setAnonymousId = async (message, event, metricMetadata) => { + const anonymousId = getAnonymousIdFromAttributes(event); + if (isDefinedAndNotNull(anonymousId)) { + message.anonymousId = anonymousId; + } else { + // if anonymousId is not present in note_attributes or note_attributes is not present, query redis for anonymousId + const cartToken = getCartToken(event); + if (cartToken) { + const redisData = await RedisDB.getVal(`pixel:${cartToken}`); + if (redisData?.anonymousId) { + message.anonymousId = redisData.anonymousId; + } + } else { + stats.increment('shopify_pixel_cart_token_not_found_server_side', { + source: metricMetadata.source, + writeKey: metricMetadata.writeKey, + }); + } + } +}; + +/** + Handles userId, email and contextual properties enrichment for the message payload + * @param {Object} message rudderstack message object + * @param {Object} event raw shopify event payload + * @param {String} shopifyTopic shopify event topic +*/ +const handleCommonProperties = (message, event, shopifyTopic) => { + if (message.userId) { + message.userId = String(message.userId); + } + if (!get(message, 'traits.email')) { + const email = extractEmailFromPayload(event); + if (email) { + message.setProperty('traits.email', email); + } + } + message.setProperty(`integrations.${INTEGERATION}`, true); + message.setProperty('context.library', { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }); + message.setProperty('context.topic', shopifyTopic); + // attaching cart, checkout and order tokens in context object + message.setProperty(`context.cart_token`, event.cart_token); + message.setProperty(`context.checkout_token`, event.checkout_token); + // raw shopify payload passed inside context object under shopifyDetails + message.setProperty('context.shopifyDetails', event); + if (shopifyTopic === 'orders_updated') { + message.setProperty(`context.order_token`, event.token); + } + message.setProperty('integrations.DATA_WAREHOUSE', { + options: { + jsonPaths: [`${message.type}.context.shopifyDetails`], + }, + }); + return message; +}; + module.exports = { createPropertiesForEcomEventFromWebhook, + getCartToken, getProductsFromLineItems, getAnonymousIdFromAttributes, + setAnonymousId, + handleCommonProperties, }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js index b1d1c8b2fac..44928686f83 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js @@ -15,7 +15,9 @@ const { checkoutEventBuilder, checkoutStepEventBuilder, searchEventBuilder, + extractCampaignParams, } = require('./pixelUtils'); +const campaignObjectMappings = require('../pixelEventsMappings/campaignObjectMappings.json'); const { INTEGERATION, PIXEL_EVENT_TOPICS, @@ -68,7 +70,7 @@ const handleCartTokenRedisOperations = async (inputEvent, clientId) => { const cartToken = extractCartToken(inputEvent); try { if (isDefinedNotNullNotEmpty(clientId) && isDefinedNotNullNotEmpty(cartToken)) { - await RedisDB.setVal(cartToken, ['anonymousId', clientId]); + await RedisDB.setVal(`pixel:${cartToken}`, ['anonymousId', clientId]); stats.increment('shopify_pixel_cart_token_set', { event: inputEvent.name, writeKey: inputEvent.query_parameters.writeKey, @@ -85,7 +87,7 @@ const handleCartTokenRedisOperations = async (inputEvent, clientId) => { function processPixelEvent(inputEvent) { // eslint-disable-next-line @typescript-eslint/naming-convention - const { name, query_parameters, clientId, data, id } = inputEvent; + const { name, query_parameters, context, clientId, data, id } = inputEvent; const shopifyDetails = { ...inputEvent }; delete shopifyDetails.context; delete shopifyDetails.query_parameters; @@ -140,6 +142,11 @@ function processPixelEvent(inputEvent) { } message.anonymousId = clientId; message.setProperty(`integrations.${INTEGERATION}`, true); + message.setProperty('integrations.DATA_WAREHOUSE', { + options: { + jsonPaths: [`${message.type}.context.shopifyDetails`], + }, + }); message.setProperty('context.library', { name: 'RudderStack Shopify Cloud', eventOrigin: 'client', @@ -147,12 +154,18 @@ function processPixelEvent(inputEvent) { }); message.setProperty('context.topic', name); message.setProperty('context.shopifyDetails', shopifyDetails); + + // adding campaign object with utm parameters to the message context + const campaignParams = extractCampaignParams(context, campaignObjectMappings); + if (campaignParams) { + message.context.campaign = campaignParams; + } message.messageId = id; message = removeUndefinedAndNullValues(message); return message; } -const processPixelWebEvents = async (event) => { +const processPixelWebEvents = (event) => { const pixelEvent = processPixelEvent(event); return removeUndefinedAndNullValues(pixelEvent); }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js index 46ae59e0cf8..388be2a16e8 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js @@ -1,4 +1,5 @@ /* eslint-disable no-param-reassign */ +const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); const Message = require('../../../../v0/sources/message'); const { EventType } = require('../../../../constants'); const { @@ -203,6 +204,41 @@ const searchEventBuilder = (inputEvent) => { ); }; +/** + * Extracts UTM parameters from the context object + * @param {*} context context object from the event + * @param {*} campaignMappings mappings for UTM parameters + * @returns campaignParams, an object containing UTM parameters + */ +const extractCampaignParams = (context, campaignMappings) => { + if (context?.document?.location?.href) { + const url = new URL(context.document.location.href); + const campaignParams = {}; + + // Loop through mappings and extract UTM parameters + campaignMappings.forEach((mapping) => { + const value = url.searchParams.get(mapping.sourceKeys); + if (isDefinedAndNotNull(value)) { + campaignParams[mapping.destKeys] = value; + } + }); + + // Extract any UTM parameters not in the mappings + const campaignObjectSourceKeys = campaignMappings.flatMap((mapping) => mapping.sourceKeys); + url.searchParams.forEach((value, key) => { + if (key.startsWith('utm_') && !campaignObjectSourceKeys.includes(key)) { + campaignParams[key] = value; + } + }); + + // Only return campaign object if we have any UTM parameters + if (Object.keys(campaignParams).length > 0) { + return campaignParams; + } + } + return null; +}; + module.exports = { pageViewedEventBuilder, cartViewedEventBuilder, @@ -212,4 +248,5 @@ module.exports = { checkoutEventBuilder, checkoutStepEventBuilder, searchEventBuilder, + extractCampaignParams, }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js index e8f53a5f153..fc82404ee73 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js @@ -7,7 +7,9 @@ const { checkoutEventBuilder, checkoutStepEventBuilder, searchEventBuilder, + extractCampaignParams, } = require('./pixelUtils'); +const campaignObjectMappings = require('../pixelEventsMappings/campaignObjectMappings.json'); const Message = require('../../../../v0/sources/message'); jest.mock('ioredis', () => require('../../../../test/__mocks__/redis')); jest.mock('../../../../v0/sources/message'); @@ -787,4 +789,59 @@ describe('utilV2.js', () => { expect(message.context).toEqual({ userAgent: 'Mozilla/5.0' }); }); }); + + describe('extractCampaignParams', () => { + it('should extract campaign parameters from URL', () => { + const context = { + document: { + location: { + href: 'https://example.com?utm_source=google&utm_medium=cpc&utm_campaign=spring_sale', + }, + }, + }; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toEqual({ + utm_source: 'google', + medium: 'cpc', + name: 'spring_sale', + }); + }); + + it('should return null if no campaign parameters are found', () => { + const context = { + document: { + location: { + href: 'https://example.com', + }, + }, + }; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toBeNull(); + }); + + it('should extract additional UTM parameters not in mappings', () => { + const context = { + document: { + location: { + href: 'https://example.com?utm_source=google&utm_term=shoes', + }, + }, + }; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toEqual({ + utm_source: 'google', + term: 'shoes', + }); + }); + + it('should handle missing context or location gracefully', () => { + const context = {}; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toBeNull(); + }); + }); }); diff --git a/src/warehouse/index.js b/src/warehouse/index.js index ea663c9b2fc..c5f167909f7 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -11,6 +11,7 @@ const { validTimestamp, getVersionedUtils, isRudderSourcesEvent, + mergeJSONPathsFromDataWarehouse, } = require('./util'); const { getMergeRuleEvent } = require('./identity'); @@ -307,8 +308,8 @@ function isStringLikeObject(obj) { let minKey = Infinity; let maxKey = -Infinity; - for (let i = 0; i < keys.length; i++) { - const key = keys[i]; + for (const element of keys) { + const key = element; const value = obj[key]; if (!isNonNegativeInteger(key)) return false; @@ -336,8 +337,8 @@ function stringLikeObjectToString(obj) { .sort((a, b) => a - b); let result = ''; - for (let i = 0; i < keys.length; i++) { - result += obj[keys[i].toString()]; + for (const element of keys) { + result += obj[element.toString()]; } return result; @@ -655,6 +656,8 @@ function processWarehouseMessage(message, options) { const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; + mergeJSONPathsFromDataWarehouse(message, options); + // underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_" // underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. diff --git a/src/warehouse/util.js b/src/warehouse/util.js index 7f4e224a349..1f2d9215f2c 100644 --- a/src/warehouse/util.js +++ b/src/warehouse/util.js @@ -136,6 +136,27 @@ const getRecordIDForExtract = (message) => { return recordId; }; +function mergeJSONPathsFromDataWarehouse(message, options) { + const dataWarehouseOptions = message.integrations?.['DATA_WAREHOUSE']?.options; + if (!dataWarehouseOptions?.jsonPaths) return; + + const dataWarehouseJSONPaths = Array.isArray(dataWarehouseOptions.jsonPaths) + ? dataWarehouseOptions.jsonPaths + : []; + const currentJSONPaths = Array.isArray(options.integrationOptions?.jsonPaths) + ? options.integrationOptions.jsonPaths + : []; + + switch (options.provider) { + case 'rs': + case 'postgres': + case 'snowflake': + case 'bq': + options.integrationOptions.jsonPaths = [...dataWarehouseJSONPaths, ...currentJSONPaths]; + break; + } +} + module.exports = { isObject, isValidJsonPathKey, @@ -148,4 +169,5 @@ module.exports = { sourceCategoriesToUseRecordId, getCloudRecordID, getRecordIDForExtract, + mergeJSONPathsFromDataWarehouse, }; diff --git a/test/__tests__/shopify_warehouse.test.js b/test/__tests__/shopify_warehouse.test.js new file mode 100644 index 00000000000..080022680bb --- /dev/null +++ b/test/__tests__/shopify_warehouse.test.js @@ -0,0 +1,103 @@ +const event = { + "request": { + "query": { + "whSchemaVersion": "v1" + } + }, + "message": { + "context": { + "shopifyDetails": { + "id": 5778367414385, + "current_total_tax": "10.00", + "current_total_tax_set": { + "shop_money": { + "amount": "10.00", + "currency_code": "USD" + }, + }, + "name": "#1017", + "phone": null, + } + }, + "integrations": { + "SHOPIFY": true, + "DATA_WAREHOUSE": { + "options": { + "jsonPaths": [ + "track.context.shopifyDetails" + ] + } + } + }, + "type": "track", + "event": "Order Updated", + "properties": { + "order_id": "5778367414385", + "currency": "USD", + "products": [ + { + "product_id": "7234590408817", + "price": 600, + "quantity": 1 + } + ] + }, + "userId": "123321", + "traits": {}, + "timestamp": "2024-01-01T01:23:45.678Z", + }, + "destination": { + "Config": {}, + } +}; + +/* + Test for warehouse agnostic DATA_WAREHOUSE JSON column support for Shopify source +*/ +describe('DATA_WAREHOUSE integrations', () => { + it('should process event and return responses for common providers for agnostic support', () => { + const responses = require('../../src/v0/destinations/snowflake/transform').process(event); + expect(responses).toHaveLength(2); + expect(responses[0].metadata.table).toBe('TRACKS'); + expect(responses[1].metadata.table).toBe('ORDER_UPDATED'); + + expect(responses[0].metadata.columns.CONTEXT_SHOPIFY_DETAILS).toBe('json'); + expect(responses[0].data.CONTEXT_SHOPIFY_DETAILS).toBe('{"id":5778367414385,"current_total_tax":"10.00","current_total_tax_set":{"shop_money":{"amount":"10.00","currency_code":"USD"}},"name":"#1017","phone":null}'); + + expect(responses[1].metadata.columns.CONTEXT_SHOPIFY_DETAILS).toBe('json'); + expect(responses[1].data.CONTEXT_SHOPIFY_DETAILS).toBe('{"id":5778367414385,"current_total_tax":"10.00","current_total_tax_set":{"shop_money":{"amount":"10.00","currency_code":"USD"}},"name":"#1017","phone":null}'); + }); + + it('should process event and return response for other providers like mssql', () => { + const responses = require('../../src/v0/destinations/mssql/transform').process(event); + expect(responses).toHaveLength(2); + expect(responses[0].metadata.table).toBe('tracks'); + expect(responses[1].metadata.table).toBe('order_updated'); + + expect(responses[0].metadata.columns.context_shopify_details).toBe(undefined); + expect(responses[0].metadata.columns.context_shopify_details_id).toBe('int'); + expect(responses[0].metadata.columns.context_shopify_details_current_total_tax).toBe('string'); + expect(responses[0].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('string'); + expect(responses[0].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('string'); + expect(responses[0].metadata.columns.context_shopify_details_name).toBe('string'); + expect(responses[0].data.context_shopify_details).toBe(undefined); + expect(responses[0].data.context_shopify_details_id).toBe(5778367414385); + expect(responses[0].data.context_shopify_details_current_total_tax).toBe('10.00'); + expect(responses[0].data.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('10.00'); + expect(responses[0].data.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('USD'); + expect(responses[0].data.context_shopify_details_name).toBe('#1017'); + + expect(responses[1].metadata.columns.context_shopify_details).toBe(undefined); + expect(responses[1].metadata.columns.context_shopify_details_id).toBe('int'); + expect(responses[1].metadata.columns.context_shopify_details_current_total_tax).toBe('string'); + expect(responses[1].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('string'); + expect(responses[1].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('string'); + expect(responses[1].metadata.columns.context_shopify_details_name).toBe('string'); + expect(responses[1].data.context_shopify_details).toBe(undefined); + expect(responses[1].data.context_shopify_details_id).toBe(5778367414385); + expect(responses[1].data.context_shopify_details_current_total_tax).toBe('10.00'); + expect(responses[1].data.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('10.00'); + expect(responses[1].data.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('USD'); + expect(responses[1].data.context_shopify_details_name).toBe('#1017'); + }); +}); \ No newline at end of file diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts index 040843d8110..a0794f4ce02 100644 --- a/test/integrations/component.test.ts +++ b/test/integrations/component.test.ts @@ -80,6 +80,7 @@ beforeAll(async () => { }); afterAll(async () => { + await createHttpTerminator({ server }).terminate(); if (opts.generate === 'true') { const callsDataStr = responses.join('\n'); const calls = ` @@ -89,7 +90,6 @@ afterAll(async () => { `; appendFileSync(join(__dirname, 'destinations', opts.destination, 'network.ts'), calls); } - await createHttpTerminator({ server }).terminate(); }); let mockAdapter; if (!opts.generate || opts.generate === 'false') { @@ -234,27 +234,31 @@ describe.each(allTestDataFilePaths)('%s Tests', (testDataPath) => { }); } - const extendedTestData: ExtendedTestCaseData[] = testData.flatMap((tcData) => { - if (tcData.module === tags.MODULES.SOURCE) { - return [ - { - tcData, - sourceTransformV2Flag: false, - descriptionSuffix: ' (sourceTransformV2Flag: false)', - }, - { - tcData, - sourceTransformV2Flag: true, - descriptionSuffix: ' (sourceTransformV2Flag: true)', - }, - ]; - } - return [{ tcData }]; - }); + const extendedTestData: ExtendedTestCaseData[] = testData.flatMap((tcData) => + tcData.module === tags.MODULES.SOURCE + ? [ + { + tcData, + sourceTransformV2Flag: false, + descriptionSuffix: ' (sourceTransformV2Flag: false)', + }, + { + tcData, + sourceTransformV2Flag: true, + descriptionSuffix: ' (sourceTransformV2Flag: true)', + }, + ] + : [ + { + tcData, + descriptionSuffix: '', + }, + ], + ); describe(`${testData[0].name} ${testData[0].module}`, () => { test.each(extendedTestData)( - '$feature -> $description$descriptionSuffix (index: $#)', + '$tcData.feature -> $tcData.description$descriptionSuffix (index: $#)', async ({ tcData, sourceTransformV2Flag }) => { tcData?.mockFns?.(mockAdapter); diff --git a/test/integrations/destinations/airship/processor/business.ts b/test/integrations/destinations/airship/processor/business.ts new file mode 100644 index 00000000000..0fbf140720c --- /dev/null +++ b/test/integrations/destinations/airship/processor/business.ts @@ -0,0 +1,232 @@ +const arrayHandlingCases = [ + { + description: + '[identify] should send array traits as is to airship when present in integrationsObject(even when similar key is present in traits)', + inputEvent: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { + email: 'testone@gmail.com', + firstName: 'test', + lastName: 'one', + colors: ['red', 'blue'], + }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + originalTimestamp: '2019-10-14T09:03:17.562Z', + anonymousId: '123456', + userId: 'testuserId1', + integrations: { + All: true, + Airship: { + JSONAttributes: { + 'colors#r012': { + colors: ['green', 'yellow'], + }, + }, + }, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + expectedOutputResponse: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/named_users/testuserId1/attributes', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + attributes: [ + { + action: 'set', + key: 'email', + value: 'testone@gmail.com', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'first_name', + value: 'test', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'last_name', + value: 'one', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'colors#r012', + value: { + colors: ['green', 'yellow'], + }, + timestamp: '2019-10-14T09:03:17Z', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + // not expected in reality & leads to error in airship, but just to test the delimiter handling + { + description: '[identify] should handle keys with delimiters and JSON attributes correctly', + inputEvent: { + channel: 'web', + context: { + traits: { + preferences: ['value1'], // should be processed as preferences_0 + 'settings.theme': 'dark', + 'data[test]_value': 'test', + simple: 'value', // no delimiters + 'company[location]': 'SF', // should be processed since not in JSONAttributes + }, + }, + type: 'identify', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + originalTimestamp: '2019-10-14T09:03:17.562Z', + userId: 'testuserId1', + integrations: { + All: true, + Airship: { + JSONAttributes: { + 'company#pow2': { + name: 'Test Corp', + size: 100, + }, + }, + }, + }, + sentAt: '2019-10-14T09:03:22.563Z', + }, + expectedOutputResponse: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/named_users/testuserId1/attributes', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + attributes: [ + { + action: 'set', + key: 'preferences[0]', + value: 'value1', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'settings_theme', + value: 'dark', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'data[test]_value', + value: 'test', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'simple', + value: 'value', + timestamp: '2019-10-14T09:03:17Z', + }, + { + action: 'set', + key: 'company#pow2', + value: { + name: 'Test Corp', + size: 100, + }, + timestamp: '2019-10-14T09:03:17Z', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, +]; + +const getIdentifyTestCase = ({ description, inputEvent, expectedOutputResponse }) => { + return { + name: 'airship', + description: description, + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: inputEvent, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'O2YARRI15I', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: expectedOutputResponse, + }, + }; +}; + +export const identifyTestCases = arrayHandlingCases.map((tc) => getIdentifyTestCase(tc)); diff --git a/test/integrations/destinations/airship/processor/data.ts b/test/integrations/destinations/airship/processor/data.ts index 973a6be0ecc..cd1a7fac9b7 100644 --- a/test/integrations/destinations/airship/processor/data.ts +++ b/test/integrations/destinations/airship/processor/data.ts @@ -1,4 +1,7 @@ +import { identifyTestCases } from './business'; + export const data = [ + ...identifyTestCases, { name: 'airship', description: 'Test 0', diff --git a/test/integrations/destinations/bluecore/identifyTestData.ts b/test/integrations/destinations/bluecore/identifyTestData.ts index fee27ccf0fa..dbc6c3967c3 100644 --- a/test/integrations/destinations/bluecore/identifyTestData.ts +++ b/test/integrations/destinations/bluecore/identifyTestData.ts @@ -237,7 +237,7 @@ export const identifyData = [ body: [ { error: - "[Bluecore] traits.action must be 'identify' for identify action: Workflow: procWorkflow, Step: prepareIdentifyPayload, ChildStep: undefined, OriginalError: [Bluecore] traits.action must be 'identify' for identify action", + "[Bluecore] traits.action must be 'identify' for identify action: Workflow: procWorkflow, Step: prepareIdentifyPayload, ChildStep: undefined, OriginalError: [Bluecore] traits.action must be 'identify' for identify action", metadata: { destinationId: '', destinationType: '', diff --git a/test/integrations/destinations/customerio_audience/common.ts b/test/integrations/destinations/customerio_audience/common.ts new file mode 100644 index 00000000000..91723cc12c6 --- /dev/null +++ b/test/integrations/destinations/customerio_audience/common.ts @@ -0,0 +1,97 @@ +import { Connection, Destination } from '../../../../src/types'; +import { VDM_V2_SCHEMA_VERSION } from '../../../../src/v0/util/constant'; + +const destType = 'customerio_audience'; +const destTypeInUpperCase = 'CUSTOMERIO_AUDIENCE'; +const displayName = 'Customer.io Audience'; +const channel = 'web'; +const destination: Destination = { + Config: { + apiKey: 'test-api-key', + appApiKey: 'test-app-api-key', + connectionMode: 'cloud', + siteId: 'test-site-id', + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: {}, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', +}; +const connection: Connection = { + sourceId: 'dummy-source-id', + destinationId: 'dummy-destination-id', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + audienceId: 'test-segment-id', + identifierMappings: [ + { + from: 'some-key', + to: 'id', + }, + ], + }, + }, +}; + +const inValidConnection: Connection = { + ...connection, + config: { + ...connection.config, + destination: { + audienceId: '', + }, + }, +}; + +const insertOrUpdateEndpoint = + 'https://track.customer.io/api/v1/segments/test-segment-id/add_customers'; + +const deleteEndpoint = 'https://track.customer.io/api/v1/segments/test-segment-id/remove_customers'; + +const processorInstrumentationErrorStatTags = { + destType: destTypeInUpperCase, + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + destinationId: 'default-destinationId', + workspaceId: 'default-workspaceId', +}; + +const RouterInstrumentationErrorStatTags = { + ...processorInstrumentationErrorStatTags, + feature: 'router', +}; + +const headers = { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC1zaXRlLWlkOnRlc3QtYXBpLWtleQ==', +}; + +const params = { + id_type: 'id', +}; + +export { + destType, + channel, + destination, + connection, + inValidConnection, + processorInstrumentationErrorStatTags, + RouterInstrumentationErrorStatTags, + headers, + params, + insertOrUpdateEndpoint, + deleteEndpoint, +}; diff --git a/test/integrations/destinations/customerio_audience/mocks.ts b/test/integrations/destinations/customerio_audience/mocks.ts new file mode 100644 index 00000000000..2c613706e67 --- /dev/null +++ b/test/integrations/destinations/customerio_audience/mocks.ts @@ -0,0 +1,5 @@ +import * as config from '../../../../src/v0/destinations/customerio_audience/config'; + +export const defaultMockFns = () => { + jest.replaceProperty(config, 'MAX_ITEMS', 3 as typeof config.MAX_ITEMS); +}; diff --git a/test/integrations/destinations/customerio_audience/router/data.ts b/test/integrations/destinations/customerio_audience/router/data.ts new file mode 100644 index 00000000000..ed486f781b6 --- /dev/null +++ b/test/integrations/destinations/customerio_audience/router/data.ts @@ -0,0 +1,369 @@ +import { generateMetadata, generateRecordPayload, overrideDestination } from '../../../testUtils'; +import { defaultMockFns } from '../mocks'; +import { + destType, + destination, + headers, + RouterInstrumentationErrorStatTags, + insertOrUpdateEndpoint, + deleteEndpoint, + connection, + params, + inValidConnection, +} from '../common'; + +const routerRequest1 = { + input: [ + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-1', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-2', + }, + action: 'insert', + }), + metadata: generateMetadata(2), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-3', + }, + action: 'insert', + }), + metadata: generateMetadata(3), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-4', + }, + action: 'insert', + }), + metadata: generateMetadata(4), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-5', + }, + action: 'update', + }), + metadata: generateMetadata(5), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-6', + }, + action: 'delete', + }), + metadata: generateMetadata(6), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-7', + }, + action: 'delete', + }), + metadata: generateMetadata(7), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-8', + }, + action: 'dummy-action', + }), + metadata: generateMetadata(8), + destination, + connection, + }, + { + message: { + type: 'identify', + anonymousId: 'anonId1', + userId: 'userId1', + integrations: { + All: true, + }, + originalTimestamp: '2024-03-04T15:32:56.409Z', + }, + metadata: generateMetadata(9), + destination, + connection, + }, + { + message: generateRecordPayload({ + action: 'insert', + }), + metadata: generateMetadata(10), + destination, + connection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-7', + email: 'test@gmail.com', + }, + action: 'insert', + }), + metadata: generateMetadata(11), + destination, + connection, + }, + ], + destType, +}; + +// scenario when all the events are malfunctioned +const routerRequest2 = { + input: [ + { + message: generateRecordPayload({ + identifiers: { + id: [], + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination, + connection: inValidConnection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-1', + }, + action: 'insert', + }), + metadata: generateMetadata(2), + destination, + connection: inValidConnection, + }, + { + message: generateRecordPayload({ + identifiers: { + id: 'test-id-1', + }, + action: 'insert', + }), + metadata: generateMetadata(3), + destination, + connection: { + ...connection, + config: { + ...connection.config, + destination: { + audienceId: 'test-audience-id', + }, + }, + }, + }, + ], + destType, +}; + +export const data = [ + { + id: 'customerio-segment-router-test-1', + name: destType, + description: 'Basic Router Test to test record payloads', + scenario: 'Framework+Business', + successCriteria: 'All events should be transformed successfully and status code should be 200', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest1, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: insertOrUpdateEndpoint, + headers, + params, + body: { + JSON: { + ids: ['test-id-1', 'test-id-2', 'test-id-3'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(1), generateMetadata(2), generateMetadata(3)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: insertOrUpdateEndpoint, + headers, + params, + body: { + JSON: { + ids: ['test-id-4', 'test-id-5'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(4), generateMetadata(5)], + batched: true, + statusCode: 200, + destination, + }, + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: deleteEndpoint, + headers, + params, + body: { + JSON: { + ids: ['test-id-6', 'test-id-7'], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [generateMetadata(6), generateMetadata(7)], + batched: true, + statusCode: 200, + destination, + }, + { + metadata: [generateMetadata(8)], + batched: false, + statusCode: 400, + error: 'action dummy-action is not supported', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(9)], + batched: false, + statusCode: 400, + error: 'message type identify is not supported', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(10)], + batched: false, + statusCode: 400, + error: 'identifiers cannot be empty', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(11)], + batched: false, + statusCode: 400, + error: 'only one identifier is supported', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, + { + id: 'customerio-segment-router-test-2', + name: destType, + description: 'Basic Router Test to test record payloads', + scenario: 'Framework', + successCriteria: 'All events should throw error', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: routerRequest2, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + metadata: [generateMetadata(1)], + batched: false, + statusCode: 400, + error: 'identifier type should be a string or integer', + statTags: { ...RouterInstrumentationErrorStatTags, errorType: 'configuration' }, + destination, + }, + { + metadata: [generateMetadata(2)], + batched: false, + statusCode: 400, + error: 'audienceId is required, aborting.', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + { + metadata: [generateMetadata(3)], + batched: false, + statusCode: 400, + error: 'identifierMappings cannot be empty', + statTags: RouterInstrumentationErrorStatTags, + destination, + }, + ], + }, + }, + }, + mockFns: defaultMockFns, + }, +]; diff --git a/test/integrations/destinations/fb_custom_audience/router/data.ts b/test/integrations/destinations/fb_custom_audience/router/data.ts index 834b6315f6a..5946c7d6b8c 100644 --- a/test/integrations/destinations/fb_custom_audience/router/data.ts +++ b/test/integrations/destinations/fb_custom_audience/router/data.ts @@ -849,9 +849,10 @@ export const data = [ }, { name: 'fb_custom_audience', - description: 'rETL record V2 tests', + description: 'rETL record V2 tests with null values', scenario: 'Framework', - successCriteria: 'all record events should be transformed correctly based on their operation', + successCriteria: + 'all record events should be transformed correctly including records with null values', feature: 'router', module: 'destination', version: 'v0', @@ -890,6 +891,7 @@ export const data = [ 'b100c2ec0718fe6b4805b623aeec6710719d042ceea55f5c8135b010ec1c7b36', '1e14a2f476f7611a8b22bc85d14237fdc88aac828737e739416c32c5bce3bd16', ], + ['', ''], ], }, }, @@ -939,6 +941,18 @@ export const data = [ userId: 'default-userId', workspaceId: 'default-workspaceId', }, + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 4, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, ], batched: true, statusCode: 200, diff --git a/test/integrations/destinations/fb_custom_audience/router/rETL.ts b/test/integrations/destinations/fb_custom_audience/router/rETL.ts index f8d5fc89a03..5182b5380df 100644 --- a/test/integrations/destinations/fb_custom_audience/router/rETL.ts +++ b/test/integrations/destinations/fb_custom_audience/router/rETL.ts @@ -116,8 +116,8 @@ export const rETLRecordV2RouterRequest: RouterTransformationRequest = { version: '895/merge', }, }, - recordId: '2', - rudderId: '2', + recordId: '3', + rudderId: '3', identifiers: { EMAIL: 'subscribed@eewrfrd.com', FI: 'ghui', @@ -126,6 +126,29 @@ export const rETLRecordV2RouterRequest: RouterTransformationRequest = { }, metadata: generateMetadata(3), }, + { + destination: destinationV2, + connection: connection, + message: { + action: 'insert', + context: { + sources: { + job_run_id: 'cgiiurt8um7k7n5dq480', + task_run_id: 'cgiiurt8um7k7n5dq48g', + job_id: '2MUWghI7u85n91dd1qzGyswpZan', + version: '895/merge', + }, + }, + recordId: '4', + rudderId: '4', + identifiers: { + EMAIL: null, + FI: null, + }, + type: 'record', + }, + metadata: generateMetadata(4), + }, ], destType: 'fb_custom_audience', }; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts index 87aafea0afa..08abb008cf5 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/business.ts @@ -235,7 +235,7 @@ export const testScenariosForV0API = [ params: params.param1, JSON: invalidArgumentRequestPayload, endpoint: - 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/11122233331/offlineUserDataJobs', }), method: 'POST', }, @@ -309,7 +309,7 @@ export const testScenariosForV0API = [ headers: headers.header1, params: params.param1, JSON: validRequestPayload1, - endpoint: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', }), method: 'POST', }, @@ -350,7 +350,7 @@ export const testScenariosForV0API = [ params: params.param2, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/1234567891:uploadClickConversions', }), method: 'POST', }, @@ -400,7 +400,7 @@ export const testScenariosForV0API = [ params: params.param3, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/1234567891:uploadClickConversions', }), method: 'POST', }, @@ -453,7 +453,7 @@ export const testScenariosForV1API = [ params: params.param1, JSON: invalidArgumentRequestPayload, endpoint: - 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/11122233331/offlineUserDataJobs', }, metadataArray, ), @@ -500,7 +500,7 @@ export const testScenariosForV1API = [ params: params.param1, JSON: validRequestPayload1, endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', }, metadataArray, ), @@ -545,7 +545,7 @@ export const testScenariosForV1API = [ params: params.param2, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/1234567891:uploadClickConversions', }, metadataArray, ), @@ -591,7 +591,7 @@ export const testScenariosForV1API = [ params: params.param3, JSON: validRequestPayload2, endpoint: - 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/1234567891:uploadClickConversions', }, metadataArray, ), @@ -637,7 +637,7 @@ export const testScenariosForV1API = [ params: params.param4, JSON: notAllowedToAccessFeatureRequestPayload, endpoint: - 'https://googleads.googleapis.com/v16/customers/1234567893:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/1234567893:uploadClickConversions', }, metadataArray, ), diff --git a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts index 4437ebb912c..2a79a5e3112 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/dataDelivery/oauth.ts @@ -95,7 +95,7 @@ export const v0oauthScenarios = [ request: { body: generateProxyV0Payload({ ...commonRequestParameters, - endpoint: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v17/customers/customerid/offlineUserDataJobs', }), method: 'POST', }, @@ -138,7 +138,7 @@ export const v0oauthScenarios = [ request: { body: generateProxyV0Payload({ ...commonRequestParameters, - endpoint: 'https://googleads.googleapis.com/v16/customers/1234/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v17/customers/1234/offlineUserDataJobs', }), method: 'POST', }, @@ -184,7 +184,7 @@ export const v1oauthScenarios = [ { ...commonRequestParameters, endpoint: - 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/customerid/offlineUserDataJobs', }, metadataArray, ), @@ -230,7 +230,7 @@ export const v1oauthScenarios = [ body: generateProxyV1Payload( { ...commonRequestParameters, - endpoint: 'https://googleads.googleapis.com/v16/customers/1234/offlineUserDataJobs', + endpoint: 'https://googleads.googleapis.com/v17/customers/1234/offlineUserDataJobs', }, metadataArray, ), @@ -282,7 +282,7 @@ export const v1oauthScenarios = [ 'login-customer-id': 'logincustomerid', }, endpoint: - 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/customerid/offlineUserDataJobs', }, metadataArray, ), @@ -354,7 +354,7 @@ export const v1oauthScenarios = [ 'login-customer-id': 'logincustomerid', }, endpoint: - 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/customerid/offlineUserDataJobs', }, metadataArray, ), diff --git a/test/integrations/destinations/google_adwords_offline_conversions/network.ts b/test/integrations/destinations/google_adwords_offline_conversions/network.ts index 0ab6bef1db7..66856933d99 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/network.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/network.ts @@ -30,7 +30,7 @@ const commonResponse = { export const networkCallsData = [ { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v17/customers/11122233331/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -59,7 +59,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1112223333/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v17/customers/1112223333/googleAds:searchStream', data: { query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'`, }, @@ -92,7 +92,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations', + url: 'https://googleads.googleapis.com/v17/customers/11122233331/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID_FOR_ADD_FAILURE:addOperations', data: { enable_partial_failure: false, enable_warnings: false, @@ -173,7 +173,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -202,7 +202,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations', + url: 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:addOperations', data: { enable_partial_failure: false, enable_warnings: false, @@ -245,7 +245,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run', + url: 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs/OFFLINE_USER_DATA_JOB_ID:run', data: { validate_only: false }, params: { destination: 'google_adwords_offline_conversion' }, headers: { @@ -267,7 +267,7 @@ export const networkCallsData = [ description: 'Mock response from destination depicting a request with invalid authentication credentials', httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v17/customers/customerid/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -303,7 +303,7 @@ export const networkCallsData = [ description: 'Mock response from destination depicting a request with invalid authentication scopes', httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v17/customers/1234/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -336,7 +336,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567890/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v17/customers/1234567890/googleAds:searchStream', data: { query: `SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'`, }, @@ -364,7 +364,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567891/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v17/customers/1234567891/googleAds:searchStream', data: { query: "SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'", @@ -397,7 +397,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567891/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v17/customers/1234567891/googleAds:searchStream', data: { query: 'SELECT conversion_custom_variable.name FROM conversion_custom_variable' }, headers: { Authorization: 'Bearer abcd1234', @@ -431,7 +431,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', + url: 'https://googleads.googleapis.com/v17/customers/1234567891:uploadClickConversions', data: { conversions: [ { @@ -498,7 +498,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567891:uploadClickConversions', + url: 'https://googleads.googleapis.com/v17/customers/1234567891:uploadClickConversions', data: { conversions: [ { @@ -559,7 +559,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567893/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v17/customers/1234567893/googleAds:searchStream', data: { query: "SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'", @@ -592,7 +592,7 @@ export const networkCallsData = [ }, { httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1234567893:uploadClickConversions', + url: 'https://googleads.googleapis.com/v17/customers/1234567893:uploadClickConversions', data: { conversions: [ { @@ -678,7 +678,7 @@ export const networkCallsData = [ description: 'Mock response from destination depicting a request from user who has not enabled 2 factor authentication', httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/customerid/offlineUserDataJobs:create', + url: 'https://googleads.googleapis.com/v17/customers/customerid/offlineUserDataJobs:create', data: { job: { storeSalesMetadata: { @@ -704,7 +704,7 @@ export const networkCallsData = [ description: 'Mock response from destination depicting a request from user who has not enabled 2 factor authentication', httpReq: { - url: 'https://googleads.googleapis.com/v16/customers/1112223333/googleAds:searchStream', + url: 'https://googleads.googleapis.com/v17/customers/1112223333/googleAds:searchStream', data: { query: "SELECT conversion_action.id FROM conversion_action WHERE conversion_action.name = 'Sign-up - click'", diff --git a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts index 3ecae2b92dd..82ea4fcfcee 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/processor/data.ts @@ -176,7 +176,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -467,7 +467,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -758,7 +758,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -1049,7 +1049,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2023,7 +2023,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2129,7 +2129,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2353,7 +2353,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2553,7 +2553,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -2789,7 +2789,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -3012,7 +3012,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -3520,7 +3520,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -3763,7 +3763,7 @@ export const data = [ XML: {}, }, endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', files: {}, headers: { Authorization: 'Bearer abcd1234', @@ -3937,7 +3937,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4129,7 +4129,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4466,7 +4466,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4665,7 +4665,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -4867,7 +4867,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5031,7 +5031,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5191,7 +5191,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5349,7 +5349,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5504,7 +5504,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5665,7 +5665,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1112223333/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1112223333/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -5929,4 +5929,112 @@ export const data = [ }, mockFns: timestampMock, }, + { + name: 'google_adwords_offline_conversions', + description: 'Test 29 : store conversion which has value less than 0, should throw error', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + traits: {}, + }, + event: 'Product Clicked', + type: 'track', + messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', + anonymousId: '00000000000000000000000000', + userId: '12345', + properties: { + item_id: 'item id', + merchant_id: 'merchant id', + currency: 'INR', + revenue: '0', + store_code: 'store code', + gclid: 'gclid', + conversionDateTime: '2019-10-14T11:15:18.299Z', + product_id: '123445', + quantity: 123, + }, + integrations: { + google_adwords_offline_conversion: { + consent: { + adUserdata: 'UNSPECIFIED', + adPersonalization: 'GRANTED', + }, + }, + }, + name: 'ApplicationLoaded', + sentAt: '2019-10-14T11:15:53.296Z', + }, + metadata: { + secret: { + access_token: 'abcd1234', + refresh_token: 'efgh5678', + developer_token: 'ijkl91011', + }, + }, + destination: { + Config: { + isCustomerAllowed: false, + customerId: '111-222-3333', + subAccount: true, + loginCustomerId: 'login-customer-id', + userDataConsent: 'GRANTED', + personalizationConsent: 'DENIED', + eventsToOfflineConversionsTypeMapping: [ + { + from: 'Product Clicked', + to: 'store', + }, + ], + eventsToConversionsNamesMapping: [ + { + from: 'Product Clicked', + to: 'Sign-up - click', + }, + ], + hashUserIdentifier: true, + defaultUserIdentifier: 'phone', + validateOnly: false, + rudderAccountId: '2EOknn1JNH7WK1MfNkgr4t3u4fGYKkRK', + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + "The value '0' does not match the regex pattern, ^([1-9]\\d*(\\.\\d+)?|0\\.\\d+)$", + metadata: { + secret: { + access_token: 'abcd1234', + developer_token: 'ijkl91011', + refresh_token: 'efgh5678', + }, + }, + statTags: { + destType: 'GOOGLE_ADWORDS_OFFLINE_CONVERSIONS', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + }, + statusCode: 400, + }, + ], + }, + }, + mockFns: timestampMock, + }, ]; diff --git a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts index bcc718485b3..bb0b4c6c44b 100644 --- a/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_offline_conversions/router/data.ts @@ -484,7 +484,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/7693729833/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/7693729833/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -566,7 +566,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/7693729833:uploadCallConversions', + 'https://googleads.googleapis.com/v17/customers/7693729833:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -681,7 +681,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadClickConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadClickConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -816,7 +816,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/9625812972:uploadCallConversions', + 'https://googleads.googleapis.com/v17/customers/9625812972:uploadCallConversions', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', @@ -1170,7 +1170,7 @@ export const data = [ type: 'REST', method: 'POST', endpoint: - 'https://googleads.googleapis.com/v16/customers/1234556775/offlineUserDataJobs', + 'https://googleads.googleapis.com/v17/customers/1234556775/offlineUserDataJobs', headers: { Authorization: 'Bearer abcd1234', 'Content-Type': 'application/json', diff --git a/test/integrations/destinations/http/common.ts b/test/integrations/destinations/http/common.ts index f0c8bc8a337..6c2887859f3 100644 --- a/test/integrations/destinations/http/common.ts +++ b/test/integrations/destinations/http/common.ts @@ -92,13 +92,18 @@ const destinations: Destination[] = [ }, { Config: { - apiUrl: 'http://abc.com/contacts/{{$.traits.email}}/', + apiUrl: 'http://abc.com/contacts/', auth: 'apiKeyAuth', apiKeyName: 'x-api-key', apiKeyValue: 'test-api-key', method: 'DELETE', isBatchingEnabled: true, maxBatchSize: 4, + pathParams: [ + { + path: '$.traits.email', + }, + ], }, DestinationDefinition: { DisplayName: displayName, @@ -114,13 +119,18 @@ const destinations: Destination[] = [ }, { Config: { - apiUrl: 'http://abc.com/contacts/{{$.traits.email}}/', + apiUrl: 'http://abc.com/contacts/', auth: 'apiKeyAuth', apiKeyName: 'x-api-key', apiKeyValue: 'test-api-key', method: 'GET', isBatchingEnabled: true, maxBatchSize: 4, + pathParams: [ + { + path: '$.traits.email', + }, + ], }, DestinationDefinition: { DisplayName: displayName, @@ -141,6 +151,7 @@ const destinations: Destination[] = [ bearerToken: 'test-token', method: 'POST', format: 'XML', + xmlRootKey: 'body', headers: [ { to: '$.h1', @@ -249,7 +260,7 @@ const destinations: Destination[] = [ }, { Config: { - apiUrl: 'http://abc.com/contacts/{{$.traits.phone}}', + apiUrl: 'http://abc.com/contacts/', auth: 'noAuth', method: 'POST', format: 'JSON', @@ -262,7 +273,341 @@ const destinations: Destination[] = [ }, { to: '$.key', - from: '.traits.key', + from: '$.traits.key', + }, + ], + pathParams: [ + { + path: '$.traits.phone', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/contacts', + auth: 'basicAuth', + username: 'test-user', + password: '', + method: 'GET', + format: 'JSON', + isBatchingEnabled: true, + maxBatchSize: 2, + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '2', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + { + to: '$.h3', + from: '$.traits.firstName', + }, + ], + queryParams: [ + { + to: "$['q1']", + from: "'val1'", + }, + { + to: '$.q2', + from: '$.traits.email', + }, + ], + pathParams: [ + { + path: '$.userId', + }, + { + path: 'c1', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/contacts', + auth: 'basicAuth', + username: 'test-user', + password: '', + method: 'GET', + format: 'JSON', + isBatchingEnabled: true, + maxBatchSize: 2, + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '2', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + { + to: '$.h3', + from: '$.traits.firstName', + }, + ], + queryParams: [ + { + to: 'user name', + from: "'val1'", + }, + { + to: '$.q2', + from: '$.traits.email', + }, + ], + pathParams: [ + { + path: '$.userId', + }, + { + path: 'c1', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'XML', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + ], + propertiesMapping: [ + { + from: '$.properties', + to: '$', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'FORM', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + ], + propertiesMapping: [ + { + from: '$.event', + to: '$.event', + }, + { + from: '$.properties.currency', + to: '$.currency', + }, + { + from: '$.userId', + to: '$.userId', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'FORM', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + ], + propertiesMapping: [ + { + from: '$.event', + to: '$.event', + }, + { + from: '$.properties.currency', + to: '$.currency', + }, + { + from: '$.userId', + to: '$.userId', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'FORM', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + ], + propertiesMapping: [ + { + from: '$.event', + to: '$.event', + }, + { + from: '$.properties.currency', + to: '$.currency', + }, + { + from: '$.userId', + to: '$.userId', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/contacts', + auth: 'noAuth', + method: 'POST', + format: 'FORM', + isBatchingEnabled: true, + maxBatchSize: '2', + propertiesMapping: [ + { + from: '$.traits.firstName', + to: '$.contacts.first_name', + }, + { + from: '$.traits.email', + to: '$.contacts.email', + }, + { + from: '$.traits.address.pinCode', + to: '$.contacts.address.pin_code', }, ], }, @@ -329,7 +674,7 @@ const properties = { const processorInstrumentationErrorStatTags = { destType: destTypeInUpperCase, errorCategory: 'dataValidation', - errorType: 'instrumentation', + errorType: 'configuration', feature: 'processor', implementation: 'cdkV2', module: 'destination', diff --git a/test/integrations/destinations/http/processor/configuration.ts b/test/integrations/destinations/http/processor/configuration.ts index 43d39952b9a..51a28fc2abf 100644 --- a/test/integrations/destinations/http/processor/configuration.ts +++ b/test/integrations/destinations/http/processor/configuration.ts @@ -1,6 +1,12 @@ import { ProcessorTestData } from '../../../testTypes'; import { generateMetadata, transformResultBuilder } from '../../../testUtils'; -import { destType, destinations, properties, traits } from '../common'; +import { + destType, + destinations, + properties, + traits, + processorInstrumentationErrorStatTags, +} from '../common'; export const configuration: ProcessorTestData[] = [ { @@ -38,6 +44,9 @@ export const configuration: ProcessorTestData[] = [ method: 'POST', userId: '', endpoint: destinations[0].Config.apiUrl, + headers: { + 'Content-Type': 'application/json', + }, JSON: { contacts: { first_name: 'John', @@ -89,8 +98,9 @@ export const configuration: ProcessorTestData[] = [ output: transformResultBuilder({ method: 'DELETE', userId: '', - endpoint: 'http://abc.com/contacts/john.doe@example.com/', + endpoint: 'http://abc.com/contacts/john.doe%40example.com', headers: { + 'Content-Type': 'application/json', 'x-api-key': 'test-api-key', }, }), @@ -137,9 +147,10 @@ export const configuration: ProcessorTestData[] = [ userId: '', endpoint: destinations[1].Config.apiUrl, headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', h1: 'val1', - h2: 2, + h2: '2', 'content-type': 'application/json', }, params: { @@ -191,14 +202,326 @@ export const configuration: ProcessorTestData[] = [ userId: '', endpoint: destinations[4].Config.apiUrl, headers: { + 'Content-Type': 'application/xml', Authorization: 'Bearer test-token', h1: 'val1', 'content-type': 'application/json', }, XML: { payload: - 'Order CompletedUSDuserId123622c6f5d5cf86a4c77358033Cones of Dunshire40577c6f5d5cf86a4c7735ba03Five Crowns5', + 'Order CompletedUSDuserId123622c6f5d5cf86a4c77358033Cones of Dunshire40577c6f5d5cf86a4c7735ba03Five Crowns5', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-5', + name: destType, + description: 'Track call with pathParams mapping', + scenario: 'Business', + successCriteria: 'Response should have the give paths added in the endpoint', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: destinations[7], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'GET', + userId: '', + endpoint: 'http://abc.com/contacts/userId123/c1', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC11c2VyOg==', + h1: 'val1', + h2: '2', + 'content-type': 'application/json', + }, + params: { + q1: 'val1', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-6', + name: destType, + description: 'Track call with query params keys containing space', + scenario: 'Business', + successCriteria: 'Response should contain query params with URI encoded keys', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: destinations[8], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'GET', + userId: '', + endpoint: 'http://abc.com/contacts/userId123/c1', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC11c2VyOg==', + h1: 'val1', + h2: '2', + 'content-type': 'application/json', + }, + params: { + 'user%20name': 'val1', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-7', + name: destType, + description: 'Identify call with properties mapping and form format with nested objects', + scenario: 'Business', + successCriteria: 'Response should be in form format with nested objects stringified', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: destinations[13], + message: { + type: 'identify', + userId: 'userId123', + anonymousId: 'anonId123', + traits, + }, + metadata: generateMetadata(1), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[13].Config.apiUrl, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + FORM: { + contacts: + '{"first_name":"John","email":"john.doe@example.com","address":{"pin_code":"123456"}}', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-8', + name: destType, + description: + 'Track call with bearer token, form format, post method, additional headers and properties mapping', + scenario: 'Business', + successCriteria: + 'Response should be in form format with post method, headers and properties mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + destination: destinations[10], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[10].Config.apiUrl, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Authorization: 'Bearer test-token', + h1: 'val1', + 'content-type': 'application/json', + }, + FORM: { + currency: 'USD', + event: 'Order Completed', + userId: 'userId123', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-9', + name: destType, + description: 'Track call with bearer token, form url encoded format', + scenario: 'Business', + successCriteria: + 'Response should be in form format with post method, headers and properties mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + destination: destinations[11], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[11].Config.apiUrl, + headers: { + Authorization: 'Bearer test-token', + h1: 'val1', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + FORM: { + currency: 'USD', + event: 'Order Completed', + userId: 'userId123', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-10', + name: destType, + description: 'empty body', + scenario: 'Business', + successCriteria: + 'Response should be in form format with post method, headers and properties mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + destination: destinations[12], + message: {}, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[12].Config.apiUrl, + headers: { + Authorization: 'Bearer test-token', + h1: 'val1', + 'Content-Type': 'application/x-www-form-urlencoded', }, + FORM: {}, }), statusCode: 200, metadata: generateMetadata(1), diff --git a/test/integrations/destinations/http/router/data.ts b/test/integrations/destinations/http/router/data.ts index ea14ec73418..c719196e6ba 100644 --- a/test/integrations/destinations/http/router/data.ts +++ b/test/integrations/destinations/http/router/data.ts @@ -182,9 +182,10 @@ export const data = [ version: '1', type: 'REST', method: 'GET', - endpoint: 'http://abc.com/contacts/john.doe@example.com/', + endpoint: 'http://abc.com/contacts/john.doe%40example.com', headers: { 'x-api-key': 'test-api-key', + 'Content-Type': 'application/json', }, params: {}, body: { @@ -233,15 +234,16 @@ export const data = [ method: 'GET', endpoint: 'http://abc.com/contacts', headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', 'content-type': 'application/json', h1: 'val1', - h2: 2, + h2: '2', h3: 'John', }, params: { q1: 'val1', - q2: 'john.doe@example.com', + q2: 'john.doe%40example.com', }, body: { JSON: {}, @@ -265,15 +267,16 @@ export const data = [ method: 'GET', endpoint: 'http://abc.com/contacts', headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', 'content-type': 'application/json', h1: 'val1', - h2: 2, + h2: '2', h3: 'John', }, params: { q1: 'val1', - q2: 'john.doe@example.com', + q2: 'john.doe%40example.com', }, body: { JSON: {}, @@ -297,15 +300,16 @@ export const data = [ method: 'GET', endpoint: 'http://abc.com/contacts', headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', 'content-type': 'application/json', h1: 'val1', - h2: 2, + h2: '2', h3: 'Alex', }, params: { q1: 'val1', - q2: 'alex.t@example.com', + q2: 'alex.t%40example.com', }, body: { JSON: {}, @@ -355,6 +359,7 @@ export const data = [ endpoint: 'http://abc.com/events', params: {}, headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', }, body: { @@ -409,6 +414,7 @@ export const data = [ method: 'POST', endpoint: 'http://abc.com/contacts/1234567890', headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', key: 'value1', }, @@ -433,6 +439,7 @@ export const data = [ method: 'POST', endpoint: 'http://abc.com/contacts/1234567890', headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', }, params: {}, @@ -456,6 +463,7 @@ export const data = [ method: 'POST', endpoint: 'http://abc.com/contacts/2234567890', headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', }, params: {}, diff --git a/test/integrations/destinations/iterable/processor/identifyTestData.ts b/test/integrations/destinations/iterable/processor/identifyTestData.ts index 792e16566c7..606a0de3103 100644 --- a/test/integrations/destinations/iterable/processor/identifyTestData.ts +++ b/test/integrations/destinations/iterable/processor/identifyTestData.ts @@ -36,6 +36,163 @@ const baseMetadata: Metadata = { }; export const identifyTestData: ProcessorTestData[] = [ + { + id: 'iterable-identify-test-has-multiple-responses', + name: 'iterable', + description: 'Indentify call to verify hasMultipleResponses', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update user payload with new email sent in payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + message: { + type: 'identify', + sentAt: '2020-08-28T16:26:16.473Z', + userId: 'userId', + channel: 'web', + context: { + os: { + name: '', + version: '1.12.3', + }, + app: { + name: 'RudderLabs JavaScript SDK', + build: '1.0.0', + version: '1.1.11', + namespace: 'com.rudderlabs.javascript', + }, + traits: { + email: 'ruchira@rudderlabs.com', + }, + locale: 'en-US', + device: { + token: 'token', + id: 'id', + type: 'ios', + }, + screen: { + density: 2, + }, + library: { + name: 'RudderLabs JavaScript SDK', + version: '1.1.11', + }, + campaign: {}, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.16; rv:84.0) Gecko/20100101 Firefox/84.0', + }, + rudderId: '62amo6xzksaeyupr4y0pfaucwj0upzs6g7yx', + messageId: 'hk02avz2xijdkid4i0mvncbm478g9lybdpgc', + anonymousId: 'anonId', + originalTimestamp: '2020-08-28T16:26:06.468Z', + }, + metadata: baseMetadata, + destination: { + ID: '123', + Name: 'iterable', + DestinationDefinition: { + ID: '123', + Name: 'iterable', + DisplayName: 'Iterable', + Config: {}, + }, + Config: { + apiKey: 'testApiKey', + dataCenter: 'USDC', + registerDeviceOrBrowserApiKey: 'randomApiKey', + preferUserId: false, + trackAllPages: true, + trackNamedPages: false, + mapToSingleEvent: false, + trackCategorisedPages: false, + }, + Enabled: true, + WorkspaceID: '123', + Transformations: [], + RevisionID: 'default-revision', + IsProcessorEnabled: true, + IsConnectionEnabled: true, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + userId: '', + method: 'POST', + endpoint: 'https://api.iterable.com/api/users/update', + headers: { + api_key: 'testApiKey', + 'Content-Type': 'application/json', + }, + params: {}, + body: { + JSON: { + email: 'ruchira@rudderlabs.com', + userId: 'userId', + dataFields: { + email: 'ruchira@rudderlabs.com', + }, + preferUserId: false, + mergeNestedObjects: true, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: baseMetadata, + statusCode: 200, + }, + { + output: { + body: { + FORM: {}, + JSON: { + device: { + platform: 'APNS', + token: 'token', + }, + email: 'ruchira@rudderlabs.com', + preferUserId: false, + userId: 'userId', + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.iterable.com/api/users/registerDeviceToken', + files: {}, + headers: { + 'Content-Type': 'application/json', + api_key: 'randomApiKey', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + metadata: baseMetadata, + statusCode: 200, + }, + ], + }, + }, + }, { id: 'iterable-identify-test-1', name: 'iterable', diff --git a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts index a50659343c5..ecc1fcd7f7f 100644 --- a/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/screenTestDataV2.ts @@ -55,7 +55,7 @@ export const screenTestData: ProcessorTestData[] = [ id: 'user@1', age: '22', email: 'test@rudderstack.com', - phone: '9112340375', + phone: '+9112340375', anonymousId: '9c6bd77ea9da3e68', append1: 'value1', }, @@ -125,7 +125,7 @@ export const screenTestData: ProcessorTestData[] = [ }, }, }, - phone_number: '9112340375', + phone_number: '+9112340375', properties: { id: 'user@1', age: '22', diff --git a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts index 78a3a0d7185..b2207998d5b 100644 --- a/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts +++ b/test/integrations/destinations/klaviyo/processor/trackTestDataV2.ts @@ -46,7 +46,7 @@ const commonOutputHeaders = { }; const profileAttributes = { email: 'test@rudderstack.com', - phone_number: '9112340375', + phone_number: '+9112340375', anonymous_id: '9c6bd77ea9da3e68', properties: { age: '22', @@ -86,7 +86,7 @@ export const trackTestData: ProcessorTestData[] = [ ...commonTraits, name: 'Test', email: 'test@rudderstack.com', - phone: '9112340375', + phone: '+9112340375', description: 'Sample description', }, }, @@ -174,7 +174,7 @@ export const trackTestData: ProcessorTestData[] = [ description: 'Sample description', name: 'Test', email: 'test@rudderstack.com', - phone: '9112340375', + phone: '+9112340375', }, }, properties: commonProps, @@ -257,7 +257,7 @@ export const trackTestData: ProcessorTestData[] = [ description: 'Sample description', name: 'Test', email: 'test@rudderstack.com', - phone: '9112340375', + phone: '+9112340375', }, }, properties: { ...commonProps, value: { price: 9.99 } }, @@ -293,4 +293,65 @@ export const trackTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'klaviyo-track-150624-test-4', + name: 'klaviyo', + description: '150624 -> Track event call, with phone not in E.164 format', + scenario: 'Business', + successCriteria: + 'Response should an error message and status code should be 400, as Phone number is not in E.164 format.', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { enforceEmailAsPrimary: true }), + message: generateSimplifiedTrackPayload({ + type: 'track', + event: 'TestEven001', + sentAt: '2025-01-01T11:11:11.111Z', + userId: 'invalidPhoneUser', + context: { + traits: { + ...commonTraits, + description: 'Sample description', + name: 'Test', + email: 'test@rudderstack.com', + phone: '9112340375', + }, + }, + properties: commonProps, + originalTimestamp: '2025-01-01T11:11:11.111Z', + }), + metadata: generateMetadata(4), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Phone number is not in E.164 format.', + statTags: { + destType: 'KLAVIYO', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'processor', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + metadata: generateMetadata(4), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/klaviyo/router/dataV2.ts b/test/integrations/destinations/klaviyo/router/dataV2.ts index 5201abc712c..d67d7947b4e 100644 --- a/test/integrations/destinations/klaviyo/router/dataV2.ts +++ b/test/integrations/destinations/klaviyo/router/dataV2.ts @@ -2050,4 +2050,108 @@ export const dataV2: RouterTestData[] = [ }, }, }, + { + id: 'klaviyo-router-150624-test-7', + name: 'klaviyo', + description: + '150624 -> Router tests to check for invalid phone number format in identify and track calls and should throw error', + scenario: 'Framework', + successCriteria: 'Should throw invalid phone number error for identify and track calls', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + userId: 'user123', + type: 'identify', + traits: { subscribe: true }, + context: { + traits: { + email: 'test@rudderstack.com', + phone: '123321000', + consent: 'email', + }, + ip: '14.5.67.21', + library: { name: 'http' }, + }, + timestamp: '2020-01-21T00:21:34.208Z', + }, + destination, + metadata: generateMetadata(1), + }, + { + message: { + type: 'track', + event: 'TestEven001', + sentAt: '2025-01-01T11:11:11.111Z', + userId: 'invalidPhoneUser', + context: { + traits: { + name: 'Test', + email: 'test@rudderstack.com', + phone: '9112340375', + }, + }, + properties: { + price: 120, + }, + originalTimestamp: '2025-01-01T11:11:11.111Z', + }, + metadata: generateMetadata(2), + destination, + }, + ], + destType: 'klaviyo', + }, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + error: 'Phone number is not in E.164 format.', + statTags: { + destType: 'KLAVIYO', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + metadata: [generateMetadata(1)], + batched: false, + statusCode: 400, + destination, + }, + { + error: 'Phone number is not in E.164 format.', + statTags: { + destType: 'KLAVIYO', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'instrumentation', + feature: 'router', + implementation: 'native', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + metadata: [generateMetadata(2)], + batched: false, + statusCode: 400, + destination, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index d13cf64cae2..9c385daee4b 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -97,7 +97,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","campaign_id":"test_name","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","campaign_id":"test_name","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -204,7 +204,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Viewed a Contact Us page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"name":"Contact Us","category":"Contact","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Viewed a Contact Us page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"name":"Contact Us","category":"Contact","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -726,7 +726,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":45.89,"counter":1,"item_purchased":"2","number_of_logins":"","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","campaign_id":"test_name","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342403,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":45.89,"counter":1,"item_purchased":"2","number_of_logins":"","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","campaign_id":"test_name","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342403,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -979,7 +979,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1138,7 +1138,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","revenue":34,"key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","first_name":"Mickey","lastName":"Mouse","name":"Mickey Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","revenue":34,"key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","first_name":"Mickey","lastName":"Mouse","name":"Mickey Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1272,7 +1272,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":" new Order Completed totally","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":" new Order Completed totally","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1406,7 +1406,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":" Order Completed ","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"Billing Amount":"77","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":" Order Completed ","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"Billing Amount":"77","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -2077,7 +2077,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstname":"Mickey","lastname":"Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstname":"Mickey","lastname":"Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -2374,7 +2374,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"path":"/tests/html/index2.html","referrer":"","search":"","title":"","url":"http://localhost/tests/html/index2.html","category":"communication","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"path":"/tests/html/index2.html","referrer":"","search":"","title":"","url":"http://localhost/tests/html/index2.html","category":"communication","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -3558,7 +3558,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"FirstTrackCall12","properties":{"foo":"bar","$deviceId":"nkasdnkasd","anonymousId":"ea776ad0-3136-44fb-9216-5b1578609a2b","userId":"as09sufa09usaf09as0f9uasf","id":"as09sufa09usaf09as0f9uasf","firstName":"Bob","lastName":"Marley","name":"Bob Marley","age":43,"email":"bob@marleymail.com","phone":"+447748544123","birthday":"1987-01-01T20:08:59+0000","createdAt":"2022-01-21T14:10:12+0000","address":"51,B.L.T road, Kolkata-700060","description":"I am great","gender":"male","title":"Founder","username":"bobm","website":"https://bobm.com","randomProperty":"randomValue","$user_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","$current_url":"http://127.0.0.1:7307/Testing/App_for_testingTool/","$referrer":"http://127.0.0.1:7307/Testing/","$screen_height":900,"$screen_width":1440,"$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.1.18","$insert_id":"0d5c1a4a-27e4-41da-a246-4d01f44e74bd","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1632986123523,"$browser":"Chrome","$browser_version":"93.0.4577.82"}}]', + '[{"event":"FirstTrackCall12","properties":{"foo":"bar","$deviceId":"nkasdnkasd","anonymousId":"ea776ad0-3136-44fb-9216-5b1578609a2b","userId":"as09sufa09usaf09as0f9uasf","id":"as09sufa09usaf09as0f9uasf","firstName":"Bob","lastName":"Marley","name":"Bob Marley","age":43,"email":"bob@marleymail.com","phone":"+447748544123","birthday":"1987-01-01T20:08:59+0000","createdAt":"2022-01-21T14:10:12+0000","address":"51,B.L.T road, Kolkata-700060","description":"I am great","gender":"male","title":"Founder","username":"bobm","website":"https://bobm.com","randomProperty":"randomValue","$user_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","$current_url":"http://127.0.0.1:7307/Testing/App_for_testingTool/","$referrer":"http://127.0.0.1:7307/Testing/","$screen_height":900,"$screen_width":1440,"$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"http://127.0.0.1:7307/Testing/","$initial_referring_domain":"127.0.0.1:7307","$app_build_number":"1.0.0","$app_version_string":"1.1.18","$insert_id":"0d5c1a4a-27e4-41da-a246-4d01f44e74bd","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1632986123523,"$browser":"Chrome","$browser_version":"93.0.4577.82"}}]', }, XML: {}, FORM: {}, @@ -5174,7 +5174,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":18.9,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$user_id":"userId01","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"userId01","time":1579847342403,"$device_id":"anonId01","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":18.9,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$user_id":"userId01","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"userId01","time":1579847342403,"$device_id":"anonId01","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -5281,7 +5281,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"$device:anonId01","time":1579847342402,"$device_id":"anonId01","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"$device:anonId01","time":1579847342402,"$device_id":"anonId01","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, diff --git a/test/integrations/destinations/mp/router/data.ts b/test/integrations/destinations/mp/router/data.ts index 8716c9daa09..67f4d006c28 100644 --- a/test/integrations/destinations/mp/router/data.ts +++ b/test/integrations/destinations/mp/router/data.ts @@ -450,7 +450,7 @@ export const data = [ JSON_ARRAY: {}, GZIP: { payload: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1173,7 +1173,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, diff --git a/test/integrations/sources/shopify/constants.ts b/test/integrations/sources/shopify/constants.ts index af53a3180e8..cd362adaec3 100644 --- a/test/integrations/sources/shopify/constants.ts +++ b/test/integrations/sources/shopify/constants.ts @@ -1,3 +1,58 @@ +const dummyResponseCommonPayload = { + navigator: { + language: 'en-US', + cookieEnabled: true, + languages: ['en-US', 'en'], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + }, + window: { + innerHeight: 1028, + innerWidth: 1362, + outerHeight: 1080, + outerWidth: 1728, + pageXOffset: 0, + pageYOffset: 0, + location: { + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + hash: '', + host: 'store.myshopify.com', + hostname: 'store.myshopify.com', + origin: 'https://store.myshopify.com', + pathname: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + port: '', + protocol: 'https:', + search: '', + }, + origin: 'https://store.myshopify.com', + screen: { + height: 1117, + width: 1728, + }, + screenX: 0, + screenY: 37, + scrollX: 0, + scrollY: 0, + }, + page: { + title: 'Checkout - pixel-testing-rs', + url: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + path: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + search: '', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + screen: { + height: 1117, + width: 1728, + }, + library: { + name: 'RudderStack Shopify Cloud', + eventOrigin: 'client', + version: '2.0.0', + }, +}; + export const dummySourceConfig = { ID: 'dummy-source-id', OriginalID: '', @@ -83,25 +138,10 @@ export const dummyContext = { }, }; -export const note_attributes = [ - { - name: 'cartId', - value: '9c623f099fc8819aa4d6a958b65dfe7d', - }, - { - name: 'cartToken', - value: 'Z2NwLXVzLWVhc3QxOjAxSkQzNUFXVEI4VkVUNUpTTk1LSzBCMzlF', - }, - { - name: 'rudderAnonymousId', - value: '50ead33e-d763-4854-b0ab-765859ef05cb', - }, -]; - -export const responseDummyContext = { +export const dummyContextwithCampaign = { document: { location: { - href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU?checkout%5Bpayment_gateway%5D=shopify_payments&utm_campaign=shopifySale&utm_medium=checkout&utm_term=term_checkout&utm_content=web&utm_custom1=customutm&tag=tag', hash: '', host: 'store.myshopify.com', hostname: 'store.myshopify.com', @@ -150,21 +190,60 @@ export const responseDummyContext = { scrollX: 0, scrollY: 0, }, - page: { - title: 'Checkout - pixel-testing-rs', - url: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', - path: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', - search: '', +}; + +export const note_attributes = [ + { + name: 'cartId', + value: '9c623f099fc8819aa4d6a958b65dfe7d', }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', - screen: { - height: 1117, - width: 1728, + { + name: 'cartToken', + value: 'Z2NwLXVzLWVhc3QxOjAxSkQzNUFXVEI4VkVUNUpTTk1LSzBCMzlF', }, - library: { - name: 'RudderStack Shopify Cloud', - eventOrigin: 'client', - version: '2.0.0', + { + name: 'rudderAnonymousId', + value: '50ead33e-d763-4854-b0ab-765859ef05cb', + }, +]; + +export const responseDummyContext = { + document: { + location: { + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + hash: '', + host: 'store.myshopify.com', + hostname: 'store.myshopify.com', + origin: 'https://store.myshopify.com', + pathname: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + port: '', + protocol: 'https:', + search: '', + }, + referrer: 'https://store.myshopify.com/cart', + characterSet: 'UTF-8', + title: 'Checkout - pixel-testing-rs', + }, + ...dummyResponseCommonPayload, +}; + +export const responseDummyContextwithCampaign = { + document: { + location: { + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU?checkout%5Bpayment_gateway%5D=shopify_payments&utm_campaign=shopifySale&utm_medium=checkout&utm_term=term_checkout&utm_content=web&utm_custom1=customutm&tag=tag', + hash: '', + host: 'store.myshopify.com', + hostname: 'store.myshopify.com', + origin: 'https://store.myshopify.com', + pathname: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + port: '', + protocol: 'https:', + search: '', + }, + referrer: 'https://store.myshopify.com/cart', + title: 'Checkout - pixel-testing-rs', + characterSet: 'UTF-8', }, + // title: 'Checkout - pixel-testing-rs', + ...dummyResponseCommonPayload, }; diff --git a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts index ff1ea39ed13..2b04a33fb8b 100644 --- a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts @@ -336,6 +336,11 @@ export const pixelCheckoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Started', @@ -775,6 +780,11 @@ export const pixelCheckoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Completed', diff --git a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts index 3db2e3ab101..95fd2ea26bd 100644 --- a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts +++ b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts @@ -387,6 +387,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Address Info Submitted', @@ -921,6 +926,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Contact Info Submitted', @@ -1470,6 +1480,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Shipping Info Submitted', @@ -2035,6 +2050,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Payment Info Entered', diff --git a/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts b/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts index 46bd4f96151..7427ea5ae76 100644 --- a/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts +++ b/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts @@ -1,5 +1,11 @@ // This file contains the test scenarios related to Shopify pixel events, emitted from web pixel on the browser. -import { dummyContext, dummySourceConfig, responseDummyContext } from '../constants'; +import { + dummyContext, + dummyContextwithCampaign, + dummySourceConfig, + responseDummyContext, + responseDummyContextwithCampaign, +} from '../constants'; export const pixelEventsTestScenarios = [ { @@ -18,7 +24,7 @@ export const pixelEventsTestScenarios = [ type: 'standard', clientId: 'c7b3f99b-4d34-463b-835f-c879482a7750', timestamp: '2024-09-15T17:24:30.373Z', - context: dummyContext, + context: dummyContextwithCampaign, pixelEventLabel: true, query_parameters: { topic: ['page_viewed'], @@ -42,7 +48,14 @@ export const pixelEventsTestScenarios = [ batch: [ { context: { - ...responseDummyContext, + ...responseDummyContextwithCampaign, + campaign: { + content: 'web', + medium: 'checkout', + name: 'shopifySale', + term: 'term_checkout', + utm_custom1: 'customutm', + }, shopifyDetails: { clientId: 'c7b3f99b-4d34-463b-835f-c879482a7750', data: {}, @@ -55,6 +68,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['page.context.shopifyDetails'], + }, + }, }, name: 'Page View', type: 'page', @@ -166,6 +184,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product Viewed', @@ -330,6 +353,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Cart Viewed', @@ -555,6 +583,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product List Viewed', @@ -725,6 +758,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product Added', @@ -866,6 +904,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product Removed', @@ -955,6 +998,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Search Submitted', diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts index b6fa5be322e..4938294ef99 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -215,12 +215,17 @@ export const checkoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', - event: 'Checkout Started - Webhook', + event: 'Checkout Started Webhook', properties: { order_id: '35550298931313', - value: '600.00', + value: 600, tax: 0, currency: 'USD', products: [ @@ -530,6 +535,11 @@ export const checkoutEventsTestScenarios = [ event: 'Checkout Updated', integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, properties: { currency: 'USD', @@ -1202,6 +1212,11 @@ export const checkoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Updated', @@ -1592,12 +1607,17 @@ export const checkoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Created', properties: { order_id: '5778367414385', - value: '600.00', + value: 600, tax: 0, currency: 'USD', products: [ @@ -1683,4 +1703,139 @@ export const checkoutEventsTestScenarios = [ }, }, }, + { + id: 'c005', + name: 'shopify', + description: 'Track Call -> Order Cancelled event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + email: 'henry@wfls.com', + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + line_items: [ + { + id: 14234727743601, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + sku: '', + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + city: 'Johnson City', + zip: '37604', + }, + query_parameters: { + topic: ['orders_cancelled'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnZkCHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + integration: { + name: 'SHOPIFY', + }, + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + shopifyDetails: { + email: 'henry@wfls.com', + line_items: [ + { + id: 14234727743601, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + sku: '', + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + address1: 'Yuimaru Kitchen', + city: 'Johnson City', + first_name: 'henry', + zip: '37604', + }, + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + }, + topic: 'orders_cancelled', + }, + event: 'Order Cancelled', + integrations: { + SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, + }, + properties: { + products: [ + { + brand: 'Hydrogen Vendor', + price: 600, + product_id: '7234590408817', + quantity: 1, + title: 'The Collection Snowboard: Hydrogen', + }, + ], + tax: 0, + value: 600, + }, + timestamp: '2024-11-06T02:54:50.000Z', + traits: { + email: 'henry@wfls.com', + shippingAddress: { + address1: 'Yuimaru Kitchen', + city: 'Johnson City', + first_name: 'henry', + zip: '37604', + }, + }, + type: 'track', + }, + ], + }, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts index d68d0a8f59c..422fe0135a2 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -68,6 +68,11 @@ export const genericTrackTestScenarios = [ event: 'Cart Update', integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, properties: { products: [], @@ -530,6 +535,11 @@ export const genericTrackTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Paid', diff --git a/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts index b03f5635b6a..346485fe1e6 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts @@ -187,6 +187,11 @@ export const identityTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['identify.context.shopifyDetails'], + }, + }, }, type: 'identify', userId: '7358220173425',