From 62cff1e1e09fb6ad4a34c43bc97f0959aa6a40d3 Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Sat, 30 Nov 2024 18:45:24 +0100 Subject: [PATCH 1/2] fix: exact path env variable in the endSpan --- src/spans/awsSpan.test.js | 30 ++++++++++++++++++++++++++++++ src/spans/awsSpan.ts | 4 +++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/spans/awsSpan.test.js b/src/spans/awsSpan.test.js index 054fb17cd..5dad24fd1 100644 --- a/src/spans/awsSpan.test.js +++ b/src/spans/awsSpan.test.js @@ -37,6 +37,7 @@ import { RedisSpanBuilder } from '../../testUtils/redisSpanBuilder'; import { PrismaSpanBuilder } from '../../testUtils/prismaSpanBuilder'; const exampleApiGatewayEvent = require('../../testUtils/testdata/events/apigw-request.json'); +const clone = require('rfdc')(); describe('awsSpan', () => { const spies = {}; @@ -444,6 +445,35 @@ describe('awsSpan', () => { expect(endSpan.envs.length).toBeGreaterThan(startSpan.envs.length); }); + test('getEndFunctionSpan does not modify returnValue payload', () => { + const functionSpan = { + id: '6d26e3c8-60a6-4cee-8a70-f525f47a4caf_started', + }; + + const handlerReturnValue = { + err: null, + data: { + string: 'value', + object: { + string: 'value', + object: { + string: 'value', + }, + }, + }, + }; + const expectedData = clone(handlerReturnValue.data); + + process.env[LUMIGO_SECRET_MASKING_EXACT_PATH] = + '["string","object.string", "object.object.string"]'; + + const endFunctionSpan = awsSpan.getEndFunctionSpan(functionSpan, handlerReturnValue); + expect(endFunctionSpan.return_value).toEqual( + '{"string":"****","object":{"string":"****","object":{"string":"****"}}}' + ); + expect(handlerReturnValue.data).toEqual(expectedData); + }); + test('Lambda invoked by S3 -> shouldnt scrub known S3 fields', () => { const { context } = TracerGlobals.getHandlerInputs(); const event = { diff --git a/src/spans/awsSpan.ts b/src/spans/awsSpan.ts index 933335615..e96ad7d46 100644 --- a/src/spans/awsSpan.ts +++ b/src/spans/awsSpan.ts @@ -51,6 +51,8 @@ export const NEO4J_SPAN = 'neo4j'; export const ENRICHMENT_SPAN = 'enrichment'; export const PRISMA_SPAN = 'prisma'; +const clone = require('rfdc')({ proto: true }); + export const getSpanInfo = (): SpanInfo => { const tracer = getTracerInfo(); @@ -242,7 +244,7 @@ export const getEndFunctionSpan = (functionSpan: GenericSpan, handlerReturnValue const ended = new Date().getTime(); let returnValue: any; try { - returnValue = payloadStringify(data); + returnValue = payloadStringify(clone(data)); } catch (e) { returnValue = truncate(data.toString(), getEventEntitySize(true)); error = parseErrorObject({ From 503bc77eb07687cfa640505aa1f56d54cbe53ecb Mon Sep 17 00:00:00 2001 From: Eugene Orlovsky Date: Sun, 1 Dec 2024 19:37:13 +0100 Subject: [PATCH 2/2] fix: scrubPayloadBasedOnExactPath --- src/spans/awsSpan.test.js | 8 +++--- src/spans/awsSpan.ts | 4 +-- src/utils/payloadStringify.js | 49 +++++++++++++++++++---------------- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/spans/awsSpan.test.js b/src/spans/awsSpan.test.js index 5dad24fd1..8a667498d 100644 --- a/src/spans/awsSpan.test.js +++ b/src/spans/awsSpan.test.js @@ -37,7 +37,6 @@ import { RedisSpanBuilder } from '../../testUtils/redisSpanBuilder'; import { PrismaSpanBuilder } from '../../testUtils/prismaSpanBuilder'; const exampleApiGatewayEvent = require('../../testUtils/testdata/events/apigw-request.json'); -const clone = require('rfdc')(); describe('awsSpan', () => { const spies = {}; @@ -462,8 +461,6 @@ describe('awsSpan', () => { }, }, }; - const expectedData = clone(handlerReturnValue.data); - process.env[LUMIGO_SECRET_MASKING_EXACT_PATH] = '["string","object.string", "object.object.string"]'; @@ -471,7 +468,10 @@ describe('awsSpan', () => { expect(endFunctionSpan.return_value).toEqual( '{"string":"****","object":{"string":"****","object":{"string":"****"}}}' ); - expect(handlerReturnValue.data).toEqual(expectedData); + // here we expect the original data to be untouched + expect(handlerReturnValue.data.string).toEqual('value'); + expect(handlerReturnValue.data.object.string).toEqual('value'); + expect(handlerReturnValue.data.object.object.string).toEqual('value'); }); test('Lambda invoked by S3 -> shouldnt scrub known S3 fields', () => { diff --git a/src/spans/awsSpan.ts b/src/spans/awsSpan.ts index e96ad7d46..933335615 100644 --- a/src/spans/awsSpan.ts +++ b/src/spans/awsSpan.ts @@ -51,8 +51,6 @@ export const NEO4J_SPAN = 'neo4j'; export const ENRICHMENT_SPAN = 'enrichment'; export const PRISMA_SPAN = 'prisma'; -const clone = require('rfdc')({ proto: true }); - export const getSpanInfo = (): SpanInfo => { const tracer = getTracerInfo(); @@ -244,7 +242,7 @@ export const getEndFunctionSpan = (functionSpan: GenericSpan, handlerReturnValue const ended = new Date().getTime(); let returnValue: any; try { - returnValue = payloadStringify(clone(data)); + returnValue = payloadStringify(data); } catch (e) { returnValue = truncate(data.toString(), getEventEntitySize(true)); error = parseErrorObject({ diff --git a/src/utils/payloadStringify.js b/src/utils/payloadStringify.js index 2fe5bc677..c4aaeba2f 100644 --- a/src/utils/payloadStringify.js +++ b/src/utils/payloadStringify.js @@ -26,6 +26,7 @@ const nativeTypes = ['string', 'bigint', 'number', 'undefined', 'boolean']; const SCRUBBED_TEXT = '****'; const TRUNCATED_TEXT = '...[too long]'; const FAILED_SCRUBBING_BY_PATH = 'Failed to scrub payload by exact path'; +const clone = require('rfdc')(); const isNativeType = (obj) => nativeTypes.includes(typeof obj); @@ -176,6 +177,30 @@ function logSecretMaskingDebug(logger, message, additionalData) { } } +function scrubPayloadBasedOnExactPath(originalPayload) { + let payload = clone(originalPayload); + let secretPaths = getSecretPaths(); + if (secretPaths.length > 0) { + const uniquePaths = getUniqPaths(secretPaths); + if (isString(payload)) { + payload = safeExecute( + scrubJsonStringBySecretPath, + FAILED_SCRUBBING_BY_PATH, + logger.LOG_LEVELS.DEBUG, + payload + )(payload, secretPaths, uniquePaths, ''); + } else { + payload = safeExecute( + scrubJsonBySecretPath, + FAILED_SCRUBBING_BY_PATH, + logger.LOG_LEVELS.DEBUG, + payload + )(payload, secretPaths, uniquePaths, ''); + } + } + return payload; +} + export const payloadStringify = ( payload, maxPayloadSize = getEventEntitySize(), @@ -190,29 +215,9 @@ export const payloadStringify = ( let isPruned = false; - if (getSecretMaskingExactPath()) { - let secretPaths = getSecretPaths(); - if (secretPaths.length > 0) { - const uniquePaths = getUniqPaths(secretPaths); - if (isString(payload)) { - payload = safeExecute( - scrubJsonStringBySecretPath, - FAILED_SCRUBBING_BY_PATH, - logger.LOG_LEVELS.DEBUG, - payload - )(payload, secretPaths, uniquePaths, ''); - } else { - payload = safeExecute( - scrubJsonBySecretPath, - FAILED_SCRUBBING_BY_PATH, - logger.LOG_LEVELS.DEBUG, - payload - )(payload, secretPaths, uniquePaths, ''); - } - } - } + let result = getSecretMaskingExactPath() ? scrubPayloadBasedOnExactPath(payload) : payload; - let result = JSON.stringify(payload, function (key, value) { + result = JSON.stringify(result, function (key, value) { const type = typeof value; const isObj = type === 'object'; const isStr = type === 'string';