new guardian subscription model

buildcheck/data/build.ts

@@ -195,6 +195,8 @@ const productSwitchApi: HandlerDefinition = {
 	},
 	devDependencies: {
 		...devDeps['@types/aws-lambda'],
+		...dep['@aws-sdk/client-cloudwatch-logs'],
+		...dep['@aws-sdk/credential-providers'],
 	},
 };
 

buildcheck/data/dependencies.ts

@@ -33,6 +33,7 @@ export const dep = separateDepRecords({
 		'client-lambda',
 		'credential-provider-node',
 		'lib-storage',
+		'client-cloudwatch-logs',
 	]),
 });
 

handlers/product-switch-api/package.json

@@ -22,6 +22,8 @@
     "zod": "catalog:"
   },
   "devDependencies": {
-    "@types/aws-lambda": "^8.10.147"
+    "@types/aws-lambda": "^8.10.147",
+    "@aws-sdk/client-cloudwatch-logs": "^3.940.0",
+    "@aws-sdk/credential-providers": "^3.940.0"
   }
 }

handlers/product-switch-api/runManual/realSusbcriptions/README.md

@@ -0,0 +1,30 @@
+# Redacted real subscription testing
+
+This lets us test the parser on redacted subscriptions that have been seen in PROD in recent days.
+
+It gives us greater confidence that a change will work in real life without breaking within a day or two.
+
+## GDPR and redaction
+
+Although subscriptions don't have much sensitive data, we still need to be careful to look after it.
+
+Risks of using real data are that it could be kept for longer than the retention period, or accidentally
+committed to a repo.
+
+- automatically replaces identifiers with arbitrary ids before storing them in the files.
+- data should be stored outside of a git root
+- tests are not run in CI so nothing could be logged publicly
+
+## How to use
+
+### Download and redact the data
+1. get AWS credentials
+2. set a suitable location in config.ts - to further reduce the risk of publication don't put it below a git root
+3. run downloadRealSubscriptions.ts (this takes up to a minute)
+
+now the location specified will be populated with redacted json files for all subscriptions found in the log files
+
+### Run the test
+1. Rename testRealSubscriptions.ts to testRealSubscriptions.test.ts so jest can run it
+1. run the test from intellij
+1. rename the test back before committing to avoid it failing in CI

handlers/product-switch-api/runManual/realSusbcriptions/config.ts

@@ -0,0 +1,7 @@
+/**
+ * to avoid accidental commits, fill in a suitable location that's outside of any repo
+ */
+export const subscriptionsDir =
+	'/Users/john_duffell/Downloads/subscriptionsJson/PROD/';
+
+export const logGroupName = '/aws/lambda/product-switch-api-PROD';

handlers/product-switch-api/runManual/realSusbcriptions/downloadRealSubscriptions.ts

@@ -0,0 +1,242 @@
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import type { GetQueryResultsCommandOutput } from '@aws-sdk/client-cloudwatch-logs';
+import {
+	CloudWatchLogsClient,
+	GetQueryResultsCommand,
+	StartQueryCommand,
+} from '@aws-sdk/client-cloudwatch-logs';
+import { fromIni } from '@aws-sdk/credential-providers';
+import { logGroupName, subscriptionsDir } from './config';
+
+/*
+ * this script is used to prepare for running testRealSubscriptions.ts.  It
+ *
+ * - downloads all the subscriptions used by the lambda over the past two weeks,
+ * - redacts the relevant ids and numbers, and
+ * - writes them as json files to the directory specified in config.ts
+ *
+ * First prepare AWS credentials, then run this script.
+ */
+
+function randomSubscription(): string {
+	return 'A-S999' + Math.floor(Math.random() * 100000);
+}
+
+function randomHex32(): string {
+	const chars = '0123456789abcdef';
+	return (
+		'999' +
+		Array.from(
+			{ length: 29 },
+			() => chars[Math.floor(Math.random() * chars.length)],
+		).join('')
+	);
+}
+
+function randomChargeNum(): string {
+	return 'C-999' + Math.floor(Math.random() * 100000);
+}
+
+function randomAccount(): string {
+	return 'A999' + Math.floor(Math.random() * 100000);
+}
+
+interface RatePlanCharge {
+	id?: string;
+	number?: string;
+	[key: string]: unknown;
+}
+
+interface RatePlan {
+	id?: string;
+	ratePlanCharges?: RatePlanCharge[];
+	[key: string]: unknown;
+}
+
+interface RedactableSubscription {
+	id?: string;
+	accountNumber?: string;
+	subscriptionNumber?: string;
+	ratePlans?: RatePlan[];
+	[key: string]: unknown;
+}
+
+function redactSubscription(obj: unknown): unknown {
+	if (typeof obj !== 'object' || obj === null) {
+		return obj;
+	}
+
+	if (Array.isArray(obj)) {
+		return obj.map(redactSubscription);
+	}
+
+	const result = {
+		...(obj as Record<string, unknown>),
+	} as RedactableSubscription;
+
+	if ('id' in result) {
+		result.id = randomHex32();
+	}
+	if ('accountNumber' in result) {
+		result.accountNumber = randomAccount();
+	}
+	if ('subscriptionNumber' in result) {
+		result.subscriptionNumber = randomSubscription();
+	}
+
+	if ('ratePlans' in result && Array.isArray(result.ratePlans)) {
+		result.ratePlans = result.ratePlans.map((plan) => {
+			const newPlan = { ...plan };
+			if ('id' in newPlan) {
+				newPlan.id = randomHex32();
+			}
+
+			if (
+				'ratePlanCharges' in newPlan &&
+				Array.isArray(newPlan.ratePlanCharges)
+			) {
+				newPlan.ratePlanCharges = newPlan.ratePlanCharges.map(
+					(charge): RatePlanCharge => ({
+						...charge,
+						id: randomHex32(),
+						number: randomChargeNum(),
+					}),
+				);
+			}
+			return newPlan;
+		});
+	}
+
+	return result;
+}
+
+function extractJsonFromLog(entry: string): string | null {
+	const lines = entry.split('\n');
+	const startIndex = lines.findIndex((line) => line === '{');
+	if (startIndex === -1) {
+		return null;
+	}
+
+	const endIndex = lines.findIndex((line, i) => i > startIndex && line === '}');
+	if (endIndex === -1) {
+		return null;
+	}
+
+	return lines.slice(startIndex, endIndex + 1).join('\n');
+}
+
+function fixBareKeys(jsonStr: string): string {
+	return jsonStr.replace(/(\s+)([a-zA-Z_][a-zA-Z0-9_]*)\s*:/g, '$1"$2":');
+}
+
+async function main() {
+	if (!subscriptionsDir.startsWith('/')) {
+		console.log(
+			'do not write to a relative path, make sure it is outside of any git roots',
+		);
+		process.exit(1);
+	}
+
+	if (!existsSync(subscriptionsDir)) {
+		mkdirSync(subscriptionsDir, { recursive: false });
+	}
+
+	const region = 'eu-west-1';
+	const queryLookbackDays = 14;
+
+	const client = new CloudWatchLogsClient({
+		region,
+		credentials: fromIni({ profile: 'membership' }),
+	});
+
+	const startTime =
+		Math.floor(Date.now() / 1000) - queryLookbackDays * 24 * 3600;
+	const endTime = Math.floor(Date.now() / 1000);
+
+	const query = `fields @timestamp, @message
+ | filter @message like /TRACE HTTP _ZuoraClient EXIT SHORT_ARGS/
+ | filter @message like /path: v1\\/subscriptions\\/A-S[0-9]+/
+ | filter @message like /method: GET/
+ | sort @timestamp desc
+ | limit 10000`;
+
+	console.log(`Starting CloudWatch query on group ${logGroupName} ...`);
+
+	const startQueryResp = await client.send(
+		new StartQueryCommand({
+			logGroupName,
+			startTime,
+			endTime,
+			queryString: query,
+		}),
+	);
+
+	const queryId = startQueryResp.queryId!;
+	let status = 'Running';
+	let results: Array<Record<string, string>> = [];
+
+	while (status === 'Running' || status === 'Scheduled') {
+		await new Promise((resolve) => setTimeout(resolve, 1000));
+		const resp: GetQueryResultsCommandOutput = await client.send(
+			new GetQueryResultsCommand({ queryId }),
+		);
+		status = resp.status!;
+
+		if (status === 'Complete' && resp.results) {
+			results = resp.results.map((row) =>
+				Object.fromEntries(
+					row.map((field) => [field.field ?? '', field.value ?? '']),
+				),
+			);
+		}
+	}
+
+	console.log(`Query ${queryId} complete, ${results.length} results`);
+
+	let count = 0;
+
+	for (const fields of results) {
+		const message = fields['@message'];
+		if (!message) {
+			continue;
+		}
+
+		console.log('message: ' + message);
+
+		const jsonStr = extractJsonFromLog(message);
+		if (!jsonStr) {
+			continue;
+		}
+
+		console.log('    jsonStr: ' + jsonStr);
+
+		const fixedJson = fixBareKeys(jsonStr);
+		console.log('    fixedJson: ' + fixedJson);
+
+		try {
+			const parsedJson: unknown = JSON.parse(fixedJson);
+			console.log('    parseResult: success');
+
+			const redacted = redactSubscription(parsedJson);
+			const timestamp = fields['@timestamp']!.replace(/ /g, 'T');
+			const fileName = `subscriptionRedacted-${timestamp}.json`;
+
+			const outFile = join(subscriptionsDir, fileName);
+			const content = JSON.stringify(redacted, null, 2);
+
+			writeFileSync(outFile, content);
+			console.log('    redacted.spaces2: ' + content);
+			console.log(`    Written ${fileName}`);
+			count++;
+		} catch (error) {
+			console.log('    parseResult: failed', error);
+		}
+	}
+
+	console.log(
+		`Finished! ${count} files written out of potential ${results.length}.`,
+	);
+}
+
+main().then(console.log).catch(console.error);

handlers/product-switch-api/runManual/realSusbcriptions/testRealSubscriptions.ts

@@ -0,0 +1,61 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { logger } from '@modules/routing/logger';
+import type { ZuoraSubscription } from '@modules/zuora/types';
+import { zuoraSubscriptionSchema } from '@modules/zuora/types';
+import dayjs from 'dayjs';
+import zuoraCatalogFixture from '../../../../modules/zuora-catalog/test/fixtures/catalog-prod.json';
+import { getSinglePlanFlattenedSubscriptionOrThrow } from '../../src/guardianSubscription/getSinglePlanFlattenedSubscriptionOrThrow';
+import { GuardianSubscriptionParser } from '../../src/guardianSubscription/guardianSubscriptionParser';
+import { SubscriptionFilter } from '../../src/guardianSubscription/subscriptionFilter';
+import { productCatalog } from '../../test/productCatalogFixture';
+import { subscriptionsDir } from './config';
+
+test('processes all PROD subscription files successfully', () => {
+	const files = fs.readdirSync(subscriptionsDir);
+	const matchingFiles = files.filter((file: string) =>
+		/^subscriptionRedacted-\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\.json$/.test(
+			file,
+		),
+	);
+
+	expect(matchingFiles.length).toBeGreaterThan(0);
+
+	const guardianSubscriptionParser = new GuardianSubscriptionParser(
+		zuoraCatalogFixture,
+		productCatalog,
+	);
+
+	matchingFiles.forEach((file: string) => {
+		const filePath = path.join(subscriptionsDir, file);
+
+		try {
+			// file names are like subscriptionRedacted-2026-02-03T08:21:22.946.json
+			const dateMatch = file.match(/subscriptionRedacted-(\d{4}-\d{2}-\d{2})/);
+			if (!dateMatch) {
+				throw new Error(`Could not extract date from filename: ${file}`);
+			}
+			const callDate = dayjs(dateMatch[1]);
+
+			const subscriptionData: ZuoraSubscription = zuoraSubscriptionSchema.parse(
+				JSON.parse(fs.readFileSync(filePath, 'utf-8')),
+			);
+			const zuoraSubscription = zuoraSubscriptionSchema.parse(subscriptionData);
+			const guardianSubscription =
+				guardianSubscriptionParser.toGuardianSubscription(zuoraSubscription);
+			const filter =
+				SubscriptionFilter.activeNonEndedSubscriptionFilter(callDate);
+			const filteredSubscription =
+				filter.filterSubscription(guardianSubscription);
+			const flattenedSubscription =
+				getSinglePlanFlattenedSubscriptionOrThrow(filteredSubscription);
+
+			expect(flattenedSubscription).toBeDefined();
+			expect(flattenedSubscription.ratePlan).toBeDefined();
+			logger.log(`Successfully processed ${file}`);
+		} catch (error) {
+			logger.log(`Failed to process ${file}`, { error });
+			throw error;
+		}
+	});
+});