Support Sourcepoint GPP consent for EC generation (#642)

Author: ChristianPavilonis

.github/actions/setup-integration-test-env/action.yml

@@ -80,7 +80,7 @@ runs:
       env:
         TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:${{ inputs.origin-port }}
         TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret
-        TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret
+        TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32
         TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false"
       run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1
 

.github/workflows/test.yml

@@ -55,7 +55,7 @@ jobs:
         env:
           TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:8080
           TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret
-          TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret
+          TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32
           TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false"
         run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1
 

crates/integration-tests/fixtures/configs/viceroy-template.toml

@@ -25,11 +25,29 @@
             key = "placeholder"
             data = "placeholder"
 
-        # Pre-seeded EC row for KV-backed EC lifecycle tests.
+        # Pre-seeded EC rows for KV-backed EC lifecycle tests. Each scenario
+        # uses a separate row so withdrawal tombstones do not leak across
+        # sequential scenario execution in the same Viceroy instance.
         [[local_server.kv_stores.ec_identity_store]]
             key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01"
             data = '{"v":1,"created":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US","region":"CA"}}'
 
+        [[local_server.kv_stores.ec_identity_store]]
+            key = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.test02"
+            data = '{"v":1,"created":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US","region":"CA"}}'
+
+        [[local_server.kv_stores.ec_identity_store]]
+            key = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc.test03"
+            data = '{"v":1,"created":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US","region":"CA"}}'
+
+        [[local_server.kv_stores.ec_identity_store]]
+            key = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd.test04"
+            data = '{"v":1,"created":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US","region":"CA"}}'
+
+        [[local_server.kv_stores.ec_identity_store]]
+            key = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.test05"
+            data = '{"v":1,"created":1700000000,"consent":{"ok":true,"updated":1700000000},"geo":{"country":"US","region":"CA"}}'
+
         [[local_server.kv_stores.ec_partner_store]]
             key = "placeholder"
             data = "placeholder"

crates/integration-tests/tests/frameworks/scenarios.rs

@@ -500,16 +500,17 @@ impl EcScenario {
 /// US Privacy signal that explicitly allows storage in the default Viceroy
 /// integration-test geo (US-CA).
 const ALLOW_US_PRIVACY_COOKIE: &str = "1YNN";
-const SEEDED_EC_ID: &str =
-    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test01";
-
 fn allow_ec_generation(client: &EcTestClient) {
     client.set_cookie("us_privacy", ALLOW_US_PRIVACY_COOKIE);
 }
 
-fn use_seeded_ec(client: &EcTestClient) -> String {
-    client.set_cookie("ts-ec", SEEDED_EC_ID);
-    normalize_ec_id(SEEDED_EC_ID)
+fn seeded_ec_id(hex_digit: char, suffix: &str) -> String {
+    format!("{}.{suffix}", hex_digit.to_string().repeat(64))
+}
+
+fn use_seeded_ec(client: &EcTestClient, ec_id: &str) -> String {
+    client.set_cookie("ts-ec", ec_id);
+    normalize_ec_id(ec_id)
 }
 
 /// Full lifecycle: seeded EC → batch sync → identify (Bearer auth) with scoped UID.
@@ -518,7 +519,8 @@ fn use_seeded_ec(client: &EcTestClient) -> String {
 fn ec_full_lifecycle(base_url: &str) -> TestResult<()> {
     let client = EcTestClient::new(base_url);
     allow_ec_generation(&client);
-    let ec_id = use_seeded_ec(&client);
+    let seeded_ec_id = seeded_ec_id('a', "test01");
+    let ec_id = use_seeded_ec(&client, &seeded_ec_id);
     log::info!("EC full lifecycle: using seeded EC ID = {ec_id}");
 
     // 2. Batch sync writes partner UID (partner "inttest" is in config)
@@ -576,7 +578,8 @@ fn ec_full_lifecycle(base_url: &str) -> TestResult<()> {
 fn ec_consent_withdrawal(base_url: &str) -> TestResult<()> {
     let client = EcTestClient::new(base_url);
     allow_ec_generation(&client);
-    let ec_id = use_seeded_ec(&client);
+    let seeded_ec_id = seeded_ec_id('b', "test02");
+    let ec_id = use_seeded_ec(&client, &seeded_ec_id);
     log::info!("EC consent withdrawal: using seeded EC = {ec_id}");
 
     // GPC overrides the allow cookie in US-CA, so this is an explicit
@@ -623,7 +626,8 @@ fn ec_identify_without_ec(base_url: &str) -> TestResult<()> {
 fn ec_identify_consent_denied(base_url: &str) -> TestResult<()> {
     let client = EcTestClient::new(base_url);
     allow_ec_generation(&client);
-    let _ec_id = use_seeded_ec(&client);
+    let seeded_ec_id = seeded_ec_id('c', "test03");
+    let _ec_id = use_seeded_ec(&client, &seeded_ec_id);
 
     // Identify with GPC=1 — in the default US-CA test geo, GPC is an explicit
     // denial that must override the allow cookie. Per spec §11.4, consent is
@@ -647,7 +651,8 @@ fn ec_identify_consent_denied(base_url: &str) -> TestResult<()> {
 fn ec_concurrent_partner_syncs(base_url: &str) -> TestResult<()> {
     let client = EcTestClient::new(base_url);
     allow_ec_generation(&client);
-    let ec_id = use_seeded_ec(&client);
+    let seeded_ec_id = seeded_ec_id('d', "test04");
+    let ec_id = use_seeded_ec(&client, &seeded_ec_id);
     log::info!("EC concurrent syncs: using seeded EC = {ec_id}");
 
     // Batch sync both partners (both are pre-configured in trusted-server.toml)
@@ -705,7 +710,8 @@ fn ec_concurrent_partner_syncs(base_url: &str) -> TestResult<()> {
 fn ec_batch_sync_happy_path(base_url: &str) -> TestResult<()> {
     let client = EcTestClient::new(base_url);
     allow_ec_generation(&client);
-    let ec_id = use_seeded_ec(&client);
+    let seeded_ec_id = seeded_ec_id('e', "test05");
+    let ec_id = use_seeded_ec(&client, &seeded_ec_id);
     log::info!("EC batch sync happy path: using seeded ec_id = {ec_id}");
 
     // Batch sync writes a UID for this EC ID (partner "inttest" is in config)

crates/js/lib/build-all.mjs

@@ -13,14 +13,22 @@
  *     names to include in the bundle (e.g. "rubicon,appnexus,openx").
  *     Each name must have a corresponding {name}BidAdapter.js module in
  *     the prebid.js package. Default: "rubicon".
+ *
+ *   TSJS_PREBID_USER_ID_MODULES — Ignored for production builds. User ID
+ *     modules are selected from src/integrations/prebid/user_id_modules.json
+ *     so attested bundles are deterministic. For local experiments only, use
+ *     TSJS_PREBID_USER_ID_MODULES_DEV_OVERRIDE.
  */
 
+import crypto from 'node:crypto';
 import fs from 'node:fs';
+import { createRequire } from 'node:module';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { build } from 'vite';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
 const srcDir = path.resolve(__dirname, 'src');
 const distDir = path.resolve(__dirname, '..', 'dist');
 const integrationsDir = path.join(srcDir, 'integrations');
@@ -30,10 +38,27 @@ const integrationsDir = path.join(srcDir, 'integrations');
 // ---------------------------------------------------------------------------
 
 const DEFAULT_PREBID_ADAPTERS = 'rubicon';
-const ADAPTERS_FILE = path.join(
+const ADAPTERS_FILE = path.join(integrationsDir, 'prebid', '_adapters.generated.ts');
+const USER_IDS_FILE = path.join(integrationsDir, 'prebid', '_user_ids.generated.ts');
+
+const USER_ID_REGISTRY_FILE = path.join(integrationsDir, 'prebid', 'user_id_modules.json');
+const USER_IDS_MANIFEST_FILE = path.join(distDir, 'prebid-user-id-modules.json');
+const LIVE_INTENT_SHIM_ALIAS = 'prebid.js/modules/liveIntentIdSystem.js';
+const PREBID_PACKAGE_DIR = path.join(__dirname, 'node_modules', 'prebid.js');
+const PREBID_LIVE_INTENT_STANDARD = path.join(
+  PREBID_PACKAGE_DIR,
+  'dist',
+  'src',
+  'libraries',
+  'liveIntentId',
+  'idSystem.js'
+);
+const PREBID_GLOBAL_MODULE = path.join(PREBID_PACKAGE_DIR, 'dist', 'src', 'src', 'prebidGlobal.js');
+const LIVE_INTENT_SHIM = path.join(
   integrationsDir,
   'prebid',
-  '_adapters.generated.ts',
+  'prebid_modules',
+  'liveIntentIdSystem.ts'
 );
 
 /**
@@ -53,17 +78,12 @@ function generatePrebidAdapters() {
   if (names.length === 0) {
     console.warn(
       '[build-all] TSJS_PREBID_ADAPTERS is empty, falling back to default:',
-      DEFAULT_PREBID_ADAPTERS,
+      DEFAULT_PREBID_ADAPTERS
     );
     names.push(DEFAULT_PREBID_ADAPTERS);
   }
 
-  const modulesDir = path.join(
-    __dirname,
-    'node_modules',
-    'prebid.js',
-    'modules',
-  );
+  const modulesDir = path.join(__dirname, 'node_modules', 'prebid.js', 'modules');
 
   // Validate each adapter and build import lines
   const imports = [];
@@ -72,7 +92,7 @@ function generatePrebidAdapters() {
     const modulePath = path.join(modulesDir, moduleFile);
     if (!fs.existsSync(modulePath)) {
       console.error(
-        `[build-all] WARNING: Prebid adapter "${name}" not found (expected ${moduleFile}), skipping`,
+        `[build-all] WARNING: Prebid adapter "${name}" not found (expected ${moduleFile}), skipping`
       );
       continue;
     }
@@ -81,7 +101,7 @@ function generatePrebidAdapters() {
 
   if (imports.length === 0) {
     console.error(
-      '[build-all] WARNING: No valid Prebid adapters found, bundle will have no client-side adapters',
+      '[build-all] WARNING: No valid Prebid adapters found, bundle will have no client-side adapters'
     );
   }
 
@@ -100,18 +120,133 @@ function generatePrebidAdapters() {
   fs.writeFileSync(ADAPTERS_FILE, content);
 
   const adapterNames = names.filter((name) =>
-    fs.existsSync(path.join(modulesDir, `${name}BidAdapter.js`)),
+    fs.existsSync(path.join(modulesDir, `${name}BidAdapter.js`))
   );
   console.log('[build-all] Prebid adapters:', adapterNames);
 }
 
+function readUserIdRegistry() {
+  return JSON.parse(fs.readFileSync(USER_ID_REGISTRY_FILE, 'utf8'));
+}
+
+function requireExistingFile(filePath, description) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`[build-all] Missing ${description}: ${filePath}`);
+  }
+}
+
+function prebidPackageVersion() {
+  const packageJsonPath = path.join(PREBID_PACKAGE_DIR, 'package.json');
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+  return packageJson.version;
+}
+
+function sourceToModuleMap(entries) {
+  const map = {};
+  for (const entry of entries) {
+    for (const source of entry.eidSources ?? []) {
+      map[source] = entry.moduleName;
+    }
+  }
+  return map;
+}
+
+function validateUserIdImport(entry) {
+  requireExistingFile(LIVE_INTENT_SHIM, 'LiveIntent ESM shim');
+  requireExistingFile(PREBID_LIVE_INTENT_STANDARD, 'Prebid LiveIntent standard ESM module');
+  requireExistingFile(PREBID_GLOBAL_MODULE, 'Prebid global module');
+
+  if (entry.moduleName === 'liveIntentIdSystem') {
+    return;
+  }
+
+  try {
+    require.resolve(entry.importPath, { paths: [__dirname] });
+  } catch (error) {
+    throw new Error(
+      `[build-all] Required Prebid user ID module "${entry.moduleName}" could not be resolved from ${entry.importPath}: ${error.message}`
+    );
+  }
+}
+
+/**
+ * Generate `_user_ids.generated.ts` with deterministic User ID imports.
+ *
+ * Production builds intentionally ignore TSJS_PREBID_USER_ID_MODULES so the
+ * attested JS artifact does not vary per publisher. A dev-only override exists
+ * for local experiments and should not be used for trusted deployments.
+ */
+function generatePrebidUserIdModules() {
+  const registry = readUserIdRegistry();
+  const entriesByModule = new Map(registry.modules.map((entry) => [entry.moduleName, entry]));
+  const override = process.env.TSJS_PREBID_USER_ID_MODULES_DEV_OVERRIDE;
+  const moduleNames = override
+    ? override
+        .split(',')
+        .map((s) => s.trim())
+        .filter(Boolean)
+    : registry.defaultPreset;
+
+  if (process.env.TSJS_PREBID_USER_ID_MODULES && !override) {
+    console.warn(
+      '[build-all] TSJS_PREBID_USER_ID_MODULES is ignored for deterministic attested builds. ' +
+        'Use TSJS_PREBID_USER_ID_MODULES_DEV_OVERRIDE only for local experiments.'
+    );
+  }
+
+  if (override) {
+    console.warn(
+      '[build-all] WARNING: using TSJS_PREBID_USER_ID_MODULES_DEV_OVERRIDE. ' +
+        'This changes the Prebid bundle and breaks production attestation assumptions.'
+    );
+  }
+
+  const selectedEntries = moduleNames.map((moduleName) => {
+    const entry = entriesByModule.get(moduleName);
+    if (!entry) {
+      throw new Error(`[build-all] Unknown Prebid user ID module in preset: ${moduleName}`);
+    }
+    validateUserIdImport(entry);
+    return entry;
+  });
+
+  const imports = selectedEntries.map((entry) => `import '${entry.importPath}';`);
+
+  const content = [
+    '// Auto-generated by build-all.mjs — manual edits will be overwritten at build time.',
+    '//',
+    '// Deterministic Prebid.js user ID module preset for attested builds.',
+    '// TSJS_PREBID_USER_ID_MODULES is intentionally ignored in production builds.',
+    '// Use TSJS_PREBID_USER_ID_MODULES_DEV_OVERRIDE only for local experiments.',
+    `// Modules: ${moduleNames.join(', ')}`,
+    '',
+    ...imports,
+    '',
+  ].join('\n');
+
+  fs.writeFileSync(USER_IDS_FILE, content);
+
+  const manifest = {
+    prebidVersion: prebidPackageVersion(),
+    deterministic: !override,
+    modules: moduleNames,
+    sourceToModule: sourceToModuleMap(registry.modules),
+    generatedFileHash: crypto.createHash('sha256').update(content).digest('hex'),
+  };
+
+  console.log('[build-all] Prebid user ID modules:', moduleNames);
+  return manifest;
+}
+
 generatePrebidAdapters();
+const prebidUserIdManifest = generatePrebidUserIdModules();
 
 // ---------------------------------------------------------------------------
 
 // Clean dist directory
 fs.rmSync(distDir, { recursive: true, force: true });
 fs.mkdirSync(distDir, { recursive: true });
+fs.writeFileSync(USER_IDS_MANIFEST_FILE, `${JSON.stringify(prebidUserIdManifest, null, 2)}\n`);
 
 // Discover integration modules: directories in src/integrations/ with index.ts
 const integrationModules = fs.existsSync(integrationsDir)
@@ -120,8 +255,7 @@ const integrationModules = fs.existsSync(integrationsDir)
       .filter((name) => {
         const fullPath = path.join(integrationsDir, name);
         return (
-          fs.statSync(fullPath).isDirectory() &&
-          fs.existsSync(path.join(fullPath, 'index.ts'))
+          fs.statSync(fullPath).isDirectory() && fs.existsSync(path.join(fullPath, 'index.ts'))
         );
       })
       .sort()
@@ -139,11 +273,15 @@ async function buildModule(name, entryPath) {
     root: __dirname,
     resolve: {
       alias: {
+        [LIVE_INTENT_SHIM_ALIAS]: LIVE_INTENT_SHIM,
+        'prebid.js/modules/liveIntentIdSystem': LIVE_INTENT_SHIM,
+        'tsjs-prebid/liveIntentIdSystemStandard': PREBID_LIVE_INTENT_STANDARD,
+        'tsjs-prebid/prebidGlobal': PREBID_GLOBAL_MODULE,
         // prebid.js doesn't expose src/adapterManager.js via its package
         // "exports" map, but we need it for client-side bidder validation.
         'prebid.js/src/adapterManager.js': path.resolve(
           __dirname,
-          'node_modules/prebid.js/dist/src/src/adapterManager.js',
+          'node_modules/prebid.js/dist/src/src/adapterManager.js'
         ),
       },
     },
@@ -176,9 +314,7 @@ async function buildModule(name, entryPath) {
 await buildModule('core', path.join(srcDir, 'core', 'index.ts'));
 
 await Promise.all(
-  integrationModules.map((name) =>
-    buildModule(name, path.join(integrationsDir, name, 'index.ts')),
-  ),
+  integrationModules.map((name) => buildModule(name, path.join(integrationsDir, name, 'index.ts')))
 );
 
 // List all built files