From 87ccc909d0d6d728a7694b218c47102ef5086000 Mon Sep 17 00:00:00 2001 From: rasmi Date: Thu, 30 Oct 2025 13:58:00 -0400 Subject: [PATCH 1/5] Added initial experiment variable functionality. --- functions/src/index.ts | 1 + functions/src/variables.endpoints.ts | 293 ++++++++++++++++++++++++ utils/package-lock.json | 17 ++ utils/package.json | 2 + utils/src/experiment.ts | 6 +- utils/src/index.ts | 5 + utils/src/variables.template.test.ts | 154 +++++++++++++ utils/src/variables.template.ts | 329 +++++++++++++++++++++++++++ utils/src/variables.ts | 168 ++++++++++++++ utils/src/variables.validation.ts | 146 ++++++++++++ 10 files changed, 1120 insertions(+), 1 deletion(-) create mode 100644 functions/src/variables.endpoints.ts create mode 100644 utils/src/variables.template.test.ts create mode 100644 utils/src/variables.template.ts create mode 100644 utils/src/variables.ts create mode 100644 utils/src/variables.validation.ts diff --git a/functions/src/index.ts b/functions/src/index.ts index a24411dc3..c77a299e3 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -11,6 +11,7 @@ export * from './cohort.endpoints'; export * from './experiment.endpoints'; export * from './mediator.endpoints'; export * from './participant.endpoints'; +export * from './variables.endpoints'; export * from './stages/asset_allocation.endpoints'; export * from './stages/chat.endpoints'; diff --git a/functions/src/variables.endpoints.ts b/functions/src/variables.endpoints.ts new file mode 100644 index 000000000..bf9de2068 --- /dev/null +++ b/functions/src/variables.endpoints.ts @@ -0,0 +1,293 @@ +import * as functions from 'firebase-functions'; +import {onCall} from 'firebase-functions/v2/https'; +import { + InitializeVariableCohortsData, + GetParticipantVariablesData, + ExperimentVariables, + validateInitialCohort, + getAssignableCohorts, + resolveParticipantVariables, + createCohortConfig, + createMetadataConfig, +} from '@deliberation-lab/utils'; +import {app} from './app'; +import {createCohortInternal} from './cohort.utils'; +import {Value} from '@sinclair/typebox/value'; +import { + getFirestoreExperiment, + getFirestoreParticipant, +} from './utils/firestore'; +import {AuthGuard} from './utils/auth-guard'; +import {prettyPrintErrors} from './utils/validation'; + +/** + * Initialize variable cohorts for an experiment + * Creates cohorts based on the variable configuration + */ +export const initializeVariableCohorts = onCall( + { + cors: true, + region: 'us-central1', + }, + async (request) => { + // Validate request data + const {data} = request; + if (!Value.Check(InitializeVariableCohortsData, data)) { + prettyPrintErrors(Value.Errors(InitializeVariableCohortsData, data)); + throw new functions.https.HttpsError( + 'invalid-argument', + 'Invalid request data', + ); + } + + // Check authentication + await AuthGuard.isExperimenter(request); + const uid = request.auth?.uid; + + const {experimentId, variables, replaceExisting = false} = data; + + // Validate variable configuration + if (!validateInitialCohort(variables)) { + throw new functions.https.HttpsError( + 'invalid-argument', + 'Only one cohort can be marked as initial cohort', + ); + } + + const assignableCohorts = getAssignableCohorts(variables); + if (assignableCohorts.length === 0) { + throw new functions.https.HttpsError( + 'invalid-argument', + 'At least one non-initial cohort is required for assignment', + ); + } + + try { + // Get the experiment to ensure it exists and user has access + const experiment = await getFirestoreExperiment(experimentId); + if (!experiment) { + throw new functions.https.HttpsError( + 'not-found', + 'Experiment not found', + ); + } + + // Check if user has write access + const userEmail = request.auth?.token?.email || ''; + if ( + experiment.permissions.visibility === 'private' && + experiment.metadata.creator !== userEmail && + !experiment.permissions.readers.includes(userEmail) + ) { + throw new functions.https.HttpsError( + 'permission-denied', + 'You do not have permission to modify this experiment', + ); + } + + // Start a transaction to ensure atomic updates + await app.firestore().runTransaction(async (transaction) => { + // If replaceExisting, delete existing cohorts first + if (replaceExisting) { + const existingCohorts = await app + .firestore() + .collection('experiments') + .doc(experimentId) + .collection('cohorts') + .get(); + + // Check if any cohort has participants + for (const cohortDoc of existingCohorts.docs) { + const participantsSnapshot = await app + .firestore() + .collection('experiments') + .doc(experimentId) + .collection('participants') + .where('currentCohortId', '==', cohortDoc.id) + .limit(1) + .get(); + + if (!participantsSnapshot.empty) { + throw new functions.https.HttpsError( + 'failed-precondition', + `Cannot delete cohort ${cohortDoc.data().metadata?.name || cohortDoc.id} - it has participants`, + ); + } + } + + // Delete existing cohorts + for (const cohortDoc of existingCohorts.docs) { + transaction.delete(cohortDoc.ref); + } + } + + // Create new cohorts based on variable configuration + const cohortIds: Record = {}; + + for (const [cohortName, cohortConfig] of Object.entries( + variables.cohorts, + )) { + // Create cohort configuration + const cohort = createCohortConfig({ + metadata: createMetadataConfig({ + name: cohortName, + publicName: cohortName, + description: + cohortConfig.description || `Variable cohort: ${cohortName}`, + tags: cohortConfig.isInitialCohort + ? ['initial', 'variables'] + : ['variables'], + }), + participantConfig: experiment.defaultCohortConfig, + }); + + // Store cohort using internal utility + await createCohortInternal(transaction, experimentId, cohort); + + cohortIds[cohortName] = cohort.id; + } + + // Update experiment with variables configuration + const experimentRef = app + .firestore() + .collection('experiments') + .doc(experimentId); + + transaction.update(experimentRef, { + variables: variables, + variableCohortIds: cohortIds, + dateEdited: new Date(), + }); + }); + + return { + success: true, + message: `Created ${Object.keys(variables.cohorts).length} variable cohorts`, + }; + } catch (error) { + console.error('Error initializing variable cohorts:', error); + + if (error instanceof functions.https.HttpsError) { + throw error; + } + + throw new functions.https.HttpsError( + 'internal', + 'Failed to initialize variable cohorts', + ); + } + }, +); + +/** + * Get resolved variables for a participant based on their cohort + */ +export const getParticipantVariables = onCall( + { + cors: true, + region: 'us-central1', + }, + async (request) => { + // Validate request data + const {data} = request; + if (!Value.Check(GetParticipantVariablesData, data)) { + prettyPrintErrors(Value.Errors(GetParticipantVariablesData, data)); + throw new functions.https.HttpsError( + 'invalid-argument', + 'Invalid request data', + ); + } + + // Check authentication - participant must be signed in + if (!request.auth?.uid) { + throw new functions.https.HttpsError( + 'unauthenticated', + 'User must be authenticated', + ); + } + const uid = request.auth.uid; + + const {experimentId, participantId} = data; + + try { + // Get experiment with variables + const experiment = await getFirestoreExperiment(experimentId); + if (!experiment) { + throw new functions.https.HttpsError( + 'not-found', + 'Experiment not found', + ); + } + + // Check if experiment has variables configured + if (!experiment.variables) { + return {variables: {}}; // No variables configured + } + + // Get participant profile + const participant = await getFirestoreParticipant( + experimentId, + participantId, + ); + if (!participant) { + throw new functions.https.HttpsError( + 'not-found', + 'Participant not found', + ); + } + + // Verify the participant matches the authenticated user + if (participant.privateId !== uid) { + throw new functions.https.HttpsError( + 'permission-denied', + 'You can only access your own variables', + ); + } + + // Get the participant's current cohort + const cohortId = participant.currentCohortId; + if (!cohortId) { + // Participant not in a cohort yet, return defaults + return {variables: {}}; + } + + // Get cohort configuration + const cohortDoc = await app + .firestore() + .collection('experiments') + .doc(experimentId) + .collection('cohorts') + .doc(cohortId) + .get(); + + if (!cohortDoc.exists) { + throw new functions.https.HttpsError('not-found', 'Cohort not found'); + } + + const cohort = cohortDoc.data(); + const cohortName = cohort?.metadata?.name || ''; + + // Resolve variables for this cohort + const resolvedVariables = resolveParticipantVariables( + experiment.variables as ExperimentVariables, + cohortName, + ); + + return { + variables: resolvedVariables, + cohortName: cohortName, + }; + } catch (error) { + console.error('Error getting participant variables:', error); + + if (error instanceof functions.https.HttpsError) { + throw error; + } + + throw new functions.https.HttpsError( + 'internal', + 'Failed to get participant variables', + ); + } + }, +); diff --git a/utils/package-lock.json b/utils/package-lock.json index caa42c689..c755027b6 100644 --- a/utils/package-lock.json +++ b/utils/package-lock.json @@ -8,7 +8,9 @@ "name": "@deliberation-lab/utils", "version": "1.0.0", "dependencies": { + "@types/mustache": "^4.2.6", "crypto-browserify": "^3.12.0", + "mustache": "^4.2.0", "process": "^0.11.10", "uuidv4": "^6.2.13" }, @@ -4195,6 +4197,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/mustache": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@types/mustache/-/mustache-4.2.6.tgz", + "integrity": "sha512-t+8/QWTAhOFlrF1IVZqKnMRJi84EgkIK5Kh0p2JV4OLywUvCwJPFxbJAl7XAow7DVIHsF+xW9f1MVzg0L6Szjw==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "20.12.8", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.8.tgz", @@ -7911,6 +7919,15 @@ "dev": true, "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", diff --git a/utils/package.json b/utils/package.json index 839227a23..2c20bc659 100644 --- a/utils/package.json +++ b/utils/package.json @@ -33,7 +33,9 @@ "typescript": "^5.4.5" }, "dependencies": { + "@types/mustache": "^4.2.6", "crypto-browserify": "^3.12.0", + "mustache": "^4.2.0", "process": "^0.11.10", "uuidv4": "^6.2.13" }, diff --git a/utils/src/experiment.ts b/utils/src/experiment.ts index 7337f5f17..e9420a6b8 100644 --- a/utils/src/experiment.ts +++ b/utils/src/experiment.ts @@ -7,6 +7,7 @@ import { generateId, } from './shared'; import {StageConfig} from './stages/stage'; +import {ExperimentVariables} from './variables'; /** Experiment types and functions. */ @@ -34,8 +35,9 @@ import {StageConfig} from './stages/stage'; * VERSION 16 - switch to new mediator workflow including updated ChatMessage * VERSION 17 - add structured output config to agent prompt configs * VERSION 18 - add agent participant config to ParticipantProfileExtended + * VERSION 19 - add variables field to Experiment for experiment variables */ -export const EXPERIMENT_VERSION_ID = 18; +export const EXPERIMENT_VERSION_ID = 19; /** Experiment. */ export interface Experiment { @@ -48,6 +50,7 @@ export interface Experiment { prolificConfig: ProlificConfig; stageIds: string[]; // Ordered list of stage IDs cohortLockMap: Record; // maps cohort ID to is locked + variables?: ExperimentVariables; // Optional experiment variables configuration } /** Experiment template (used to load experiments). */ @@ -101,6 +104,7 @@ export function createExperimentConfig( prolificConfig: config.prolificConfig ?? createProlificConfig(), stageIds: stages.map((stage) => stage.id), cohortLockMap: config.cohortLockMap ?? {}, + variables: config.variables, // Optional, don't set default }; } diff --git a/utils/src/index.ts b/utils/src/index.ts index 6808ace91..637a70bc8 100644 --- a/utils/src/index.ts +++ b/utils/src/index.ts @@ -124,3 +124,8 @@ export * from './utils/condition'; export * from './utils/object.utils'; export * from './utils/random.utils'; export * from './utils/string.utils'; + +// Variables +export * from './variables'; +export * from './variables.template'; +export * from './variables.validation'; diff --git a/utils/src/variables.template.test.ts b/utils/src/variables.template.test.ts new file mode 100644 index 000000000..0147d238b --- /dev/null +++ b/utils/src/variables.template.test.ts @@ -0,0 +1,154 @@ +import { + resolveTemplate, + extractVariableReferences, + validateTemplateVariables, +} from './variables.template'; +import {VariableDefinition} from './variables'; + +describe('Mustache Template Resolution', () => { + describe('resolveTemplate', () => { + it('should substitute simple variables', () => { + const template = 'Hello {{name}}, welcome to {{place}}!'; + const variables = {name: 'Alice', place: 'Wonderland'}; + const result = resolveTemplate(template, variables); + expect(result).toBe('Hello Alice, welcome to Wonderland!'); + }); + + it('should handle nested object properties', () => { + const template = 'Policy: {{policy.name}} costs ${{policy.cost}}'; + const variables = { + policy: { + name: 'Universal Healthcare', + cost: 1000000, + }, + }; + const result = resolveTemplate(template, variables); + expect(result).toBe('Policy: Universal Healthcare costs $1000000'); + }); + + it('should escape HTML by default', () => { + const template = 'Message: {{message}}'; + const variables = {message: ''}; + const result = resolveTemplate(template, variables); + // Mustache escapes forward slash as / + expect(result).toBe( + 'Message: <script>alert("XSS")</script>', + ); + }); + + it('should allow unescaped HTML with triple mustache', () => { + const template = 'HTML: {{{htmlContent}}}'; + const variables = {htmlContent: 'Bold Text'}; + const result = resolveTemplate(template, variables); + expect(result).toBe('HTML: Bold Text'); + }); + + it('should use default values for missing variables', () => { + const template = 'Color: {{color}}, Size: {{size}}'; + const variables = {color: 'blue'}; + const defaults = {color: 'red', size: 'medium'}; + const result = resolveTemplate(template, variables, defaults); + expect(result).toBe('Color: blue, Size: medium'); + }); + + it('should handle conditional sections', () => { + const template = '{{#showMessage}}Important: {{message}}{{/showMessage}}'; + const variables = {showMessage: true, message: 'Hello'}; + const result = resolveTemplate(template, variables); + expect(result).toBe('Important: Hello'); + }); + + it('should handle inverted sections', () => { + const template = '{{^hasData}}No data available{{/hasData}}'; + const variables = {hasData: false}; + const result = resolveTemplate(template, variables); + expect(result).toBe('No data available'); + }); + + it('should handle lists/arrays', () => { + const template = 'Items: {{#items}}- {{name}} {{/items}}'; + const variables = { + items: [{name: 'Item 1'}, {name: 'Item 2'}, {name: 'Item 3'}], + }; + const result = resolveTemplate(template, variables); + expect(result).toBe('Items: - Item 1 - Item 2 - Item 3 '); + }); + }); + + describe('extractVariableReferences', () => { + it('should extract simple variable names', () => { + const template = 'Hello {{name}}, you are {{age}} years old'; + const refs = extractVariableReferences(template); + expect(refs).toEqual(['name', 'age']); + }); + + it('should extract nested properties', () => { + const template = '{{user.name}} lives in {{user.address.city}}'; + const refs = extractVariableReferences(template); + expect(refs).toEqual(['user.name', 'user.address.city']); + }); + + it('should extract from sections', () => { + const template = '{{#section}}{{variable}}{{/section}}'; + const refs = extractVariableReferences(template); + expect(refs).toEqual(['section', 'variable']); + }); + + it('should extract from triple mustache', () => { + const template = '{{{unescapedHtml}}} and {{normalVar}}'; + const refs = extractVariableReferences(template); + expect(refs).toEqual(['unescapedHtml', 'normalVar']); + }); + + it('should remove duplicates', () => { + const template = '{{name}} and {{name}} again, plus {{age}}'; + const refs = extractVariableReferences(template); + expect(refs).toEqual(['name', 'age']); + }); + }); + + describe('validateTemplateVariables', () => { + const definitions: Record = { + name: {type: 'string'}, + age: {type: 'number'}, + policy: { + type: 'object', + schema: { + name: {type: 'string'}, + cost: {type: 'number'}, + }, + }, + }; + + it('should validate defined variables', () => { + const template = 'Hello {{name}}, you are {{age}} years old'; + const result = validateTemplateVariables(template, definitions); + expect(result.valid).toBe(true); + expect(result.missingVariables).toEqual([]); + expect(result.syntaxError).toBeUndefined(); + }); + + it('should detect missing variables', () => { + const template = 'Hello {{name}}, from {{location}}'; + const result = validateTemplateVariables(template, definitions); + expect(result.valid).toBe(false); + expect(result.missingVariables).toEqual(['location']); + expect(result.syntaxError).toBeUndefined(); + }); + + it('should validate nested properties', () => { + const template = 'Policy: {{policy.name}}'; + const result = validateTemplateVariables(template, definitions); + expect(result.valid).toBe(true); + expect(result.missingVariables).toEqual([]); + expect(result.syntaxError).toBeUndefined(); + }); + + it('should detect syntax errors', () => { + const template = 'Unclosed {{#section}} without closing'; + const result = validateTemplateVariables(template, definitions); + expect(result.valid).toBe(false); + expect(result.syntaxError).toBeDefined(); + }); + }); +}); diff --git a/utils/src/variables.template.ts b/utils/src/variables.template.ts new file mode 100644 index 000000000..ca7758c2e --- /dev/null +++ b/utils/src/variables.template.ts @@ -0,0 +1,329 @@ +import Mustache from 'mustache'; +import { + ExperimentVariables, + VariableDefinition, + VariableCohort, + VariableType, + VariableValue, +} from './variables'; + +/** + * Template resolution utilities for experiment variables + * + * IMPLEMENTATION: + * - Uses Mustache for template rendering (lightweight, ~15KB) + * - Supports {{variableName}} and {{object.property}} syntax + * - HTML escaping is automatic by default (use {{{variableName}}} for unescaped) + * - Logic-less templates keep complexity low + * + * MUSTACHE SYNTAX: + * - {{name}} - Variable substitution (HTML escaped) + * - {{{name}}} - Unescaped variable substitution + * - {{#section}}...{{/section}} - Conditional sections (if truthy) + * - {{^section}}...{{/section}} - Inverted sections (if falsy) + * - {{! comment }} - Comments + */ + +// ************************************************************************* // +// Template Resolution +// ************************************************************************* // + +/** + * Extract all variable references from a Mustache template + * Uses Mustache's own parser to get accurate variable names + */ +export function extractVariableReferences(template: string): string[] { + try { + const tokens = Mustache.parse(template); + const references = new Set(); + + // Recursively extract variable names from parsed tokens + function extractFromTokens(tokens: unknown[]): void { + for (const token of tokens) { + // Mustache tokens are tuples with varying lengths depending on token type + // We only care about extracting the type and name fields + if (!Array.isArray(token)) continue; + const [type, name, , , subTokens] = token as [ + string, + string, + number, + number, + unknown[]?, + ]; + + // Token types: 'name' for variables, '#' for sections, '^' for inverted sections + if ( + (type === 'name' || type === '#' || type === '^' || type === '&') && + name + ) { + references.add(name); + } + + // Recursively process sub-tokens in sections + if (subTokens && Array.isArray(subTokens)) { + extractFromTokens(subTokens); + } + } + } + + extractFromTokens(tokens); + return Array.from(references); + } catch (error) { + // If parsing fails, return empty array + console.warn('Failed to parse template for variable extraction:', error); + return []; + } +} + +/** + * Resolve a template string with variable values using Mustache + * + * @param template - The template string with {{variables}} + * @param variables - Variable values to substitute + * @param defaultValues - Default values for missing variables + * @param _options - Options (kept for API compatibility, escaping is handled by Mustache) + */ +export function resolveTemplate( + template: string, + variables: Record, + defaultValues?: Record, + _options: {escapeHtml?: boolean} = {}, +): string { + // Merge variables with defaults + const context = { + ...defaultValues, + ...variables, + }; + + // Configure Mustache to not escape HTML if specified + // By default, Mustache escapes HTML automatically with {{variable}} + // Use {{{variable}}} in templates for unescaped values + + try { + // Parse the template first to catch syntax errors + Mustache.parse(template); + + // Render the template with the context + return Mustache.render(template, context); + } catch (error) { + // If there's an error parsing/rendering, return the original template + // This maintains backward compatibility with templates that might not be valid Mustache + console.warn('Failed to render Mustache template:', error); + return template; + } +} + +// ************************************************************************* // +// Variable Resolution for Participants +// ************************************************************************* // + +/** + * Resolve variables for a participant based on their cohort + * Returns merged cohort variables with defaults + */ +export function resolveParticipantVariables( + experimentVariables: ExperimentVariables, + cohortName: string, +): Record { + const cohort = experimentVariables.cohorts[cohortName]; + const defaults = getDefaultValues(experimentVariables.definitions); + + // Merge defaults with cohort variables (cohort overrides defaults) + return cohort ? {...defaults, ...cohort.variables} : defaults; +} + +/** + * Get default values for all defined variables + * Used as fallback when variables are not specified + */ +function getDefaultValues( + definitions: Record, +): Record { + const defaults: Record = {}; + + for (const [name, definition] of Object.entries(definitions)) { + defaults[name] = definition.defaultValue ?? getTypeDefault(definition.type); + } + + return defaults; +} + +/** Get sensible default value for a type */ +function getTypeDefault(type: VariableType): VariableValue { + switch (type) { + case 'string': + return ''; + case 'number': + return 0; + case 'boolean': + return false; + case 'object': + return {}; + } +} + +// ************************************************************************* // +// Validation +// ************************************************************************* // + +/** + * Validate that a template's variable references are defined + * Also validates that the template is valid Mustache syntax + */ +export function validateTemplateVariables( + template: string, + definitions: Record, +): {valid: boolean; missingVariables: string[]; syntaxError?: string} { + try { + // First, check if template is valid Mustache syntax + Mustache.parse(template); + + // Extract variable references + const references = extractVariableReferences(template); + const missingVariables: string[] = []; + + for (const ref of references) { + const baseName = ref.split('.')[0]; + if (!(baseName in definitions)) { + missingVariables.push(baseName); + } + } + + return { + valid: missingVariables.length === 0, + missingVariables: [...new Set(missingVariables)], + }; + } catch (error) { + // Template has syntax errors + return { + valid: false, + missingVariables: [], + syntaxError: + error instanceof Error ? error.message : 'Invalid template syntax', + }; + } +} + +/** Validate that cohort variables match their definitions */ +export function validateCohortVariables( + cohort: VariableCohort, + definitions: Record, +): {valid: boolean; errors: string[]} { + const errors: string[] = []; + + // Check each variable in the cohort + for (const [name, value] of Object.entries(cohort.variables)) { + const definition = definitions[name]; + + if (!definition) { + errors.push(`Variable "${name}" is not defined`); + continue; + } + + // Type validation + const actualType = Array.isArray(value) ? 'array' : typeof value; + + if (definition.type === 'object') { + if (actualType !== 'object' || value === null) { + errors.push( + `Variable "${name}" should be an object but got ${actualType}`, + ); + } else if ( + definition.schema && + typeof value === 'object' && + value !== null + ) { + // Validate object properties against schema + const objValue = value as Record; + for (const [prop, propSchema] of Object.entries(definition.schema)) { + if (prop in objValue) { + const propType = typeof objValue[prop]; + if (propType !== propSchema.type) { + errors.push( + `Property "${name}.${prop}" should be ${propSchema.type} but got ${propType}`, + ); + } + } + } + } + } else if (actualType !== definition.type) { + errors.push( + `Variable "${name}" should be ${definition.type} but got ${actualType}`, + ); + } + } + + // Check for required variables (those without defaults) + for (const [name, definition] of Object.entries(definitions)) { + if (definition.defaultValue === undefined && !(name in cohort.variables)) { + errors.push(`Required variable "${name}" is missing`); + } + } + + return { + valid: errors.length === 0, + errors, + }; +} + +// ************************************************************************* // +// Template Preview +// ************************************************************************* // + +/** + * Generate preview of template with sample or actual cohort values + * Useful for showing experimenters how their templates will look + */ +export function previewTemplate( + template: string, + definitions: Record, + cohortVariables?: Record, +): string { + // Use provided cohort variables or generate sample values + const variables = cohortVariables || generateSampleValues(definitions); + return resolveTemplate(template, variables); +} + +/** + * Generate sample values for preview purposes + * Creates readable placeholder values for each variable type + */ +function generateSampleValues( + definitions: Record, +): Record { + const samples: Record = {}; + + for (const [name, definition] of Object.entries(definitions)) { + // Use default if specified, otherwise generate sample + if (definition.defaultValue !== undefined) { + samples[name] = definition.defaultValue; + } else if (definition.type === 'object' && definition.schema) { + // For objects, generate sample for each property + const obj: Record = {}; + for (const [prop, propSchema] of Object.entries(definition.schema)) { + obj[prop] = getSampleValue(prop, propSchema.type); + } + samples[name] = obj; + } else { + samples[name] = getSampleValue(name, definition.type); + } + } + + return samples; +} + +/** Get a sample value for a given type */ +function getSampleValue(name: string, type: string): VariableValue { + switch (type) { + case 'string': + return `[${name}]`; + case 'number': + return 42; + case 'boolean': + return true; + case 'object': + return {}; + default: + return null; + } +} diff --git a/utils/src/variables.ts b/utils/src/variables.ts new file mode 100644 index 000000000..b1191a81e --- /dev/null +++ b/utils/src/variables.ts @@ -0,0 +1,168 @@ +import {Condition} from './utils/condition'; +import {SeedStrategy} from './utils/random.utils'; + +/** Experiment Variables - Defines experimental conditions and their values */ + +// ************************************************************************* // +// Variable Type Definitions +// ************************************************************************* // + +export type VariableType = 'string' | 'number' | 'boolean' | 'object'; + +/** Type for variable values (can be primitives, objects, or arrays) */ +export type VariableValue = + | string + | number + | boolean + | Record + | unknown[] + | null + | undefined; + +/** Schema for object variable properties */ +export interface VariablePropertySchema { + type: 'string' | 'number' | 'boolean'; + description?: string; +} + +/** Definition of a single variable */ +export interface VariableDefinition { + type: VariableType; + description?: string; + defaultValue?: VariableValue; + /** For object types, defines the structure of properties */ + schema?: Record; +} + +// ************************************************************************* // +// Variable Assignment +// ************************************************************************* // + +/** Method for assigning participants to cohorts */ +export type AssignmentMethod = 'random' | 'manual' | 'conditional'; + +/** Configuration for random assignment */ +export interface RandomAssignmentConfig { + seedStrategy: SeedStrategy; + customSeed?: string; + /** Optional weights per cohort (defaults to equal distribution) */ + weights?: Record; +} + +/** Rule for conditional assignment */ +export interface ConditionalAssignmentRule { + condition: Condition; + cohort: string; +} + +/** Configuration for conditional assignment */ +export interface ConditionalAssignmentConfig { + /** Maps conditions to cohorts */ + rules: ConditionalAssignmentRule[]; + /** Fallback if no condition matches */ + defaultCohort?: string; +} + +/** Assignment strategy for distributing participants */ +export interface AssignmentConfig { + method: AssignmentMethod; + /** Configuration for random assignment */ + random?: RandomAssignmentConfig; + /** Configuration for conditional assignment */ + conditional?: ConditionalAssignmentConfig; +} + +// ************************************************************************* // +// Variable Cohort +// ************************************************************************* // + +/** Configuration for a variable cohort */ +export interface VariableCohort { + description?: string; + /** Mark as initial cohort (max 1 per experiment) */ + isInitialCohort?: boolean; + /** Variable values for this cohort */ + variables: Record; +} + +// ************************************************************************* // +// Main Variables Configuration +// ************************************************************************* // + +/** Complete experiment variables configuration */ +export interface ExperimentVariables { + /** Define available variables and their types */ + definitions: Record; + + /** Define cohorts and their variable values */ + cohorts: Record; + + /** Assignment strategy for distributing participants */ + assignment: AssignmentConfig; +} + +// ************************************************************************* // +// Helper Functions +// ************************************************************************* // + +/** Create a default experiment variables configuration */ +export function createExperimentVariables(): ExperimentVariables { + return { + definitions: {}, + cohorts: {}, + assignment: { + method: 'manual', + }, + }; +} + +/** Create a variable definition */ +export function createVariableDefinition( + type: VariableType, + config: Partial = {}, +): VariableDefinition { + return { + type, + description: config.description, + defaultValue: config.defaultValue, + schema: config.schema, + }; +} + +/** Create a variable cohort */ +export function createVariableCohort( + config: Partial = {}, +): VariableCohort { + return { + description: config.description, + isInitialCohort: config.isInitialCohort ?? false, + variables: config.variables ?? {}, + }; +} + +/** Validate that only one cohort is marked as initial */ +export function validateInitialCohort(variables: ExperimentVariables): boolean { + const initialCohorts = Object.values(variables.cohorts).filter( + (c) => c.isInitialCohort, + ); + return initialCohorts.length <= 1; +} + +/** Get the initial cohort name if one exists */ +export function getInitialCohortName( + variables: ExperimentVariables, +): string | null { + for (const [name, cohort] of Object.entries(variables.cohorts)) { + if (cohort.isInitialCohort) { + return name; + } + } + return null; +} + +/** Get cohorts available for assignment (excludes initial cohort) */ +export function getAssignableCohorts(variables: ExperimentVariables): string[] { + return Object.entries(variables.cohorts) + .filter(([_, cohort]) => !cohort.isInitialCohort) + .map(([name, _]) => name); +} diff --git a/utils/src/variables.validation.ts b/utils/src/variables.validation.ts new file mode 100644 index 000000000..3080f34a1 --- /dev/null +++ b/utils/src/variables.validation.ts @@ -0,0 +1,146 @@ +import {Type, type Static} from '@sinclair/typebox'; +import {SeedStrategy} from './utils/random.utils'; + +/** Shorthand for strict TypeBox object validation */ +const strict = {additionalProperties: false} as const; + +// ************************************************************************* // +// Variable Value Schema +// ************************************************************************* // + +/** TypeBox schema for VariableValue type */ +export const VariableValueData = Type.Union([ + Type.String(), + Type.Number(), + Type.Boolean(), + Type.Record(Type.String(), Type.Unknown()), + Type.Array(Type.Unknown()), + Type.Null(), + Type.Undefined(), +]); + +// ************************************************************************* // +// Variable Definition Schema +// ************************************************************************* // + +const VariablePropertySchemaData = Type.Object( + { + type: Type.Union([ + Type.Literal('string'), + Type.Literal('number'), + Type.Literal('boolean'), + ]), + description: Type.Optional(Type.String()), + }, + strict, +); + +const VariableDefinitionData = Type.Object( + { + type: Type.Union([ + Type.Literal('string'), + Type.Literal('number'), + Type.Literal('boolean'), + Type.Literal('object'), + ]), + description: Type.Optional(Type.String()), + defaultValue: Type.Optional(VariableValueData), + schema: Type.Optional( + Type.Record(Type.String(), VariablePropertySchemaData), + ), + }, + strict, +); + +// ************************************************************************* // +// Variable Cohort Schema +// ************************************************************************* // + +const VariableCohortData = Type.Object( + { + description: Type.Optional(Type.String()), + isInitialCohort: Type.Optional(Type.Boolean()), + variables: Type.Record(Type.String(), VariableValueData), + }, + strict, +); + +// ************************************************************************* // +// Assignment Configuration Schema +// ************************************************************************* // + +const RandomAssignmentConfigData = Type.Object( + { + seedStrategy: Type.Union([ + Type.Literal(SeedStrategy.EXPERIMENT), + Type.Literal(SeedStrategy.COHORT), + Type.Literal(SeedStrategy.PARTICIPANT), + Type.Literal(SeedStrategy.CUSTOM), + ]), + customSeed: Type.Optional(Type.String()), + weights: Type.Optional(Type.Record(Type.String(), Type.Number())), + }, + strict, +); + +const AssignmentConfigData = Type.Object( + { + method: Type.Union([ + Type.Literal('random'), + Type.Literal('manual'), + Type.Literal('conditional'), + ]), + random: Type.Optional(RandomAssignmentConfigData), + // Note: conditional assignment would need Condition schema imported + }, + strict, +); + +// ************************************************************************* // +// Main Experiment Variables Schema +// ************************************************************************* // + +export const ExperimentVariablesData = Type.Object( + { + definitions: Type.Record(Type.String(), VariableDefinitionData), + cohorts: Type.Record(Type.String(), VariableCohortData), + assignment: AssignmentConfigData, + }, + strict, +); + +export type ExperimentVariablesData = Static; + +// ************************************************************************* // +// initializeVariableCohorts endpoint +// ************************************************************************* // + +export const InitializeVariableCohortsData = Type.Object( + { + experimentId: Type.String({minLength: 1}), + variables: ExperimentVariablesData, + // Whether to delete existing cohorts before creating new ones + replaceExisting: Type.Optional(Type.Boolean()), + }, + strict, +); + +export type InitializeVariableCohortsData = Static< + typeof InitializeVariableCohortsData +>; + +// ************************************************************************* // +// getParticipantVariables endpoint +// ************************************************************************* // + +export const GetParticipantVariablesData = Type.Object( + { + experimentId: Type.String({minLength: 1}), + participantId: Type.String({minLength: 1}), + }, + strict, +); + +export type GetParticipantVariablesData = Static< + typeof GetParticipantVariablesData +>; From cec8d4b04c5985666fa0f81c93807b794ee1d781 Mon Sep 17 00:00:00 2001 From: rasmi Date: Thu, 30 Oct 2025 14:14:10 -0400 Subject: [PATCH 2/5] Load variables in Info stage. --- frontend/src/components/stages/info_view.ts | 12 +++++++-- frontend/src/services/participant.service.ts | 28 ++++++++++++++++++++ frontend/src/shared/callables.ts | 17 ++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/stages/info_view.ts b/frontend/src/components/stages/info_view.ts index 42d769a5a..3fa976f28 100644 --- a/frontend/src/components/stages/info_view.ts +++ b/frontend/src/components/stages/info_view.ts @@ -7,11 +7,13 @@ import {MobxLitElement} from '@adobe/lit-mobx'; import {CSSResultGroup, html, nothing} from 'lit'; import {customElement, property} from 'lit/decorators.js'; -import {InfoStageConfig} from '@deliberation-lab/utils'; +import {InfoStageConfig, resolveTemplate} from '@deliberation-lab/utils'; import {unsafeHTML} from 'lit/directives/unsafe-html.js'; import {convertMarkdownToHTML} from '../../shared/utils'; import {styles} from './info_view.scss'; +import {core} from '../../core/core'; +import {ParticipantService} from '../../services/participant.service'; /** Info stage view for participants. */ @customElement('info-view') @@ -20,12 +22,18 @@ export class InfoView extends MobxLitElement { @property() stage: InfoStageConfig | null = null; + private readonly participantService = core.getService(ParticipantService); + override render() { if (!this.stage) { return nothing; } - const infoLinesJoined = this.stage?.infoLines.join('\n\n'); + // Resolve templates in info lines using participant variables + const resolvedInfoLines = this.stage.infoLines.map((line) => + resolveTemplate(line, this.participantService.variables), + ); + const infoLinesJoined = resolvedInfoLines.join('\n\n'); return html`
diff --git a/frontend/src/services/participant.service.ts b/frontend/src/services/participant.service.ts index 66de79ac0..cf9a369f5 100644 --- a/frontend/src/services/participant.service.ts +++ b/frontend/src/services/participant.service.ts @@ -19,6 +19,7 @@ import { SurveyStageParticipantAnswer, UnifiedTimestamp, UpdateChatStageParticipantAnswerData, + VariableValue, createChatMessage, createChatStageParticipantAnswer, createParticipantChatMessage, @@ -46,6 +47,7 @@ import {Service} from './service'; import { acceptParticipantCheckCallable, + getParticipantVariablesCallable, acceptParticipantExperimentStartCallable, acceptParticipantTransferCallable, createChatMessageCallable, @@ -106,6 +108,7 @@ export class ParticipantService extends Service { {}; @observable privateChatMap: Record = {}; @observable alertMap: Record = {}; + @observable variables: Record = {}; // Loading @observable unsubscribe: Unsubscribe[] = []; @@ -287,6 +290,9 @@ export class ParticipantService extends Service { ); } + // Load participant variables + await this.loadVariables(); + // Load profile to participant answer service this.sp.participantAnswerService.setProfile(this.profile); // Set current stage (use undefined if experiment not started) @@ -333,6 +339,28 @@ export class ParticipantService extends Service { this.loadAlertMessages(); } + /** Load participant variables from the backend. */ + private async loadVariables() { + if (!this.experimentId || !this.participantId) { + this.variables = {}; + return; + } + + try { + const result = await getParticipantVariablesCallable( + this.sp.firebaseService.functions, + { + experimentId: this.experimentId, + participantId: this.participantId, + }, + ); + this.variables = result.variables || {}; + } catch (error) { + console.error('Failed to load participant variables:', error); + this.variables = {}; + } + } + /** Subscribe to private chat message collections for each stage ID. */ private async loadPrivateChatMessages() { if (!this.experimentId || !this.participantId) return; diff --git a/frontend/src/shared/callables.ts b/frontend/src/shared/callables.ts index f5b6c78a1..2567c4210 100644 --- a/frontend/src/shared/callables.ts +++ b/frontend/src/shared/callables.ts @@ -42,6 +42,8 @@ import { UpdateRankingStageParticipantAnswerData, UpdateSurveyPerParticipantStageParticipantAnswerData, UpdateSurveyStageParticipantAnswerData, + GetParticipantVariablesData, + VariableValue, } from '@deliberation-lab/utils'; import {Functions, httpsCallable} from 'firebase/functions'; @@ -632,3 +634,18 @@ export const ackAlertMessageCallable = async ( )(config); return data; }; + +/** Get participant variables for an experiment. */ +export const getParticipantVariablesCallable = async ( + functions: Functions, + config: GetParticipantVariablesData, +) => { + const {data} = await httpsCallable< + GetParticipantVariablesData, + {variables: Record; cohortName?: string} + >( + functions, + 'getParticipantVariables', + )(config); + return data; +}; From 69f3975260477aa36ee8d9adaa994566f9e6165e Mon Sep 17 00:00:00 2001 From: rasmi Date: Fri, 31 Oct 2025 14:37:51 -0400 Subject: [PATCH 3/5] Allow seed to accept strings. --- utils/src/utils/random.utils.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/utils/src/utils/random.utils.ts b/utils/src/utils/random.utils.ts index c0c523de3..e94b832a9 100644 --- a/utils/src/utils/random.utils.ts +++ b/utils/src/utils/random.utils.ts @@ -22,9 +22,17 @@ export interface ShuffleConfig { // Seed shared by all random functions. let RANDOM_SEED = 0; -/** Initialize the seed with a custom value */ -export const seed = (value: number) => { - RANDOM_SEED = value; +/** Initialize the seed with a custom value (number or string) */ +export const seed = (value: number | string) => { + if (typeof value === 'string') { + let seedValue = 0; + for (let i = 0; i < value.length; i++) { + seedValue += value.charCodeAt(i); + } + RANDOM_SEED = seedValue; + } else { + RANDOM_SEED = value; + } }; /** Update the seed using a Linear Congruential Generator */ @@ -77,16 +85,7 @@ export const shuffleWithSeed = ( array: readonly T[], seedString: string = '', ): T[] => { - // Convert string to numeric seed - let seedValue = 0; - for (let i = 0; i < seedString.length; i++) { - seedValue += seedString.charCodeAt(i); - } - - // Set the seed - seed(seedValue); - - // Return all items in shuffled order + seed(seedString); return choices(array, array.length); }; From 3171af31c46762b7beeef75a61f762259657f6a8 Mon Sep 17 00:00:00 2001 From: rasmi Date: Fri, 31 Oct 2025 15:01:58 -0400 Subject: [PATCH 4/5] Add Distribution cohort assignment for Variables. --- functions/src/participant.utils.ts | 217 +++++++++++++++++++++++---- functions/src/variables.endpoints.ts | 15 +- utils/src/stages/transfer_stage.ts | 21 ++- utils/src/variables.ts | 33 ++-- utils/src/variables.validation.ts | 13 +- 5 files changed, 229 insertions(+), 70 deletions(-) diff --git a/functions/src/participant.utils.ts b/functions/src/participant.utils.ts index f0e6ca34b..8355e4c9c 100644 --- a/functions/src/participant.utils.ts +++ b/functions/src/participant.utils.ts @@ -2,6 +2,7 @@ import {Timestamp} from 'firebase-admin/firestore'; import { AutoTransferType, ChipItem, + ChipStagePublicData, Experiment, ParticipantProfileExtended, ParticipantStatus, @@ -15,7 +16,9 @@ import { SurveyQuestionKind, createChipStageParticipantAnswer, createPayoutStageParticipantAnswer, - ChipStagePublicData, + seed, + random, + SeedStrategy, } from '@deliberation-lab/utils'; import {completeStageAsAgentParticipant} from './agent_participant.utils'; import {getFirestoreActiveParticipants} from './utils/firestore'; @@ -185,6 +188,165 @@ export async function updateCohortStageUnlocked( }); } +/** Transfer a participant to a new cohort by setting transferCohortId and status. */ +function transferParticipantCohort( + transaction: FirebaseFirestore.Transaction, + experimentId: string, + participant: ParticipantProfileExtended, + targetCohortId: string, + transferType: string, +) { + const firestore = app.firestore(); + + const participantDoc = firestore + .collection('experiments') + .doc(experimentId) + .collection('participants') + .doc(participant.privateId); + + transaction.update(participantDoc, { + transferCohortId: targetCohortId, + currentStatus: ParticipantStatus.TRANSFER_PENDING, + }); + + // Update the passed-in participant as a side-effect + participant.currentStatus = ParticipantStatus.TRANSFER_PENDING; + participant.transferCohortId = targetCohortId; + + console.log( + `${transferType} transfer: participant ${participant.publicId} -> cohort ${targetCohortId}`, + ); +} + +/** Handle variable-based automatic transfer. */ +async function handleVariableAutoTransfer( + transaction: FirebaseFirestore.Transaction, + experimentId: string, + stageConfig: TransferStageConfig, + participant: ParticipantProfileExtended, +): Promise<{currentStageId: string; endExperiment: boolean} | null> { + const firestore = app.firestore(); + + // Get the experiment to access variable configuration + const experimentDoc = await transaction.get( + firestore.collection('experiments').doc(experimentId), + ); + const experiment = experimentDoc.data() as Experiment; + + if (!experiment?.variables) { + console.error('No variables configured for experiment'); + return null; + } + + // Get the assignable cohorts (non-initial cohorts) + const assignableCohorts = Object.entries(experiment.variables.cohorts) + .filter(([_, cohort]) => !cohort.isInitialCohort) + .map(([name]) => name); + + if (assignableCohorts.length === 0) { + console.error('No assignable cohorts found in variable configuration'); + return null; + } + + // Determine target cohort based on assignment method + let targetCohortName: string | null = null; + const assignmentConfig = experiment.variables.assignment; + + switch (assignmentConfig.method) { + case 'manual': + // Manual assignment should not use auto-transfer + console.error('Manual assignment method does not support auto-transfer'); + return null; + + case 'distribution': { + // Probability distribution assignment + const distributionConfig = assignmentConfig.distribution; + if (!distributionConfig) { + console.error('Distribution assignment config missing'); + return null; + } + + // Determine seed string for distributing participants to cohorts. + let seedString = ''; + switch (distributionConfig.seedStrategy) { + case SeedStrategy.PARTICIPANT: + seedString = participant.privateId; + break; + case SeedStrategy.EXPERIMENT: + seedString = experimentId; + break; + case SeedStrategy.CUSTOM: + seedString = distributionConfig.customSeed || ''; + break; + case SeedStrategy.COHORT: + seedString = participant.currentCohortId; + break; + default: + seedString = String(Date.now()); + } + + // Set seed from string and generate random value + seed(seedString); + const randomValue = random(); + + // Get probabilities (defaults to equal distribution) + const probabilities = distributionConfig.probabilities || {}; + const defaultProb = 1.0 / assignableCohorts.length; + + // Build cumulative probability distribution + let cumulative = 0; + for (const cohortName of assignableCohorts) { + const prob = probabilities[cohortName] ?? defaultProb; + cumulative += prob; + + if (randomValue < cumulative) { + targetCohortName = cohortName; + break; + } + } + + // Fallback to last cohort if not assigned (handles rounding errors) + if (!targetCohortName) { + targetCohortName = assignableCohorts[assignableCohorts.length - 1]; + } + + break; + } + + default: + console.error(`Unknown assignment method: ${assignmentConfig.method}`); + return null; + } + + // Validate target cohort name + if (!targetCohortName) { + console.error('No target cohort determined for participant'); + return null; + } + + // Get the cohort ID from the cohort configuration + const targetCohortConfig = experiment.variables.cohorts[targetCohortName]; + const targetCohortId = targetCohortConfig?.cohortId; + + if (!targetCohortId) { + console.error( + `No cohort ID found for variable cohort: ${targetCohortName}`, + ); + return null; + } + + // Transfer the participant to the target cohort + transferParticipantCohort( + transaction, + experimentId, + participant, + targetCohortId, + `Variable-based (${assignmentConfig.method})`, + ); + + return {currentStageId: stageConfig.id, endExperiment: false}; +} + /** Automatically transfer participants based on survey answers. */ export async function handleAutomaticTransfer( transaction: FirebaseFirestore.Transaction, @@ -195,18 +357,31 @@ export async function handleAutomaticTransfer( const firestore = app.firestore(); // If stage config does not have an auto-transfer config, ignore - // TODO: Remove temporary ignore of "default" transfer type - if ( - !stageConfig.autoTransferConfig || - stageConfig.autoTransferConfig.type !== AutoTransferType.SURVEY - ) { + if (!stageConfig.autoTransferConfig) { return null; } - // Auto-transfer config - // TODO: Add switch statement depending on transfer type + // Handle different transfer types const autoTransferConfig = stageConfig.autoTransferConfig; + switch (autoTransferConfig.type) { + case AutoTransferType.VARIABLE: + return await handleVariableAutoTransfer( + transaction, + experimentId, + stageConfig, + participant, + ); + case AutoTransferType.SURVEY: + // Continue with existing survey transfer logic + break; + case AutoTransferType.DEFAULT: + // TODO: Implement default transfer type + return null; + default: + return null; + } + // Do a read to lock the current participant's document for this transaction // The data itself might be outdated, so we discard it const participantDocRef = firestore @@ -356,28 +531,16 @@ export async function handleAutomaticTransfer( await createCohortInternal(transaction, experimentId, cohortConfig); - for (const participant of cohortParticipants) { - const participantDoc = firestore - .collection('experiments') - .doc(experimentId) - .collection('participants') - .doc(participant.privateId); - - transaction.update(participantDoc, { - transferCohortId: cohortConfig.id, - currentStatus: ParticipantStatus.TRANSFER_PENDING, - }); - - console.log( - `Transferring participant ${participant.publicId} to cohort ${cohortConfig.id}`, + for (const cohortParticipant of cohortParticipants) { + transferParticipantCohort( + transaction, + experimentId, + cohortParticipant, + cohortConfig.id, + 'Survey-based', ); } - // Update the passed-in participant as a side-effect, since this is how we merge all the changes - // from the updateParticipantToNextStage endpoint - participant.currentStatus = ParticipantStatus.TRANSFER_PENDING; - participant.transferCohortId = cohortConfig.id; - return {currentStageId: stageConfig.id, endExperiment: false}; } diff --git a/functions/src/variables.endpoints.ts b/functions/src/variables.endpoints.ts index bf9de2068..5c4c7f207 100644 --- a/functions/src/variables.endpoints.ts +++ b/functions/src/variables.endpoints.ts @@ -4,6 +4,7 @@ import { InitializeVariableCohortsData, GetParticipantVariablesData, ExperimentVariables, + VariableCohort, validateInitialCohort, getAssignableCohorts, resolveParticipantVariables, @@ -42,7 +43,6 @@ export const initializeVariableCohorts = onCall( // Check authentication await AuthGuard.isExperimenter(request); - const uid = request.auth?.uid; const {experimentId, variables, replaceExisting = false} = data; @@ -122,11 +122,8 @@ export const initializeVariableCohorts = onCall( } // Create new cohorts based on variable configuration - const cohortIds: Record = {}; - - for (const [cohortName, cohortConfig] of Object.entries( - variables.cohorts, - )) { + for (const cohortName of Object.keys(variables.cohorts)) { + const cohortConfig: VariableCohort = variables.cohorts[cohortName]; // Create cohort configuration const cohort = createCohortConfig({ metadata: createMetadataConfig({ @@ -144,10 +141,11 @@ export const initializeVariableCohorts = onCall( // Store cohort using internal utility await createCohortInternal(transaction, experimentId, cohort); - cohortIds[cohortName] = cohort.id; + // Store the cohort ID in the cohort config + cohortConfig.cohortId = cohort.id; } - // Update experiment with variables configuration + // Update experiment with variables configuration (now includes cohort IDs) const experimentRef = app .firestore() .collection('experiments') @@ -155,7 +153,6 @@ export const initializeVariableCohorts = onCall( transaction.update(experimentRef, { variables: variables, - variableCohortIds: cohortIds, dateEdited: new Date(), }); }); diff --git a/utils/src/stages/transfer_stage.ts b/utils/src/stages/transfer_stage.ts index 17525ba63..3fdc0d707 100644 --- a/utils/src/stages/transfer_stage.ts +++ b/utils/src/stages/transfer_stage.ts @@ -25,11 +25,13 @@ export interface TransferStageConfig extends BaseStageConfig { export type AutoTransferConfig = | DefaultAutoTransferConfig - | SurveyAutoTransferConfig; + | SurveyAutoTransferConfig + | VariableAutoTransferConfig; export enum AutoTransferType { DEFAULT = 'default', // group only based on number of participants SURVEY = 'survey', // match based on responses to specific survey question + VARIABLE = 'variable', // assign to cohorts based on experiment variables } export interface BaseAutoTransferConfig { @@ -53,6 +55,12 @@ export interface SurveyAutoTransferConfig extends BaseAutoTransferConfig { participantCounts: {[key: string]: number}; } +export interface VariableAutoTransferConfig extends BaseAutoTransferConfig { + type: AutoTransferType.VARIABLE; + // Use experiment's variable configuration for cohort assignment + // No additional configuration needed - uses ExperimentVariables +} + // ************************************************************************* // // FUNCTIONS // // ************************************************************************* // @@ -89,3 +97,14 @@ export function createSurveyAutoTransferConfig( participantCounts: config.participantCounts ?? {}, }; } + +/** Create variable auto-transfer config. */ +export function createVariableAutoTransferConfig( + config: Partial = {}, +): VariableAutoTransferConfig { + return { + type: AutoTransferType.VARIABLE, + autoCohortParticipantConfig: + config.autoCohortParticipantConfig ?? createCohortParticipantConfig(), + }; +} diff --git a/utils/src/variables.ts b/utils/src/variables.ts index b1191a81e..8fcf8ce00 100644 --- a/utils/src/variables.ts +++ b/utils/src/variables.ts @@ -1,4 +1,3 @@ -import {Condition} from './utils/condition'; import {SeedStrategy} from './utils/random.utils'; /** Experiment Variables - Defines experimental conditions and their values */ @@ -39,37 +38,21 @@ export interface VariableDefinition { // ************************************************************************* // /** Method for assigning participants to cohorts */ -export type AssignmentMethod = 'random' | 'manual' | 'conditional'; +export type AssignmentMethod = 'distribution' | 'manual'; -/** Configuration for random assignment */ -export interface RandomAssignmentConfig { +/** Configuration for probability distribution assignment */ +export interface DistributionConfig { seedStrategy: SeedStrategy; customSeed?: string; - /** Optional weights per cohort (defaults to equal distribution) */ - weights?: Record; -} - -/** Rule for conditional assignment */ -export interface ConditionalAssignmentRule { - condition: Condition; - cohort: string; -} - -/** Configuration for conditional assignment */ -export interface ConditionalAssignmentConfig { - /** Maps conditions to cohorts */ - rules: ConditionalAssignmentRule[]; - /** Fallback if no condition matches */ - defaultCohort?: string; + /** Optional probabilities per cohort (defaults to equal distribution) */ + probabilities?: Record; } /** Assignment strategy for distributing participants */ export interface AssignmentConfig { method: AssignmentMethod; - /** Configuration for random assignment */ - random?: RandomAssignmentConfig; - /** Configuration for conditional assignment */ - conditional?: ConditionalAssignmentConfig; + /** Configuration for probability distribution assignment */ + distribution?: DistributionConfig; } // ************************************************************************* // @@ -83,6 +66,8 @@ export interface VariableCohort { isInitialCohort?: boolean; /** Variable values for this cohort */ variables: Record; + /** Cohort ID (set when cohort is created) */ + cohortId?: string; } // ************************************************************************* // diff --git a/utils/src/variables.validation.ts b/utils/src/variables.validation.ts index 3080f34a1..15a9563db 100644 --- a/utils/src/variables.validation.ts +++ b/utils/src/variables.validation.ts @@ -69,7 +69,7 @@ const VariableCohortData = Type.Object( // Assignment Configuration Schema // ************************************************************************* // -const RandomAssignmentConfigData = Type.Object( +const DistributionConfigData = Type.Object( { seedStrategy: Type.Union([ Type.Literal(SeedStrategy.EXPERIMENT), @@ -78,20 +78,15 @@ const RandomAssignmentConfigData = Type.Object( Type.Literal(SeedStrategy.CUSTOM), ]), customSeed: Type.Optional(Type.String()), - weights: Type.Optional(Type.Record(Type.String(), Type.Number())), + probabilities: Type.Optional(Type.Record(Type.String(), Type.Number())), }, strict, ); const AssignmentConfigData = Type.Object( { - method: Type.Union([ - Type.Literal('random'), - Type.Literal('manual'), - Type.Literal('conditional'), - ]), - random: Type.Optional(RandomAssignmentConfigData), - // Note: conditional assignment would need Condition schema imported + method: Type.Union([Type.Literal('distribution'), Type.Literal('manual')]), + distribution: Type.Optional(DistributionConfigData), }, strict, ); From 9adb57b8f859c53aa8f6f4361cd1376aa015799c Mon Sep 17 00:00:00 2001 From: rasmi Date: Fri, 31 Oct 2025 15:06:37 -0400 Subject: [PATCH 5/5] Initial docs (need to tidy up and add to nav.yml) --- docs/features/variables.md | 440 +++++++++++++++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 docs/features/variables.md diff --git a/docs/features/variables.md b/docs/features/variables.md new file mode 100644 index 000000000..bb66138b6 --- /dev/null +++ b/docs/features/variables.md @@ -0,0 +1,440 @@ +# Experiment Variables Design Document + +## Overview + +Experiment Variables enable researchers to create different experimental conditions by defining values that vary across cohorts. This allows for A/B testing, multi-arm experiments, and conditional content presentation while maintaining a single experiment configuration. + +## Core Concepts + +### Variables +Variables are named values that can be referenced in stage configurations using template syntax (`{{variableName}}`). They can be: +- **Primitives**: strings, numbers, booleans +- **Objects**: structured data with multiple properties (e.g., `{{policy.name}}`, `{{policy.cost}}`) + +### Variable Cohorts +Cohorts created through the variables system that each receive different variable values. Participants in different cohorts see different content based on their cohort's variable assignments. + +### Template Resolution +Stage content containing `{{variableName}}` placeholders gets resolved to actual values based on the participant's cohort variables. + +## Architecture + +### Data Structure + +```typescript +interface ExperimentVariables { + // Define available variables and their types + definitions: { + [variableName: string]: { + type: 'string' | 'number' | 'boolean' | 'object'; + description?: string; + defaultValue?: any; + // For object types, define property structure + schema?: { + [property: string]: { + type: 'string' | 'number' | 'boolean'; + description?: string; + } + }; + } + }; + + // Define cohorts and their variable values + cohorts: { + [cohortName: string]: { + description?: string; + isInitialCohort?: boolean; // Mark as initial cohort (max 1 per experiment) + variables: { + [variableName: string]: any; + }; + cohortId?: string; // Set when cohort is created + } + }; + + // Assignment strategy for distributing participants + assignment: { + method: 'distribution' | 'manual'; + + // For probability distribution assignment + distribution?: { + seedStrategy: SeedStrategy; + customSeed?: string; + // Optional probabilities per cohort (0.0-1.0, defaults to equal distribution) + // Uses cumulative probability distribution for assignment + probabilities?: { + [cohortName: string]: number; + }; + }; + }; +} +``` + +## Key Design Decisions + +### 1. Variables Define Cohorts +**Decision**: Variable configuration includes cohort definitions. Cohorts are created from the variable configuration. + +**Rationale**: +- Avoids the ID mapping problem (cohort IDs are auto-generated) +- Single source of truth for experimental conditions +- Clear mental model: define conditions and their values together + +**Alternative Considered**: Mapping variables to existing cohorts by ID or index +- **Problem**: Cohort IDs aren't known until creation, indices change with cohort order + +### 2. Leverage Transfer Stage for Assignment +**Decision**: Use the existing transfer stage mechanism for cohort assignment rather than building new assignment infrastructure. + +**Rationale**: +- Transfer stage already handles participant distribution logic +- Supports waiting for participant thresholds +- Has timeout mechanisms +- Reduces code duplication + +**Implementation**: The "Initialize Variable Cohorts" action will: +1. Create cohorts based on variable configuration +2. Configure a transfer stage with appropriate assignment logic + +### 3. Assignment Strategy at Variables Level +**Decision**: Define assignment strategy (distribution, manual) at the variables level, not per-cohort. + +**Rationale**: +- Avoids confusing "probability of remaining" calculations +- Single source of truth for assignment logic +- Clearer mental model for experimenters + +**Example Problem We Avoided**: +```typescript +// Confusing: Each cohort defines probability of "remaining" participants +cohortA: { probability: 0.333 } // 33.3% of all +cohortB: { probability: 0.5 } // 50% of remaining = 33.3% of all +cohortC: { /* gets remainder */ } // 33.3% of all +``` + +**Our Solution**: +```typescript +// Clear: Define probabilities at assignment level (uses same logic as RandomCondition) +assignment: { + method: 'distribution', + distribution: { + seedStrategy: SeedStrategy.PARTICIPANT, + probabilities: { + 'cohortA': 0.33, // 33% probability + 'cohortB': 0.33, // 33% probability + 'cohortC': 0.34 // 34% probability (totals 1.0) + } + // Or omit probabilities for equal distribution across all cohorts + } +} +``` + +### 4. Deterministic Probability Distribution +**Decision**: Use seeded randomization with SeedStrategy (PARTICIPANT, EXPERIMENT, CUSTOM, COHORT) matching `RandomCondition` implementation. + +**Rationale**: +- Reproducible assignments (same participant always gets same cohort) +- Supports testing and debugging +- Consistent with existing `RandomCondition` logic in condition system +- Uses cumulative probability distribution for assignment +- Leverages existing random utilities (`seed()` and `random()`) with identical approach +- Clear distinction from `RandomCondition` (which is for binary decisions) + +### 5. Template Syntax Choice +**Decision**: Use double-brace syntax `{{variableName}}` for template variables. + +**Rationale**: +- Not currently used in the codebase (verified via grep) +- Familiar from templating systems (Handlebars, etc.) +- Visually distinct from other syntax +- Supports nested access: `{{object.property}}` + +### 6. Optional Initial Cohort Pattern +**Decision**: Support an optional initial cohort where all participants start before treatment assignment. + +**Rationale**: +- Simple experiments can use direct assignment +- Complex experiments can use an initial cohort for pre-treatment stages +- Clearer mental model with single entry point + +**Patterns Supported**: +1. **Direct Assignment**: Participants immediately assigned to treatment cohorts on join +2. **Initial Cohort Pattern**: All participants start in initial cohort, then transfer to treatment cohorts +3. **Pre-Assignment Stages**: Complete consent/survey in initial cohort before assignment + +**Implementation**: +- Exactly one cohort can be marked with `isInitialCohort: true` +- Initial cohort is automatically excluded from random assignment +- Variables in initial cohort typically empty or use default values +- Templates before transfer stage show default values +- Validation ensures maximum one initial cohort per experiment + +## Implementation Status + +### Phase 1: Core Infrastructure ✅ COMPLETED +- [x] Variable types and interfaces in utils package +- [x] Template resolution utilities using Mustache +- [x] Support for string, number, boolean, and object variables + +### Phase 2: Cohort Management ✅ COMPLETED +- [x] Initialize Variable Cohorts endpoint +- [x] Integration with transfer stage (VariableAutoTransferConfig) +- [x] Distribution assignment strategy implementation +- [x] Seeded randomization for reproducible assignments +- [x] Helper function for participant cohort transfers + +### Phase 3: UI Components ⏳ TODO +- [ ] Variable editor in experiment builder +- [ ] Cohort configuration interface +- [ ] Template validation and preview + +### Phase 4: Stage Integration 🚧 IN PROGRESS +- [x] Variable resolution in InfoStage +- [ ] Extend to Survey, Chat, and other text-containing stages +- [ ] Template preview in stage editors + +### Phase 5: Advanced Features 🚧 IN PROGRESS +- [x] Object variable support with nested properties +- [ ] Conditional assignment rules (deferred for simplicity) +- [ ] Variable usage analytics + +## Usage Examples + +### Simple A/B Test with Initial Cohort +```typescript +{ + definitions: { + buttonColor: { type: 'string', defaultValue: 'gray' }, + buttonText: { type: 'string', defaultValue: 'Next' } + }, + + cohorts: { + 'Onboarding': { + description: 'Initial cohort for consent and demographics', + isInitialCohort: true, // Marked as the initial cohort + variables: {} // Uses default values + }, + 'Control': { + variables: { + buttonColor: 'blue', + buttonText: 'Submit' + } + }, + 'Treatment': { + variables: { + buttonColor: 'green', + buttonText: 'Continue' + } + } + }, + + assignment: { + method: 'distribution', + distribution: { + seedStrategy: SeedStrategy.PARTICIPANT + // Initial cohort automatically excluded from assignment + } + } +} +``` + +### Multi-Arm Policy Experiment +```typescript +{ + definitions: { + policy: { + type: 'object', + schema: { + name: { type: 'string' }, + description: { type: 'string' }, + cost: { type: 'number' }, + coverage: { type: 'string' } + } + } + }, + + cohorts: { + 'Universal Healthcare': { + variables: { + policy: { + name: 'Medicare for All', + description: 'Government-run single-payer system', + cost: 3200000000000, + coverage: '100% of population' + } + } + }, + 'Public Option': { + variables: { + policy: { + name: 'Public Option', + description: 'Government insurance competing with private', + cost: 1500000000000, + coverage: '95% of population' + } + } + }, + 'Status Quo': { + variables: { + policy: { + name: 'Current System', + description: 'Employer-based with marketplace', + cost: 3800000000000, + coverage: '91% of population' + } + } + } + }, + + assignment: { + method: 'distribution', + distribution: { + seedStrategy: SeedStrategy.PARTICIPANT, + // Probabilities optional - defaults to equal distribution across all cohorts + probabilities: { + 'Universal Healthcare': 0.33, + 'Public Option': 0.33, + 'Status Quo': 0.34 + } + } + } +} +``` + +### Template Usage in Stages + +**InfoStage Content:** +```markdown +## {{policy.name}} + +{{policy.description}} + +**Estimated Annual Cost**: ${{policy.cost}} +**Population Coverage**: {{policy.coverage}} + +Please review this policy carefully before proceeding to the discussion. +``` + +**Survey Question:** +``` +How strongly do you support {{policy.name}}? +``` + +**Chat System Prompt:** +``` +Participants will discuss {{policy.name}}, which costs ${{policy.cost}} +annually and covers {{policy.coverage}}. +``` + +## Migration and Compatibility + +### Existing Experiments +- Variables are opt-in; existing experiments continue working unchanged +- Can gradually adopt variables by adding configuration and recreating cohorts + +### Participant Flow with Variables +1. **Join Experiment**: Participant clicks link, joins cohort marked with `isInitialCohort: true` +2. **Pre-Assignment Stages**: Complete consent, demographics, etc. in initial cohort + - Templates show default values or placeholders + - Variables not yet resolved to treatment values +3. **Transfer Stage**: Participant assigned to treatment cohort + - Assignment based on method (distribution/manual) + - Initial cohort excluded from assignment pool +4. **Post-Assignment Stages**: See treatment-specific content + - Templates now resolve to cohort-specific variables + - Full variable values available + +### Automatic Cohort Transfer Service +- Compatible with variable cohorts +- Transfer stage handles assignment based on variable configuration +- Service can distribute participants according to assignment strategy + +### Manual Cohort Management +- Experimenters can still manually create and manage cohorts +- Variable cohorts marked with metadata for identification +- Can mix variable and non-variable cohorts in same experiment + +## Security and Validation + +### Template Injection Prevention +- Templates resolved server-side only +- No client-side template evaluation +- Variable values sanitized before insertion + +### Type Validation +- Variable definitions enforce type constraints +- Schema validation for object variables +- Runtime type checking during template resolution + +### Configuration Validation +- Maximum one cohort with `isInitialCohort: true` +- At least one non-initial cohort required for assignment +- Variable names must be valid identifiers + +### Access Control +- Variables resolved based on participant's cohort +- Participants cannot access other cohorts' variables +- Experimenter-only access to variable configuration + +## Future Enhancements + +### Dynamic Variables +- Computed variables (e.g., `{{participantCount}}`) +- Time-based variables (e.g., `{{experimentDay}}`) +- Aggregate variables from participant responses + +### Advanced Assignment + +#### Conditional Assignment (Future) +Conditional assignment would allow cohort assignment based on participant data (e.g., survey responses). This was deferred from the initial implementation to keep things simple. + +**Proposed Design**: +```typescript +assignment: { + method: 'conditional', + conditional: { + conditions: { + 'Senior': createComparisonCondition( + {stageId: 'survey', questionId: 'age'}, + 'greater_than_or_equal', + 65 + ), + 'Adult': createComparisonCondition( + {stageId: 'survey', questionId: 'age'}, + 'greater_than_or_equal', + 18 + ), + }, + defaultCohort: 'Child' + } +} +``` + +**Implementation Challenges**: +- **Context Passing**: Need to pass participant answers and stage data to evaluation context + - Must fetch stage answers for dependencies extracted from conditions + - Need to build `targetValues` mapping from participant's stage answers + - Requires access to `experimentId`, `cohortId`, `participantId` for RandomCondition seeds +- **Mutually Exclusive Conditions**: Conditions should be mutually exclusive to avoid ambiguity + - If multiple conditions match, behavior is undefined + - Validation tooling needed to detect overlapping conditions + - Consider runtime warnings if multiple conditions evaluate to true +- **Data Availability**: Transfer must occur after all dependency stages are completed + - Conditions reference specific stage/question answers + - Must ensure those stages have been completed before transfer + - Complex dependency tracking may be needed + +**Alternative Approach**: Use survey-based transfer (existing `AutoTransferType.SURVEY`) which already handles condition evaluation with proper context. + +#### Other Advanced Assignment Features +- Multi-factor assignment (combine multiple strategies) +- Sequential assignment patterns +- Adaptive assignment based on cohort balance + +### Variable Analytics +- Track which variables are actually used +- Analyze variable impact on outcomes +- A/B test result calculation + +## Conclusion + +The Experiment Variables system provides a powerful, flexible way to create experimental conditions while maintaining clean separation between experimental design (what varies) and operational concerns (how to run the experiment). By leveraging existing infrastructure (transfer stages, conditions) and making thoughtful design decisions (variables define cohorts, assignment at variables level), we create a system that is both powerful and intuitive for researchers. \ No newline at end of file