Metrics are only counted for users who have been exposed to the experiment. This
- ensures fair comparison between control and test groups.
+ ensures a fair comparison between baseline and comparison groups.
{exposureCriteria
diff --git a/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx b/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx
index 2d8d36d356a7..c13776b09bd1 100644
--- a/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx
@@ -117,11 +117,11 @@ export function DistributionTable(): JSX.Element {
const variants = getExperimentVariants(experiment)
/**
- * We use this check to disable the toggle if there's only one test variant left.
+ * We use this check to disable the toggle if there's only one comparison variant left.
* - not the baseline variant
* - not excluded
*/
- const hasOnlyOneTestVariant =
+ const hasOnlyOneComparisonVariant =
variants.filter(({ key }) => key !== baselineKey && !excludedVariants.includes(key)).length <= 1
const onSelectElement = (variant: string): void => {
@@ -195,7 +195,7 @@ export function DistributionTable(): JSX.Element {
* - the variant is not excluded: we have to allow re-including it
* - there's only one variant left when we remove the baseline
*/
- const disableToggle = !excluded && hasOnlyOneTestVariant
+ const disableToggle = !excluded && hasOnlyOneComparisonVariant
return (
setVariantExcluded(key, !checked)}
disabledReason={
disableToggle
- ? 'At least one test variant must remain in analysis'
+ ? 'At least one comparison variant must remain in analysis'
: undefined
}
loading={experimentUpdateLoading}
@@ -315,9 +315,9 @@ export function DistributionTable(): JSX.Element {
variants are modified to show their relative rollout percentage.
)}
- {excludedVariants.length > 0 && hasOnlyOneTestVariant && (
+ {excludedVariants.length > 0 && hasOnlyOneComparisonVariant && (
- At least one test variant must remain in analysis. Re-include a variant to exclude others.
+ At least one comparison variant must remain in analysis. Re-include a variant to exclude others.
)}
!experiment.excluded_variants?.includes(key))
+ const isBaselineVariantValid = availableBaselineVariants.some(({ key }) => key === baselineVariantKey)
// Only show alerts section for saved experiments, as the alert relies on experiment.id for filtering
const shouldShowSignificanceAlerts = typeof experiment.id === 'number'
@@ -90,18 +95,32 @@ export function SettingsTab(): JSX.Element {
Baseline variant
-
({
- value: v.key,
- label: v.key,
- }))}
- onChange={(value) => {
- updateExperimentSettings({
- stats_config: { ...experiment.stats_config, baseline_variant_key: value },
- })
- }}
- />
+ {!isBaselineVariantValid && configuredBaselineVariantKey && (
+
+ The baseline variant "{configuredBaselineVariantKey}" is no longer available. Select another
+ baseline before refreshing results.
+
+ )}
+
+ ({
+ value: v.key,
+ label: v.key,
+ }))}
+ onChange={(value) => {
+ if (experimentUpdateLoading) {
+ return
+ }
+ updateExperimentSettings({
+ stats_config: { ...experiment.stats_config, baseline_variant_key: value },
+ })
+ }}
+ disabledReason={experimentUpdateLoading ? 'Saving the baseline variant' : undefined}
+ data-attr="experiment-baseline-variant"
+ />
+ {experimentUpdateLoading && }
+
The variant all others are compared against.
diff --git a/frontend/src/scenes/experiments/ExperimentWizard/experimentWizardLogic.ts b/frontend/src/scenes/experiments/ExperimentWizard/experimentWizardLogic.ts
index 63f7339226ff..d54795ce213c 100644
--- a/frontend/src/scenes/experiments/ExperimentWizard/experimentWizardLogic.ts
+++ b/frontend/src/scenes/experiments/ExperimentWizard/experimentWizardLogic.ts
@@ -102,12 +102,38 @@ export interface experimentWizardLogicActions {
ensure_experience_continuity?: boolean
feature_flag_key?: string
rollout_percentage?: number
+ stats_config?: Experiment['stats_config']
variants?: MultivariateFlagVariant[]
}) => {
config: {
ensure_experience_continuity?: boolean | undefined
feature_flag_key?: string | undefined
rollout_percentage?: number | undefined
+ stats_config?:
+ | {
+ baseline_variant_key?: string | undefined
+ bayesian?:
+ | {
+ ci_level?: number | undefined
+ }
+ | undefined
+ cuped?:
+ | {
+ enabled?: boolean | undefined
+ lookback_days?: number | undefined
+ }
+ | undefined
+ frequentist?:
+ | {
+ alpha?: number | undefined
+ sequential_testing_enabled?: boolean | undefined
+ sequential_tuning_parameter?: number | undefined
+ }
+ | undefined
+ method?: import('~/types').ExperimentStatsMethod | undefined
+ version?: number | undefined
+ }
+ | undefined
variants?: MultivariateFlagVariant[] | undefined
}
} // createExperimentLogic
diff --git a/frontend/src/scenes/experiments/ExperimentWizard/steps/AboutStep.tsx b/frontend/src/scenes/experiments/ExperimentWizard/steps/AboutStep.tsx
index e33c8865e4dd..c6b218e02f36 100644
--- a/frontend/src/scenes/experiments/ExperimentWizard/steps/AboutStep.tsx
+++ b/frontend/src/scenes/experiments/ExperimentWizard/steps/AboutStep.tsx
@@ -13,7 +13,7 @@ import type { FeatureFlagType } from '~/types'
import { SelectExistingFeatureFlagModal } from '../../ExperimentForm/SelectExistingFeatureFlagModal'
import { selectExistingFeatureFlagModalLogic } from '../../ExperimentForm/selectExistingFeatureFlagModalLogic'
import { VariantsPanelLinkFeatureFlag } from '../../ExperimentForm/VariantsPanelLinkFeatureFlag'
-import { getFlagVariants } from '../../utils'
+import { getBaselineVariantKey, getDefaultBaselineVariantKey, getFlagVariants } from '../../utils'
import { experimentWizardLogic } from '../experimentWizardLogic'
export function AboutStep(): JSX.Element {
@@ -49,11 +49,16 @@ export function AboutStep(): JSX.Element {
(isDeparted && !experiment.feature_flag_key?.trim() ? 'Feature flag key is required' : undefined)
const linkExistingFlag = (flag: FeatureFlagType): void => {
+ const variants = getFlagVariants(flag)
setLinkedFeatureFlag(flag)
clearFeatureFlagKeyValidation()
setFeatureFlagConfig({
feature_flag_key: flag.key,
- variants: getFlagVariants(flag),
+ variants,
+ stats_config: {
+ ...experiment.stats_config,
+ baseline_variant_key: getDefaultBaselineVariantKey(variants),
+ },
})
}
@@ -96,6 +101,7 @@ export function AboutStep(): JSX.Element {
{
setLinkedFeatureFlag(null)
setExperimentValue('feature_flag_key', '')
diff --git a/frontend/src/scenes/experiments/ExperimentWizard/steps/VariantsStep.tsx b/frontend/src/scenes/experiments/ExperimentWizard/steps/VariantsStep.tsx
index d464a819acc1..f971103ae878 100644
--- a/frontend/src/scenes/experiments/ExperimentWizard/steps/VariantsStep.tsx
+++ b/frontend/src/scenes/experiments/ExperimentWizard/steps/VariantsStep.tsx
@@ -2,6 +2,8 @@ import { useActions, useValues } from 'kea'
import { getSeriesColor } from 'lib/colors'
import { LemonBanner } from 'lib/lemon-ui/LemonBanner'
+import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
+import { LemonTag } from 'lib/lemon-ui/LemonTag'
import { Lettermark, LettermarkColor } from 'lib/lemon-ui/Lettermark'
import { formatPercentage } from 'lib/utils/numbers'
import { alphabet } from 'lib/utils/strings'
@@ -10,13 +12,16 @@ import type { FeatureFlagType } from '~/types'
import { TrafficPreview } from '../../ExperimentForm/VariantDistributionEditor'
import { VariantsPanelCreateFeatureFlag } from '../../ExperimentForm/VariantsPanelCreateFeatureFlag'
-import { getFlagVariants } from '../../utils'
+import { getBaselineVariantKey, getFlagVariants } from '../../utils'
import { experimentWizardLogic } from '../experimentWizardLogic'
const ReadOnlyVariantsStep = ({ flag }: { flag: FeatureFlagType }): JSX.Element => {
+ const { experiment } = useValues(experimentWizardLogic)
+ const { setExperimentValue } = useActions(experimentWizardLogic)
const variants = getFlagVariants(flag)
const rolloutPercentage = flag.filters.groups?.[0]?.rollout_percentage ?? 100
const variantRolloutSum = variants.reduce((sum, { rollout_percentage }) => sum + rollout_percentage, 0)
+ const baselineVariantKey = getBaselineVariantKey(experiment)
return (
<>
@@ -67,7 +72,12 @@ const ReadOnlyVariantsStep = ({ flag }: { flag: FeatureFlagType }): JSX.Element
- {variant.key}
+
+ {variant.key}
+ {variant.key === baselineVariantKey && (
+ Baseline
+ )}
+
|
@@ -81,6 +91,21 @@ const ReadOnlyVariantsStep = ({ flag }: { flag: FeatureFlagType }): JSX.Element
+
+ Baseline variant
+ ({ value: key, label: key }))}
+ onChange={(baseline_variant_key) => {
+ setExperimentValue('stats_config', {
+ ...experiment.stats_config,
+ baseline_variant_key,
+ })
+ }}
+ data-attr="experiment-baseline-variant"
+ />
+ The variant all others are compared against.
+
Credible interval (95%)
-
+
),
render: function Key(_, v): JSX.Element {
const variant = v as TrendExperimentVariant
- if (variant.key === 'control') {
+ if (variant.key === baselineVariantKey) {
return Baseline
}
@@ -217,17 +218,17 @@ export function LegacySummaryTable({
title: (
),
render: function Key(_, item): JSX.Element {
- if (item.key === 'control') {
+ if (item.key === baselineVariantKey) {
return Baseline
}
- const controlConversionRate = legacyConversionRateForVariant(result, 'control')
+ const controlConversionRate = legacyConversionRateForVariant(result, baselineVariantKey)
const variantConversionRate = legacyConversionRateForVariant(result, item.key)
if (!controlConversionRate || !variantConversionRate) {
@@ -249,13 +250,13 @@ export function LegacySummaryTable({
title: (
Credible interval (95%)
-
+
),
render: function Key(_, item): JSX.Element {
- if (item.key === 'control') {
+ if (item.key === baselineVariantKey) {
return Baseline
}
diff --git a/frontend/src/scenes/experiments/legacy/metricsView/LegacyDeltaChart.tsx b/frontend/src/scenes/experiments/legacy/metricsView/LegacyDeltaChart.tsx
index 1f67cdde0677..d57d1288b039 100644
--- a/frontend/src/scenes/experiments/legacy/metricsView/LegacyDeltaChart.tsx
+++ b/frontend/src/scenes/experiments/legacy/metricsView/LegacyDeltaChart.tsx
@@ -168,10 +168,11 @@ function VariantBar({ variant, index }: { variant: any; index: number }): JSX.El
const deltaResult = legacyCalculateDelta(result, variant.key, metricType)
const delta = deltaResult?.delta || 0
+ const baselineVariantKey = result.variants[0]?.key ?? 'control'
let hasEnoughData: boolean
if (metricType === InsightType.TRENDS) {
- const controlVariant = result.variants.find((v: any) => v.key === 'control')
+ const controlVariant = result.variants.find((v: any) => v.key === baselineVariantKey)
const variantData = result.variants.find((v: any) => v.key === variant.key)
if (
@@ -226,7 +227,7 @@ function VariantBar({ variant, index }: { variant: any; index: number }): JSX.El
>
- {variant.key === 'control' ? (
+ {variant.key === baselineVariantKey ? (
= {
- [ResultErrorCode.NO_CONTROL_VARIANT]: 'Events with the control variant not received',
- [ResultErrorCode.NO_TEST_VARIANT]: 'Events with at least one test variant not received',
+ [ResultErrorCode.NO_CONTROL_VARIANT]: 'Events with the baseline variant not received',
+ [ResultErrorCode.NO_TEST_VARIANT]: 'Events with at least one comparison variant not received',
[ResultErrorCode.NO_EXPOSURES]: 'Exposure events not received',
}
const successText: Record = {
- [ResultErrorCode.NO_CONTROL_VARIANT]: 'Events with the control variant received',
- [ResultErrorCode.NO_TEST_VARIANT]: 'Events with at least one test variant received',
+ [ResultErrorCode.NO_CONTROL_VARIANT]: 'Events with the baseline variant received',
+ [ResultErrorCode.NO_TEST_VARIANT]: 'Events with at least one comparison variant received',
[ResultErrorCode.NO_EXPOSURES]: 'Exposure events have been received',
}
const insightType = getInsightType(metric)
const hasMissingExposure = errorCode === ResultErrorCode.NO_EXPOSURES
+ const baselineVariantKey = getBaselineVariantKey(experiment)
const requiredEvent =
insightType === InsightType.TRENDS
@@ -73,8 +74,8 @@ function ChecklistItem({
value: hasMissingExposure
? variants.map((variant: any) => variant.key)
: errorCode === ResultErrorCode.NO_CONTROL_VARIANT
- ? ['control']
- : variants.slice(1).map((variant: any) => variant.key),
+ ? [baselineVariantKey]
+ : variants.filter(({ key }) => key !== baselineVariantKey).map(({ key }) => key),
operator: 'exact',
type: 'event',
},
diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.ts
index 2c517bdfa94d..01a60c35c195 100644
--- a/frontend/src/scenes/experiments/utils.ts
+++ b/frontend/src/scenes/experiments/utils.ts
@@ -125,7 +125,10 @@ export function getBaselineVariantKey(experiment: Partial | null | u
if (configured) {
return configured
}
- const variants = getExperimentVariants(experiment)
+ return getDefaultBaselineVariantKey(getExperimentVariants(experiment))
+}
+
+export function getDefaultBaselineVariantKey(variants: Array<{ key: string }>): string {
if (variants.length === 0 || variants.some((variant) => variant.key === 'control')) {
return 'control'
}
diff --git a/posthog/schema.py b/posthog/schema.py
index 47537fcab76d..adc94ab6e980 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -1422,6 +1422,45 @@ class ExperimentParameters(BaseModel):
)
+class ExperimentBayesianStatsConfig(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ ci_level: float | None = None
+
+
+class ExperimentFrequentistStatsConfig(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ alpha: float | None = None
+ sequential_testing_enabled: bool | None = None
+ sequential_tuning_parameter: float | None = None
+
+
+class ExperimentCupedStatsConfig(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ enabled: bool | None = None
+ lookback_days: int | None = None
+
+
+class ExperimentStatsConfig(BaseModel):
+ model_config = ConfigDict(extra="allow")
+
+ baseline_variant_key: str | None = Field(
+ default=None,
+ description=(
+ "Variant key all other variants are compared against. The key must exist in the experiment's variants "
+ "and cannot be excluded from analysis. When omitted, analysis uses 'control' if present, otherwise the "
+ "first configured variant."
+ ),
+ )
+ bayesian: ExperimentBayesianStatsConfig | None = None
+ cuped: ExperimentCupedStatsConfig | None = None
+ frequentist: ExperimentFrequentistStatsConfig | None = None
+ method: Literal["bayesian", "frequentist"] | None = None
+ version: int | None = None
+
+
class ExperimentRunningTimeCalculation(BaseModel):
model_config = ConfigDict(
extra="forbid",
diff --git a/posthog/temporal/experiments/activities.py b/posthog/temporal/experiments/activities.py
index 3862420918c9..1bdca8d6a77c 100644
--- a/posthog/temporal/experiments/activities.py
+++ b/posthog/temporal/experiments/activities.py
@@ -25,6 +25,7 @@
from posthog.temporal.experiments.utils import DEFAULT_EXPERIMENT_RECALCULATION_HOUR, check_significance_transition
from products.experiments.backend.facade.timeseries import backfill_experiment_timeseries
+from products.experiments.backend.hogql_queries import get_experiment_fingerprint_baseline_variant_key
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window_end
from products.experiments.backend.hogql_queries.error_handling import capture_experiment_metric_error_event
from products.experiments.backend.hogql_queries.experiment_metric_fingerprint import compute_metric_fingerprint
@@ -86,6 +87,8 @@ def _get_experiment_regular_metrics_for_hour_sync(hour: int) -> list[ExperimentR
get_experiment_stats_method(experiment),
experiment.exposure_criteria,
only_count_matured_users=experiment.only_count_matured_users,
+ excluded_variants=experiment.excluded_variants,
+ baseline_variant_key=get_experiment_fingerprint_baseline_variant_key(experiment),
)
experiment_metrics.append(
@@ -377,6 +380,8 @@ def _get_experiment_saved_metrics_for_hour_sync(hour: int) -> list[ExperimentSav
get_experiment_stats_method(experiment),
experiment.exposure_criteria,
only_count_matured_users=experiment.only_count_matured_users,
+ excluded_variants=experiment.excluded_variants,
+ baseline_variant_key=get_experiment_fingerprint_baseline_variant_key(experiment),
)
experiment_metrics.append(
diff --git a/products/experiments/backend/experiment_service.py b/products/experiments/backend/experiment_service.py
index 14d56b81eca1..874e2da7392c 100644
--- a/products/experiments/backend/experiment_service.py
+++ b/products/experiments/backend/experiment_service.py
@@ -56,7 +56,11 @@
from products.actions.backend.models.action import Action
from products.cohorts.backend.models.cohort import Cohort
from products.experiments.backend.flag_cleanup import build_cleanup_prompt, cleanup_plan
-from products.experiments.backend.hogql_queries import CONTROL_VARIANT_KEY, get_baseline_variant_key
+from products.experiments.backend.hogql_queries import (
+ CONTROL_VARIANT_KEY,
+ get_baseline_variant_key,
+ get_fingerprint_baseline_variant_key,
+)
from products.experiments.backend.hogql_queries.base_query_utils import is_threshold_supported_math
from products.experiments.backend.hogql_queries.experiment_metric_fingerprint import compute_metric_fingerprint
from products.experiments.backend.hogql_queries.exposure_query_logic import (
@@ -1318,6 +1322,7 @@ def create_experiment(
exposure_criteria,
only_count_matured_users=only_count_matured_users,
excluded_variants=excluded_variants,
+ baseline_variant_key=get_fingerprint_baseline_variant_key(stats_config, used_variant_keys),
)
if metrics_secondary is not None:
for metric in metrics_secondary:
@@ -1328,6 +1333,7 @@ def create_experiment(
exposure_criteria,
only_count_matured_users=only_count_matured_users,
excluded_variants=excluded_variants,
+ baseline_variant_key=get_fingerprint_baseline_variant_key(stats_config, used_variant_keys),
)
self.validate_no_duplicate_metric_uuids(metrics, metrics_secondary)
@@ -1700,6 +1706,7 @@ def _recompute_fingerprints(
exposure_criteria: dict | None,
only_count_matured_users: bool = False,
excluded_variants: list[str] | None = None,
+ variant_keys: list[str] | None = None,
) -> list[dict]:
"""Recompute fingerprints for a list of metrics. Returns a new list with updated fingerprints."""
stats_method = "bayesian" if stats_config is None else stats_config.get("method", "bayesian")
@@ -1713,6 +1720,7 @@ def _recompute_fingerprints(
exposure_criteria,
only_count_matured_users=only_count_matured_users,
excluded_variants=excluded_variants,
+ baseline_variant_key=get_fingerprint_baseline_variant_key(stats_config, variant_keys or []),
)
updated.append(metric_copy)
return updated
@@ -1945,6 +1953,7 @@ def launch_experiment(self, experiment: Experiment, *, request: Any | None = Non
experiment.stats_config,
experiment.exposure_criteria,
excluded_variants=experiment.excluded_variants or [],
+ variant_keys=flag_variant_keys,
),
)
@@ -2736,7 +2745,12 @@ def _maybe_open_cleanup_pr(
experiment.repository = picked_repository
experiment.save(update_fields=["repository"])
- plan = cleanup_plan(conclusion, experiment.feature_flag.variants or [])
+ variants = experiment.feature_flag.variants or []
+ plan = cleanup_plan(
+ conclusion,
+ variants,
+ get_baseline_variant_key(experiment.stats_config, self._variant_keys(variants)),
+ )
title, description = build_cleanup_prompt(experiment, flag_key, plan)
team = experiment.team
user_id = self.user.id
@@ -3740,6 +3754,7 @@ def update_experiment(
exposure_criteria,
only_count_matured_users=only_count_matured_users,
excluded_variants=excluded_variants,
+ variant_keys=self._resolved_variant_keys(experiment, feature_flag_config),
)
# --- metric ordering sync + validation -----------------------------
diff --git a/products/experiments/backend/flag_cleanup.py b/products/experiments/backend/flag_cleanup.py
index 0ed349e15479..3a2e69872e13 100644
--- a/products/experiments/backend/flag_cleanup.py
+++ b/products/experiments/backend/flag_cleanup.py
@@ -47,17 +47,20 @@ def _fully_rolled_out_variant(variants: list[dict]) -> str | None:
return at_100[0] if len(at_100) == 1 else None
-def cleanup_plan(conclusion: str, variants: list[dict]) -> CleanupPlan:
+def cleanup_plan(conclusion: str, variants: list[dict], baseline_variant_key: str | None = None) -> CleanupPlan:
"""Decide which variant's code path to keep, from the outcome and the flag's variants.
"won" keeps the shipped variant (the one rolled out to 100%); a plain win with nothing
- shipped falls back to the single non-control variant as a best guess. Every other
- outcome rolls back to the baseline ("control"). Anything uncertain is marked so the
+ shipped falls back to the single comparison variant as a best guess. Every other
+ outcome rolls back to the configured baseline. Anything uncertain is marked so the
operator (and the PR) flag it for human review rather than guessing silently.
"""
keys = variant_keys(variants)
- non_control = [k for k in keys if k != "control"]
- has_control = "control" in keys
+ configured_baseline_key = baseline_variant_key
+ is_configured_baseline_missing = configured_baseline_key is not None and configured_baseline_key not in keys
+ if baseline_variant_key not in keys:
+ baseline_variant_key = "control" if "control" in keys else (keys[0] if keys else None)
+ comparison_variants = [key for key in keys if key != baseline_variant_key]
def plan(keep: str | None, rationale: str, confident: bool) -> CleanupPlan:
return CleanupPlan(
@@ -75,20 +78,26 @@ def plan(keep: str | None, rationale: str, confident: bool) -> CleanupPlan:
f'The experiment won and variant "{shipped}" was shipped (100% rollout). Keep it as the new default.',
True,
)
- if len(non_control) == 1:
+ if len(comparison_variants) == 1:
return plan(
- non_control[0],
- f'The experiment won but no variant is at 100% rollout. Best guess: keep the winning variant "{non_control[0]}". Confirm which variant you kept in the PR.',
+ comparison_variants[0],
+ f'The experiment won but no variant is at 100% rollout. Best guess: keep the winning variant "{comparison_variants[0]}". Confirm which variant you kept in the PR.',
False,
)
return plan(
None,
- "The experiment won, but the winning variant can't be determined automatically (more than one non-control variant). Decide per code site and explain your choice in the PR.",
+ "The experiment won, but the winning variant can't be determined automatically (more than one comparison variant). Decide per code site and explain your choice in the PR.",
False,
)
- keep = "control" if has_control else (keys[0] if keys else None)
+ keep = baseline_variant_key
if conclusion in ("lost", "invalid"):
+ if is_configured_baseline_missing:
+ return plan(
+ keep,
+ f'The configured baseline "{configured_baseline_key}" no longer exists. Best guess: keep "{keep}" and confirm the rollback in the PR.',
+ False,
+ )
return plan(
keep,
f'The experiment {CONCLUSION_LABELS.get(conclusion, conclusion)}. Roll back to the baseline: keep "{keep}" and remove the feature.',
diff --git a/products/experiments/backend/hogql_queries/__init__.py b/products/experiments/backend/hogql_queries/__init__.py
index 2a2ca64d59a4..fee93f821791 100644
--- a/products/experiments/backend/hogql_queries/__init__.py
+++ b/products/experiments/backend/hogql_queries/__init__.py
@@ -1,3 +1,9 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from products.experiments.backend.models.experiment import Experiment
+
+
# The FF variant name for control
CONTROL_VARIANT_KEY = "control"
@@ -16,6 +22,27 @@ def get_baseline_variant_key(stats_config: dict | None, variant_keys: list[str])
return variant_keys[0]
+def get_experiment_baseline_variant_key(experiment: "Experiment") -> str:
+ variants = experiment.feature_flag.variants or []
+ variant_keys = [variant["key"] for variant in variants if variant.get("key")]
+ return get_baseline_variant_key(experiment.stats_config, variant_keys)
+
+
+def get_fingerprint_baseline_variant_key(stats_config: dict | None, variant_keys: list[str]) -> str | None:
+ configured = (stats_config or {}).get("baseline_variant_key")
+ if configured:
+ return configured
+ if not variant_keys or CONTROL_VARIANT_KEY in variant_keys:
+ return None
+ return variant_keys[0]
+
+
+def get_experiment_fingerprint_baseline_variant_key(experiment: "Experiment") -> str | None:
+ variants = experiment.feature_flag.variants or []
+ variant_keys = [variant["key"] for variant in variants if variant.get("key")]
+ return get_fingerprint_baseline_variant_key(experiment.stats_config, variant_keys)
+
+
# The FF variant name for multiple variants
MULTIPLE_VARIANT_KEY = "$multiple"
diff --git a/products/experiments/backend/hogql_queries/experiment_funnels_query_runner.py b/products/experiments/backend/hogql_queries/experiment_funnels_query_runner.py
index 073075bd5112..78dbb5036bc7 100644
--- a/products/experiments/backend/hogql_queries/experiment_funnels_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_funnels_query_runner.py
@@ -26,7 +26,7 @@
from posthog.hogql_queries.insights.funnels.funnels_query_runner import FunnelsQueryRunner
from posthog.hogql_queries.query_runner import QueryRunner
-from products.experiments.backend.hogql_queries import CONTROL_VARIANT_KEY
+from products.experiments.backend.hogql_queries import get_baseline_variant_key
from products.experiments.backend.hogql_queries.funnels_statistics_v2 import (
are_results_significant_v2,
calculate_credible_intervals_v2,
@@ -48,7 +48,10 @@ def __init__(self, *args, **kwargs):
self.experiment = Experiment.objects.get(id=self.query.experiment_id, team=self.team)
self.feature_flag = self.experiment.feature_flag
self.feature_flag_key = self.feature_flag.key_without_tombstone()
- self.variants = [variant["key"] for variant in self.feature_flag.variants]
+ configured_variant_keys = [variant["key"] for variant in self.feature_flag.variants]
+ self.baseline_variant_key = get_baseline_variant_key(self.experiment.stats_config, configured_variant_keys)
+ excluded_variants = set(self.experiment.excluded_variants or [])
+ self.variants = [key for key in configured_variant_keys if key not in excluded_variants]
if self.experiment.holdout:
self.variants.append(f"holdout-{self.experiment.holdout.id}")
@@ -158,7 +161,7 @@ def _get_variants_with_base_stats(
breakdown_value = cast(list[str], first_step["breakdown_value"])[0]
- if breakdown_value == CONTROL_VARIANT_KEY:
+ if breakdown_value == self.baseline_variant_key:
control_variant = ExperimentVariantFunnelsBaseStats(
key=breakdown_value,
success_count=int(success),
@@ -172,7 +175,7 @@ def _get_variants_with_base_stats(
)
if control_variant is None:
- raise ValueError("Control variant not found in count results")
+ raise ValueError(f"Baseline variant '{self.baseline_variant_key}' not found in count results")
return control_variant, test_variants
@@ -193,15 +196,14 @@ def _validate_event_variants(self, funnels_result: FunnelsQueryResponse):
if event_dict.get("order") == 0:
eventsWithOrderZero.append(event_dict)
- # Check if "control" is present
for event in eventsWithOrderZero:
event_variant = event.get("breakdown_value", [None])[0]
- if event_variant == "control":
+ if event_variant == self.baseline_variant_key:
errors[ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT] = False
break
# Check if at least one of the test variants is present
- test_variants = [variant for variant in self.variants if variant != "control"]
+ test_variants = [variant for variant in self.variants if variant != self.baseline_variant_key]
for event in eventsWithOrderZero:
event_variant = event.get("breakdown_value", [None])[0]
if event_variant in test_variants:
@@ -215,6 +217,12 @@ def _validate_event_variants(self, funnels_result: FunnelsQueryResponse):
def to_query(self) -> ast.SelectQuery:
raise ValueError(f"Cannot convert source query of type {self.query.funnels_query.kind} to query")
+ def get_cache_payload(self) -> dict:
+ payload = super().get_cache_payload()
+ payload["baseline_variant_key"] = self.baseline_variant_key
+ payload["excluded_variants"] = sorted(self.experiment.excluded_variants or [])
+ return payload
+
# Cache results for 24 hours
def cache_target_age(self, last_refresh: Optional[datetime], lazy: bool = False) -> Optional[datetime]:
if last_refresh is None:
diff --git a/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py b/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py
index 02941cae8cd9..b650f73c1ba2 100644
--- a/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py
+++ b/products/experiments/backend/hogql_queries/experiment_metric_fingerprint.py
@@ -29,6 +29,7 @@ def compute_metric_fingerprint(
exposure_criteria: dict | None = None,
only_count_matured_users: bool = False,
excluded_variants: list[str] | None = None,
+ baseline_variant_key: str | None = None,
) -> str:
"""
Compute fingerprint for a metric.
@@ -41,6 +42,7 @@ def compute_metric_fingerprint(
only_count_matured_users
excluded_variants: Variant keys excluded from analysis — changing the set
invalidates cached results since it alters which data is computed
+ baseline_variant_key: Explicit variant key used as the statistical baseline
Returns:
SHA256 hash string representing the metric fingerprint
@@ -74,6 +76,9 @@ def compute_metric_fingerprint(
if excluded_variants:
fingerprint_data["excluded_variants"] = sorted(set(excluded_variants))
+ if baseline_variant_key:
+ fingerprint_data["baseline_variant_key"] = baseline_variant_key
+
# Create deterministic JSON string with sorted keys at all levels
json_str = json.dumps(fingerprint_data, sort_keys=True, separators=(",", ":"))
diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.py
index ba154b73439e..422af940bc9f 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py
@@ -776,7 +776,7 @@ def _compute_breakdown_statistics(
variants_present = {v.key for v in breakdown_variants}
for expected_variant in self.variants:
if expected_variant not in variants_present:
- # Add missing variant with zero stats to avoid "No control variant found" error
+ # Add a missing variant with zero stats so baseline selection still succeeds.
breakdown_variants.append(
ExperimentStatsBase(
key=expected_variant,
@@ -988,6 +988,7 @@ def get_cache_payload(self) -> dict:
payload = super().get_cache_payload()
payload["experiment_response_version"] = 2
payload["stats_method"] = self.stats_method
+ payload["baseline_variant_key"] = self.baseline_variant_key
return payload
def _is_stale(self, last_refresh: Optional[datetime], lazy: bool = False) -> bool:
diff --git a/products/experiments/backend/hogql_queries/experiment_trends_query_runner.py b/products/experiments/backend/hogql_queries/experiment_trends_query_runner.py
index 09f072604be2..c720ca5f7ad7 100644
--- a/products/experiments/backend/hogql_queries/experiment_trends_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_trends_query_runner.py
@@ -37,7 +37,7 @@
from posthog.hogql_queries.query_runner import QueryRunner
from posthog.queries.trends.util import ALL_SUPPORTED_MATH_FUNCTIONS
-from products.experiments.backend.hogql_queries import CONTROL_VARIANT_KEY
+from products.experiments.backend.hogql_queries import get_baseline_variant_key
from products.experiments.backend.hogql_queries.trends_statistics_v2_continuous import (
are_results_significant_v2_continuous,
calculate_credible_intervals_v2_continuous,
@@ -65,7 +65,10 @@ def __init__(self, *args, **kwargs):
self.experiment = Experiment.objects.get(id=self.query.experiment_id, team=self.team)
self.feature_flag = self.experiment.feature_flag
self.feature_flag_key = self.feature_flag.key_without_tombstone()
- self.variants = [variant["key"] for variant in self.feature_flag.variants]
+ configured_variant_keys = [variant["key"] for variant in self.feature_flag.variants]
+ self.baseline_variant_key = get_baseline_variant_key(self.experiment.stats_config, configured_variant_keys)
+ excluded_variants = set(self.experiment.excluded_variants or [])
+ self.variants = [key for key in configured_variant_keys if key not in excluded_variants]
if self.experiment.holdout:
self.variants.append(f"holdout-{self.experiment.holdout.id}")
self.breakdown_key = f"$feature/{self.feature_flag_key}"
@@ -330,13 +333,16 @@ def _get_variants_with_base_stats(
test_variants = []
exposure_counts = {}
exposure_ratios = {}
+ included_variants = set(self.variants)
for result in exposure_results.results:
count = result.get("count", 0)
breakdown_value = result.get("breakdown_value")
+ if breakdown_value not in included_variants:
+ continue
exposure_counts[breakdown_value] = count
- control_exposure = exposure_counts.get(CONTROL_VARIANT_KEY, 0)
+ control_exposure = exposure_counts.get(self.baseline_variant_key, 0)
if control_exposure != 0:
for key, count in exposure_counts.items():
@@ -345,7 +351,9 @@ def _get_variants_with_base_stats(
for result in count_results.results:
count = result.get("count", 0)
breakdown_value = result.get("breakdown_value")
- if breakdown_value == CONTROL_VARIANT_KEY:
+ if breakdown_value not in included_variants:
+ continue
+ if breakdown_value == self.baseline_variant_key:
absolute_exposure = exposure_counts.get(breakdown_value, 0)
control_variant = ExperimentVariantTrendsBaseStats(
key=breakdown_value,
@@ -365,7 +373,7 @@ def _get_variants_with_base_stats(
)
if control_variant is None:
- raise ValueError("Control variant not found in count results")
+ raise ValueError(f"Baseline variant '{self.baseline_variant_key}' not found in count results")
return control_variant, test_variants
@@ -384,14 +392,13 @@ def _validate_event_variants(self, count_result: TrendsQueryResponse, exposure_r
if not count_result.results or not count_result.results[0]:
raise ValidationError(code="no-results", detail=json.dumps(errors))
- # Check if "control" is present
for event in count_result.results:
event_variant = event.get("breakdown_value")
- if event_variant == "control":
+ if event_variant == self.baseline_variant_key:
errors[ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT] = False
break
# Check if at least one of the test variants is present
- test_variants = [variant for variant in self.variants if variant != "control"]
+ test_variants = [variant for variant in self.variants if variant != self.baseline_variant_key]
for event in count_result.results:
event_variant = event.get("breakdown_value")
@@ -409,6 +416,12 @@ def _is_data_warehouse_query(self, query: TrendsQuery) -> bool:
def to_query(self) -> ast.SelectQuery:
raise ValueError(f"Cannot convert source query of type {self.query.count_query.kind} to query")
+ def get_cache_payload(self) -> dict:
+ payload = super().get_cache_payload()
+ payload["baseline_variant_key"] = self.baseline_variant_key
+ payload["excluded_variants"] = sorted(self.experiment.excluded_variants or [])
+ return payload
+
# Cache results for 24 hours
def cache_target_age(self, last_refresh: Optional[datetime], lazy: bool = False) -> Optional[datetime]:
if last_refresh is None:
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py
index b6c702fa4a97..0a6cdef4e663 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_excluded_variants.py
@@ -90,3 +90,15 @@ def test_fingerprint_changes_when_excluded_variants_change():
assert fp_two != fp_one
assert fp_two_reversed == fp_two, "Order of excluded keys must not affect fingerprint"
assert fp_one != fp_one_other
+
+
+def test_fingerprint_changes_when_baseline_variant_changes():
+ metric = {"kind": "ExperimentMeanMetric", "source": {"kind": "EventsNode", "event": "$pageview"}}
+ start = "2026-01-01T00:00:00+00:00"
+
+ legacy_fingerprint = compute_metric_fingerprint(metric, start)
+ control_fingerprint = compute_metric_fingerprint(metric, start, baseline_variant_key="control")
+ test_fingerprint = compute_metric_fingerprint(metric, start, baseline_variant_key="test")
+
+ assert legacy_fingerprint != control_fingerprint
+ assert control_fingerprint != test_fingerprint
diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_funnels_query_runner.py b/products/experiments/backend/hogql_queries/test/test_experiment_funnels_query_runner.py
index 9d3536610817..743a42fad005 100644
--- a/products/experiments/backend/hogql_queries/test/test_experiment_funnels_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/test_experiment_funnels_query_runner.py
@@ -1,6 +1,7 @@
import json
from datetime import datetime, timedelta
-from typing import cast
+from types import SimpleNamespace
+from typing import Any, cast
import pytest
from freezegun import freeze_time
@@ -118,6 +119,41 @@ def test_deleted_feature_flag_tombstone_uses_original_key_in_prepared_query(self
f"$feature/{original_key}",
)
+ def test_uses_configured_baseline_for_statistics_and_cache(self):
+ experiment = self.create_experiment()
+ experiment.stats_config = {"baseline_variant_key": "test"}
+ experiment.save(update_fields=["stats_config"])
+ query_runner = ExperimentFunnelsQueryRunner(
+ query=ExperimentFunnelsQuery(
+ experiment_id=experiment.id,
+ kind="ExperimentFunnelsQuery",
+ funnels_query=FunnelsQuery(series=[EventsNode(event="$pageview"), EventsNode(event="purchase")]),
+ ),
+ team=self.team,
+ )
+ result = cast(
+ Any,
+ SimpleNamespace(
+ results=[
+ [
+ {"order": 0, "count": 10, "breakdown_value": ["control"]},
+ {"order": 1, "count": 5, "breakdown_value": ["control"]},
+ ],
+ [
+ {"order": 0, "count": 12, "breakdown_value": ["test"]},
+ {"order": 1, "count": 8, "breakdown_value": ["test"]},
+ ],
+ ]
+ ),
+ )
+
+ baseline, comparisons = query_runner._get_variants_with_base_stats(result)
+ query_runner._validate_event_variants(result)
+
+ self.assertEqual(baseline.key, "test")
+ self.assertEqual([variant.key for variant in comparisons], ["control"])
+ self.assertEqual(query_runner.get_cache_payload()["baseline_variant_key"], "test")
+
@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner(self):
feature_flag = self.create_feature_flag()
diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_trends_query_runner.py b/products/experiments/backend/hogql_queries/test/test_experiment_trends_query_runner.py
index 9ad88a4b655d..a98cffd7bacf 100644
--- a/products/experiments/backend/hogql_queries/test/test_experiment_trends_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/test_experiment_trends_query_runner.py
@@ -1,5 +1,6 @@
import json
from datetime import datetime, timedelta
+from types import SimpleNamespace
from typing import Any, cast
import pytest
@@ -142,6 +143,87 @@ def test_deleted_feature_flag_tombstone_uses_original_key_in_prepared_queries(se
)
self.assertEqual(feature_flag_filter.value, [original_key])
+ def test_uses_configured_baseline_for_statistics_and_cache(self):
+ experiment = self.create_experiment()
+ experiment.stats_config = {"baseline_variant_key": "test"}
+ experiment.save(update_fields=["stats_config"])
+ query_runner = ExperimentTrendsQueryRunner(
+ query=ExperimentTrendsQuery(
+ experiment_id=experiment.id,
+ kind="ExperimentTrendsQuery",
+ count_query=TrendsQuery(series=[EventsNode(event="$pageview")]),
+ ),
+ team=self.team,
+ )
+ count_result = cast(
+ Any,
+ SimpleNamespace(
+ results=[
+ {"count": 5, "breakdown_value": "control"},
+ {"count": 8, "breakdown_value": "test"},
+ ]
+ ),
+ )
+ exposure_result = cast(
+ Any,
+ SimpleNamespace(
+ results=[
+ {"count": 10, "breakdown_value": "control"},
+ {"count": 12, "breakdown_value": "test"},
+ ]
+ ),
+ )
+
+ baseline, comparisons = query_runner._get_variants_with_base_stats(count_result, exposure_result)
+ query_runner._validate_event_variants(count_result, exposure_result)
+
+ self.assertEqual(baseline.key, "test")
+ self.assertEqual([variant.key for variant in comparisons], ["control"])
+ self.assertEqual(query_runner.get_cache_payload()["baseline_variant_key"], "test")
+
+ def test_excluded_variant_is_omitted_from_statistics(self):
+ feature_flag = self.create_feature_flag()
+ feature_flag.filters["multivariate"]["variants"].append(
+ {"key": "excluded", "name": "Excluded", "rollout_percentage": 0}
+ )
+ feature_flag.save(update_fields=["filters"])
+ experiment = self.create_experiment(feature_flag=feature_flag)
+ experiment.excluded_variants = ["excluded"]
+ experiment.save(update_fields=["excluded_variants"])
+ query_runner = ExperimentTrendsQueryRunner(
+ query=ExperimentTrendsQuery(
+ experiment_id=experiment.id,
+ kind="ExperimentTrendsQuery",
+ count_query=TrendsQuery(series=[EventsNode(event="$pageview")]),
+ ),
+ team=self.team,
+ )
+ count_result = cast(
+ Any,
+ SimpleNamespace(
+ results=[
+ {"count": 5, "breakdown_value": "control"},
+ {"count": 8, "breakdown_value": "test"},
+ {"count": 100, "breakdown_value": "excluded"},
+ ]
+ ),
+ )
+ exposure_result = cast(
+ Any,
+ SimpleNamespace(
+ results=[
+ {"count": 10, "breakdown_value": "control"},
+ {"count": 12, "breakdown_value": "test"},
+ {"count": 100, "breakdown_value": "excluded"},
+ ]
+ ),
+ )
+
+ baseline, comparisons = query_runner._get_variants_with_base_stats(count_result, exposure_result)
+
+ self.assertEqual(baseline.key, "control")
+ self.assertEqual([variant.key for variant in comparisons], ["test"])
+
@freeze_time("2020-01-01T12:00:00Z")
def test_query_runner(self):
feature_flag = self.create_feature_flag()
diff --git a/products/experiments/backend/hogql_queries/test/test_stats_config.py b/products/experiments/backend/hogql_queries/test/test_stats_config.py
index fae3d19e66fc..e13c713bee3e 100644
--- a/products/experiments/backend/hogql_queries/test/test_stats_config.py
+++ b/products/experiments/backend/hogql_queries/test/test_stats_config.py
@@ -366,6 +366,7 @@ def test_split_baseline_and_test_variants_missing_key_raises(self):
("stats_config_custom_baseline", {"baseline_variant_key": "test"}, ["control", "test"], "test"),
("no_control_falls_back_to_first_variant", None, ["variant_a", "variant_b"], "variant_a"),
("control_not_first_still_baseline", None, ["test", "control"], "control"),
+ ("capitalized_control_is_not_legacy_default", None, ["variant_a", "Control"], "variant_a"),
]
)
def test_experiment_query_runner_reads_baseline_from_stats_config(
diff --git a/products/experiments/backend/hogql_queries/utils.py b/products/experiments/backend/hogql_queries/utils.py
index c14ca690b173..d6b3f9a50cf5 100644
--- a/products/experiments/backend/hogql_queries/utils.py
+++ b/products/experiments/backend/hogql_queries/utils.py
@@ -120,12 +120,12 @@ def split_baseline_and_test_variants(
variants: list[V],
baseline_key: str = CONTROL_VARIANT_KEY,
) -> tuple[V, list[V]]:
- control_variants = [variant for variant in variants if variant.key == baseline_key]
- if not control_variants:
- raise ValueError("No control variant found")
- if len(control_variants) > 1:
- raise ValueError("Multiple control variants found")
- control_variant = control_variants[0]
+ baseline_variants = [variant for variant in variants if variant.key == baseline_key]
+ if not baseline_variants:
+ raise ValueError(f"Baseline variant '{baseline_key}' not found")
+ if len(baseline_variants) > 1:
+ raise ValueError(f"Multiple baseline variants found for '{baseline_key}'")
+ control_variant = baseline_variants[0]
test_variants = [variant for variant in variants if variant.key != baseline_key]
return control_variant, test_variants
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index 99e0905cceeb..99aa736ceb59 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -22,6 +22,7 @@
ExperimentApiMetric,
ExperimentParameters,
ExperimentRunningTimeCalculation,
+ ExperimentStatsConfig,
MultipleVariantHandling,
)
@@ -34,6 +35,7 @@
from products.ai_observability.backend.models.llm_prompt import LLMPrompt
from products.experiments.backend.experiment_service import ExperimentService
from products.experiments.backend.facade.contracts import CreateExperimentInput
+from products.experiments.backend.hogql_queries import get_experiment_fingerprint_baseline_variant_key
from products.experiments.backend.hogql_queries.experiment_metric_fingerprint import compute_metric_fingerprint
from products.experiments.backend.hogql_queries.exposure_query_logic import resolve_default_exposure_event
from products.experiments.backend.hogql_queries.utils import get_experiment_stats_method
@@ -90,21 +92,6 @@ def _with_split_percent(variants: list) -> list:
def _normalized_flag_variants(variants: list) -> list:
"""Returns a new list, leaving the caller's input (e.g. request.data) untouched."""
variants = deepcopy(variants)
- # Normalize a case-insensitive 'control' key (e.g. 'Control', 'CONTROL') down
- # to lowercase 'control'. 'control' is the conventional baseline key (the default
- # baseline when present), and a typo in casing was the leading cause of the
- # "Feature flag variants must contain a control variant" error in MCP traces —
- # most often from LLM-generated payloads. Only rewrite when no exact 'control'
- # match already exists, so we never collapse two distinct keys into a duplicate.
- existing_keys = {v.get("key") for v in variants if isinstance(v, dict)}
- if "control" not in existing_keys:
- for variant in variants:
- if not isinstance(variant, dict):
- continue
- key = variant.get("key")
- if isinstance(key, str) and key != "control" and key.lower() == "control":
- variant["key"] = "control"
- break
for variant in variants:
if isinstance(variant, dict) and "split_percent" in variant:
# split_percent wins in case both keys present, as rollout_percentage deprecated
@@ -132,6 +119,11 @@ class ExperimentRunningTimeCalculationField(serializers.JSONField):
pass
+@extend_schema_field(ExperimentStatsConfig) # type: ignore[arg-type]
+class ExperimentStatsConfigField(serializers.JSONField):
+ pass
+
+
class ExperimentBaseSerializer(UserAccessControlSerializerMixin, serializers.ModelSerializer):
"""Shared read-side fields for the full and list experiment serializers.
@@ -187,6 +179,14 @@ class ExperimentBaseSerializer(UserAccessControlSerializerMixin, serializers.Mod
"which historically lived in `parameters`."
),
)
+ stats_config = ExperimentStatsConfigField(
+ required=False,
+ allow_null=True,
+ help_text=(
+ "Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are "
+ "compared against. When omitted, analysis uses `control` if present, otherwise the first variant."
+ ),
+ )
excluded_variants = serializers.ListField(
child=serializers.CharField(),
required=False,
@@ -460,6 +460,7 @@ class Meta:
"exposure_cohort",
"parameters",
"running_time_calculation",
+ "stats_config",
"excluded_variants",
"secondary_metrics",
"saved_metrics",
@@ -474,7 +475,6 @@ class Meta:
"exposure_criteria",
"metrics",
"metrics_secondary",
- "stats_config",
"scheduling_config",
"allow_unknown_events",
"_create_in_folder",
@@ -577,6 +577,7 @@ def to_representation(self, instance):
instance.exposure_criteria,
only_count_matured_users=instance.only_count_matured_users,
excluded_variants=instance.excluded_variants or [],
+ baseline_variant_key=get_experiment_fingerprint_baseline_variant_key(instance),
)
return data
diff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.py
index 5143973d5eb2..576beff5da5d 100644
--- a/products/experiments/backend/recalculation.py
+++ b/products/experiments/backend/recalculation.py
@@ -30,6 +30,7 @@
from posthog.settings import CLICKHOUSE_CLUSTER
from posthog.temporal.common.client import sync_connect
+from products.experiments.backend.hogql_queries import get_experiment_fingerprint_baseline_variant_key
from products.experiments.backend.hogql_queries.experiment_metric_fingerprint import compute_metric_fingerprint
from products.experiments.backend.hogql_queries.utils import get_experiment_stats_method
from products.experiments.backend.models.experiment import (
@@ -391,6 +392,7 @@ def _recalc_fingerprints_for_run(experiment: Experiment, recalc: ExperimentMetri
experiment.exposure_criteria,
only_count_matured_users=experiment.only_count_matured_users,
excluded_variants=experiment.excluded_variants,
+ baseline_variant_key=get_experiment_fingerprint_baseline_variant_key(experiment),
)
fingerprints[metric_uuid] = compute_recalc_fingerprint(config_fp)
return fingerprints
@@ -455,6 +457,7 @@ def build_timeseries_cold_start_payload(experiment: Experiment) -> dict | None:
experiment.exposure_criteria,
only_count_matured_users=experiment.only_count_matured_users,
excluded_variants=experiment.excluded_variants,
+ baseline_variant_key=get_experiment_fingerprint_baseline_variant_key(experiment),
)
row = (
ExperimentMetricResult.objects.filter(
diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.py
index e90fbd07449f..c6bb21cd9634 100644
--- a/products/experiments/backend/temporal/recalculation_logic.py
+++ b/products/experiments/backend/temporal/recalculation_logic.py
@@ -28,6 +28,7 @@
from posthog.models.scoping import team_scope
from posthog.sync import database_sync_to_async_pool
+from products.experiments.backend.hogql_queries import get_experiment_fingerprint_baseline_variant_key
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window_end
from products.experiments.backend.hogql_queries.error_handling import (
classify_experiment_query_error,
@@ -600,6 +601,7 @@ def _calculate_experiment_metric_for_recalculation_sync(
experiment.exposure_criteria,
only_count_matured_users=experiment.only_count_matured_users,
excluded_variants=experiment.excluded_variants,
+ baseline_variant_key=get_experiment_fingerprint_baseline_variant_key(experiment),
)
recalc_fp = compute_recalc_fingerprint(config_fp)
diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.py
index 1b803415b017..740fbd274c26 100644
--- a/products/experiments/backend/temporal/test_recalculation_activities.py
+++ b/products/experiments/backend/temporal/test_recalculation_activities.py
@@ -839,6 +839,45 @@ def test_excluded_variants_change_recomputes(self):
mock_runner.assert_called_once()
+ def test_implicit_baseline_change_recomputes(self):
+ metric = _mean_metric("m1")
+ exp = self._experiment(flag_key="calc-baseline-order", metrics=[metric])
+ variants = [
+ {"key": "first", "name": "First", "rollout_percentage": 50},
+ {"key": "second", "name": "Second", "rollout_percentage": 50},
+ ]
+ exp.feature_flag.filters["multivariate"]["variants"] = variants
+ exp.feature_flag.save(update_fields=["filters"])
+ query_to = datetime.fromisoformat(_QUERY_TO)
+ recalc_fp = compute_recalc_fingerprint(
+ compute_metric_fingerprint(
+ metric,
+ exp.start_date,
+ get_experiment_stats_method(exp),
+ exp.exposure_criteria,
+ only_count_matured_users=exp.only_count_matured_users,
+ baseline_variant_key="first",
+ )
+ )
+ ExperimentMetricResult.objects.create(
+ experiment=exp,
+ metric_uuid="m1",
+ fingerprint=recalc_fp,
+ query_from=query_to,
+ query_to=query_to,
+ status=ExperimentMetricResult.Status.COMPLETED,
+ result={"stale": True},
+ )
+ exp.feature_flag.filters["multivariate"]["variants"] = list(reversed(variants))
+ exp.feature_flag.save(update_fields=["filters"])
+ recalc = self._recalc(exp, metric_uuids=["m1"])
+
+ with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ mock_runner.return_value.run.return_value.model_dump.return_value = {}
+ _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
+
+ mock_runner.assert_called_once()
+
def test_store_result_updates_existing_row_with_different_fingerprint_in_place(self):
# The unique constraint is (experiment, metric_uuid, query_to); fingerprint is not part of it. A row may
# already occupy that key under a different fingerprint (an earlier run written under the old per-run
diff --git a/products/experiments/backend/test/test_experiment_service.py b/products/experiments/backend/test/test_experiment_service.py
index d1166fb48361..1973bfd04d9e 100644
--- a/products/experiments/backend/test/test_experiment_service.py
+++ b/products/experiments/backend/test/test_experiment_service.py
@@ -47,6 +47,7 @@
_merge_saved_metric_links,
_resolve_scalar_updates,
)
+from products.experiments.backend.hogql_queries.experiment_metric_fingerprint import compute_metric_fingerprint
from products.experiments.backend.models.experiment import (
EXPOSURE_FROZEN_COHORT_KEY,
EXPOSURE_FROZEN_GROUP_KEY,
@@ -294,27 +295,33 @@ def test_metric_fingerprints_computed(self):
self._create_flag(key="fingerprint-test")
service = self._service()
- metrics = [
- {
- "kind": "ExperimentMetric",
- "metric_type": "mean",
- "uuid": "uuid-1",
- "source": {"kind": "EventsNode", "event": "$pageview"},
- },
- ]
+ metric = {
+ "kind": "ExperimentMetric",
+ "metric_type": "mean",
+ "uuid": "uuid-1",
+ "source": {"kind": "EventsNode", "event": "$pageview"},
+ }
+ metrics = [deepcopy(metric)]
+ start_date = timezone.now()
experiment = service.create_experiment(
name="Fingerprint Test",
feature_flag_key="fingerprint-test",
allow_unknown_events=True,
metrics=metrics,
+ start_date=start_date,
)
assert experiment.metrics is not None
assert len(experiment.metrics) == 1
- assert "fingerprint" in experiment.metrics[0]
- assert isinstance(experiment.metrics[0]["fingerprint"], str)
- assert len(experiment.metrics[0]["fingerprint"]) == 64 # SHA256 hex
+ assert experiment.metrics[0]["fingerprint"] == compute_metric_fingerprint(
+ metric=metric,
+ start_date=start_date,
+ stats_method=(experiment.stats_config or {}).get("method", "bayesian"),
+ exposure_criteria=experiment.exposure_criteria,
+ only_count_matured_users=experiment.only_count_matured_users,
+ excluded_variants=experiment.excluded_variants,
+ )
def test_lifecycle_save_does_not_clobber_concurrent_metric_change(self):
from django.utils import timezone
@@ -2488,6 +2495,7 @@ def test_duplicate_experiment_creates_draft_copy(self):
feature_flag_key="dup-source",
description="Original desc",
start_date=timezone.now(),
+ stats_config={"method": "bayesian", "baseline_variant_key": "test"},
)
dup = service.duplicate_experiment(source)
@@ -2499,6 +2507,8 @@ def test_duplicate_experiment_creates_draft_copy(self):
assert dup.end_date is None
assert dup.archived is False
assert dup.deleted is False
+ assert dup.stats_config is not None
+ assert dup.stats_config["baseline_variant_key"] == "test"
assert dup.id != source.id
# Same flag key → reuses the existing flag
assert dup.feature_flag.id == source.feature_flag.id
@@ -6876,6 +6886,42 @@ def test_create_experiment_validates_baseline_against_existing_flag_variants(sel
stats_config={"baseline_variant_key": "test"},
)
+ @parameterized.expand(
+ [
+ ("draft", None, None, True),
+ ("running", "2026-01-01T00:00:00+00:00", None, True),
+ ("paused", "2026-01-01T00:00:00+00:00", None, False),
+ ("completed", "2026-01-01T00:00:00+00:00", "2026-01-08T00:00:00+00:00", False),
+ ]
+ )
+ def test_update_experiment_allows_baseline_change_in_any_lifecycle_state(
+ self, _name: str, start_date: str | None, end_date: str | None, flag_active: bool
+ ) -> None:
+ flag = self._create_flag(
+ key=f"baseline-lifecycle-{_name}",
+ variants=[
+ {"key": "control", "name": "Control", "rollout_percentage": 50},
+ {"key": "test", "name": "Test", "rollout_percentage": 50},
+ ],
+ )
+ experiment = self._service().create_experiment(
+ name=f"Baseline lifecycle {_name}",
+ feature_flag_key=flag.key,
+ )
+ experiment.start_date = datetime.fromisoformat(start_date) if start_date else None
+ experiment.end_date = datetime.fromisoformat(end_date) if end_date else None
+ experiment.save(update_fields=["start_date", "end_date"])
+ flag.active = flag_active
+ flag.save(update_fields=["active"])
+
+ updated = self._service().update_experiment(
+ experiment,
+ {"stats_config": {"baseline_variant_key": "test"}},
+ )
+
+ assert updated.stats_config is not None
+ assert updated.stats_config["baseline_variant_key"] == "test"
+
def test_update_experiment_revalidates_baseline_when_variants_change(self) -> None:
self._create_flag(
key="baseline-update-flag",
diff --git a/products/experiments/backend/test/test_flag_cleanup.py b/products/experiments/backend/test/test_flag_cleanup.py
index d6929c74389b..86f2f248c831 100644
--- a/products/experiments/backend/test/test_flag_cleanup.py
+++ b/products/experiments/backend/test/test_flag_cleanup.py
@@ -23,9 +23,7 @@ class TestCleanupPlan(TestCase):
{"control", "red"},
True,
),
- # Won but nothing shipped: single non-control is a best guess, flagged low-confidence.
("won_unshipped_single", "won", _variants(("control", 50), ("test", 50)), "test", {"control"}, False),
- # Won but nothing shipped and multiple non-control: don't guess — keep nothing, low-confidence.
(
"won_unshipped_ambiguous",
"won",
@@ -55,3 +53,33 @@ def test_cleanup_plan(self, _name, conclusion, variants, keep, remove, confident
self.assertEqual(plan.keep_variant, keep)
self.assertEqual(set(plan.remove_variants), remove)
self.assertEqual(plan.confident, confident)
+
+ @parameterized.expand(
+ [
+ ("lost", "variant-b", {"control", "variant-a"}, True),
+ ("invalid", "variant-b", {"control", "variant-a"}, True),
+ ("inconclusive", "variant-b", {"control", "variant-a"}, False),
+ ("stopped_early", "variant-b", {"control", "variant-a"}, False),
+ ]
+ )
+ def test_cleanup_plan_uses_configured_baseline(self, conclusion, keep, remove, confident):
+ plan = cleanup_plan(
+ conclusion,
+ _variants(("control", 34), ("variant-a", 33), ("variant-b", 33)),
+ baseline_variant_key="variant-b",
+ )
+
+ self.assertEqual(plan.keep_variant, keep)
+ self.assertEqual(set(plan.remove_variants), remove)
+ self.assertEqual(plan.confident, confident)
+
+ def test_cleanup_plan_does_not_keep_missing_configured_baseline(self):
+ plan = cleanup_plan(
+ "lost",
+ _variants(("control", 50), ("variant-a", 50)),
+ baseline_variant_key="removed",
+ )
+
+ self.assertEqual(plan.keep_variant, "control")
+ self.assertEqual(plan.remove_variants, ["variant-a"])
+ self.assertFalse(plan.confident)
diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py
index 8e02969fc76e..595b92ccb382 100644
--- a/products/experiments/backend/test/test_presentation_api.py
+++ b/products/experiments/backend/test/test_presentation_api.py
@@ -3062,27 +3062,17 @@ def test_creating_multivariate_experiment_without_control_variant(self):
# The inferred baseline is pinned, not left implicit (order-sensitive).
self.assertEqual(response.json()["stats_config"]["baseline_variant_key"], "test_0")
- @parameterized.expand(
- [
- ("Control",),
- ("CONTROL",),
- ("cOnTrOl",),
- ]
- )
- def test_creating_experiment_normalizes_capitalized_control_key(self, control_key: str):
- # LLM callers often emit `Control` or `CONTROL` from natural-language input.
- # The serializer should rewrite it to lowercase `control` instead of rejecting,
- # since intent is unambiguous and the runtime treats `control` as a reserved key.
- ff_key = f"case-insensitive-{control_key.lower()}-{control_key}"
+ def test_creating_experiment_preserves_capitalized_variant_key(self):
+ ff_key = "capitalized-baseline"
response = self.client.post(
f"/api/projects/{self.team.id}/experiments/",
{
- "name": f"Capitalized control {control_key}",
+ "name": "Capitalized baseline",
"description": "",
"feature_flag_key": ff_key,
"parameters": {
"feature_flag_variants": [
- {"key": control_key, "name": "Control", "split_percent": 50},
+ {"key": "Control", "name": "Control", "split_percent": 50},
{"key": "test", "name": "Test", "split_percent": 50},
]
},
@@ -3092,41 +3082,11 @@ def test_creating_experiment_normalizes_capitalized_control_key(self, control_ke
self.assertEqual(response.status_code, status.HTTP_201_CREATED, response.content)
variants = response.json()["parameters"]["feature_flag_variants"]
- self.assertEqual([v["key"] for v in variants], ["control", "test"])
- # The persisted flag should also use lowercase `control`.
+ self.assertEqual([v["key"] for v in variants], ["Control", "test"])
+ self.assertEqual(response.json()["stats_config"]["baseline_variant_key"], "Control")
flag = FeatureFlag.objects.get(key=ff_key)
flag_keys = [v["key"] for v in flag.filters["multivariate"]["variants"]]
- self.assertEqual(flag_keys, ["control", "test"])
-
- def test_creating_experiment_does_not_collapse_when_control_already_present(self):
- # If both `control` and `Control` are passed, normalization must NOT run —
- # otherwise it would rewrite `Control` → `control` and produce two duplicate
- # entries. The downstream FeatureFlagSerializer may then accept (variants
- # preserved) or reject (duplicate-key error) — both prove the normalization
- # path was skipped. A wrong rewrite would surface as duplicate `control` keys
- # in the 201 response, which the assertion below would catch.
- ff_key = "control-and-capital-control"
- response = self.client.post(
- f"/api/projects/{self.team.id}/experiments/",
- {
- "name": "Both controls",
- "description": "",
- "feature_flag_key": ff_key,
- "parameters": {
- "feature_flag_variants": [
- {"key": "control", "name": "lowercase", "split_percent": 50},
- {"key": "Control", "name": "Capitalized", "split_percent": 50},
- ]
- },
- },
- format="json",
- )
-
- # Must land on a deterministic outcome — not silently bypass.
- self.assertIn(response.status_code, [status.HTTP_201_CREATED, status.HTTP_400_BAD_REQUEST])
- if response.status_code == status.HTTP_201_CREATED:
- variants = response.json()["parameters"]["feature_flag_variants"]
- self.assertEqual([v["key"] for v in variants], ["control", "Control"])
+ self.assertEqual(flag_keys, ["Control", "test"])
def test_creating_updating_experiment_with_group_aggregation(self):
ff_key = "a-b-tests"
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index ec5f6cd9a108..f266e8e08523 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -914,6 +914,44 @@ export interface ExperimentFeatureFlagInputApi {
ensure_experience_continuity?: boolean | null
}
+export interface ExperimentBayesianStatsConfigApi {
+ ci_level?: number | null
+ [key: string]: unknown
+}
+
+export interface ExperimentCupedStatsConfigApi {
+ enabled?: boolean | null
+ lookback_days?: number | null
+ [key: string]: unknown
+}
+
+export interface ExperimentFrequentistStatsConfigApi {
+ alpha?: number | null
+ sequential_testing_enabled?: boolean | null
+ sequential_tuning_parameter?: number | null
+ [key: string]: unknown
+}
+
+export type ExperimentStatsConfigApiMethod =
+ | (typeof ExperimentStatsConfigApiMethod)[keyof typeof ExperimentStatsConfigApiMethod]
+ | null
+
+export const ExperimentStatsConfigApiMethod = {
+ Bayesian: 'bayesian',
+ Frequentist: 'frequentist',
+} as const
+
+export interface ExperimentStatsConfigApi {
+ /** Variant key all other variants are compared against. The key must exist in the experiment's variants and cannot be excluded from analysis. When omitted, analysis uses 'control' if present, otherwise the first configured variant. */
+ baseline_variant_key?: string | null
+ bayesian?: ExperimentBayesianStatsConfigApi | null
+ cuped?: ExperimentCupedStatsConfigApi | null
+ frequentist?: ExperimentFrequentistStatsConfigApi | null
+ method?: ExperimentStatsConfigApiMethod
+ version?: number | null
+ [key: string]: unknown
+}
+
export interface ExperimentToSavedMetricApi {
readonly id: number
experiment: number
@@ -1426,6 +1464,8 @@ export interface ExperimentWriteApi {
parameters?: ExperimentParametersApi | null
/** Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`. */
running_time_calculation?: ExperimentRunningTimeCalculationApi | null
+ /** Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant. */
+ stats_config?: ExperimentStatsConfigApi | null
/**
* Variant keys to exclude from metric result calculations. Excluded variants are still served to users but omitted from statistical analysis. The baseline variant and holdout pseudo-variants cannot be excluded. Canonical home for what historically lived in `parameters.excluded_variants`.
* @nullable
@@ -1457,7 +1497,6 @@ export interface ExperimentWriteApi {
metrics?: _ExperimentApiMetricsListApi | null
/** Secondary metrics for additional measurements. Same format as primary metrics. */
metrics_secondary?: _ExperimentApiMetricsListApi | null
- stats_config?: unknown
scheduling_config?: unknown
/** Suppresses the validation that rejects metrics referencing events not yet ingested by this project. REQUIRES explicit user confirmation before being set to true — never flip this silently to retry a failed call. The default validation catches typo'd event names and missing instrumentation. Set this to true only when the user has confirmed the event is intentional (e.g. they are about to instrument it). */
allow_unknown_events?: boolean
@@ -1563,6 +1602,8 @@ export interface ExperimentApi {
parameters?: ExperimentParametersApi | null
/** Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`. */
running_time_calculation?: ExperimentRunningTimeCalculationApi | null
+ /** Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant. */
+ stats_config?: ExperimentStatsConfigApi | null
/**
* Variant keys to exclude from metric result calculations. Excluded variants are still served to users but omitted from statistical analysis. The baseline variant and holdout pseudo-variants cannot be excluded. Canonical home for what historically lived in `parameters.excluded_variants`.
* @nullable
@@ -1594,7 +1635,6 @@ export interface ExperimentApi {
metrics?: _ExperimentApiMetricsListApi | null
/** Secondary metrics for additional measurements. Same format as primary metrics. */
metrics_secondary?: _ExperimentApiMetricsListApi | null
- stats_config?: unknown
scheduling_config?: unknown
/** Suppresses the validation that rejects metrics referencing events not yet ingested by this project. REQUIRES explicit user confirmation before being set to true — never flip this silently to retry a failed call. The default validation catches typo'd event names and missing instrumentation. Set this to true only when the user has confirmed the event is intentional (e.g. they are about to instrument it). */
allow_unknown_events?: boolean
@@ -1696,6 +1736,8 @@ export interface PatchedExperimentWriteApi {
parameters?: ExperimentParametersApi | null
/** Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`. */
running_time_calculation?: ExperimentRunningTimeCalculationApi | null
+ /** Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant. */
+ stats_config?: ExperimentStatsConfigApi | null
/**
* Variant keys to exclude from metric result calculations. Excluded variants are still served to users but omitted from statistical analysis. The baseline variant and holdout pseudo-variants cannot be excluded. Canonical home for what historically lived in `parameters.excluded_variants`.
* @nullable
@@ -1727,7 +1769,6 @@ export interface PatchedExperimentWriteApi {
metrics?: _ExperimentApiMetricsListApi | null
/** Secondary metrics for additional measurements. Same format as primary metrics. */
metrics_secondary?: _ExperimentApiMetricsListApi | null
- stats_config?: unknown
scheduling_config?: unknown
/** Suppresses the validation that rejects metrics referencing events not yet ingested by this project. REQUIRES explicit user confirmation before being set to true — never flip this silently to retry a failed call. The default validation catches typo'd event names and missing instrumentation. Set this to true only when the user has confirmed the event is intentional (e.g. they are about to instrument it). */
allow_unknown_events?: boolean
diff --git a/services/mcp/scripts/generate-orval-schemas.mjs b/services/mcp/scripts/generate-orval-schemas.mjs
index f6739d91b323..8a1c854b5a33 100644
--- a/services/mcp/scripts/generate-orval-schemas.mjs
+++ b/services/mcp/scripts/generate-orval-schemas.mjs
@@ -26,7 +26,7 @@ import {
} from '@posthog/openapi-codegen'
import { discoverDefinitions, resolveSchemaPath } from './lib/definitions.mjs'
-import { stripEnumMinLength, stripUuidFormat } from './lib/schema-transforms.mjs'
+import { preserveAdditionalPropertiesForOrval, stripEnumMinLength, stripUuidFormat } from './lib/schema-transforms.mjs'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const mcpRoot = path.resolve(__dirname, '..')
@@ -221,6 +221,7 @@ for (const def of definitions) {
stripDefaultsFromPatchedSchemas(filtered)
stripUuidFormat(filtered)
+ preserveAdditionalPropertiesForOrval(filtered)
stripReadOnlyFromRequired(filtered)
applyNestedExclusions(filtered, schemaExclusions)
stripEnumMinLength(filtered)
diff --git a/services/mcp/scripts/lib/schema-transforms.mjs b/services/mcp/scripts/lib/schema-transforms.mjs
index 23efbf2f7e76..52febc4441db 100644
--- a/services/mcp/scripts/lib/schema-transforms.mjs
+++ b/services/mcp/scripts/lib/schema-transforms.mjs
@@ -44,3 +44,20 @@ export function stripUuidFormat(obj) {
stripUuidFormat(value)
}
}
+
+/**
+ * Orval ignores additionalProperties on objects with named fields. Intersecting
+ * the named object with an open record preserves unknown keys during Zod parsing.
+ */
+export function preserveAdditionalPropertiesForOrval(obj) {
+ if (!obj || typeof obj !== 'object') {
+ return
+ }
+ if (obj.additionalProperties === true && obj.properties) {
+ obj.allOf = [...(obj.allOf ?? []), { type: 'object', additionalProperties: {} }]
+ delete obj.additionalProperties
+ }
+ for (const value of Object.values(obj)) {
+ preserveAdditionalPropertiesForOrval(value)
+ }
+}
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 1340f786de4c..c176a6040e59 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -31042,6 +31042,43 @@ export namespace Schemas {
recommended_sample_size?: number | null;
}
+ export interface ExperimentBayesianStatsConfig {
+ ci_level?: number | null;
+ [key: string]: unknown;
+ }
+
+ export interface ExperimentCupedStatsConfig {
+ enabled?: boolean | null;
+ lookback_days?: number | null;
+ [key: string]: unknown;
+ }
+
+ export interface ExperimentFrequentistStatsConfig {
+ alpha?: number | null;
+ sequential_testing_enabled?: boolean | null;
+ sequential_tuning_parameter?: number | null;
+ [key: string]: unknown;
+ }
+
+ export type ExperimentStatsConfigMethod = typeof ExperimentStatsConfigMethod[keyof typeof ExperimentStatsConfigMethod] | null;
+
+
+ export const ExperimentStatsConfigMethod = {
+ Bayesian: 'bayesian',
+ Frequentist: 'frequentist',
+ } as const;
+
+ export interface ExperimentStatsConfig {
+ /** Variant key all other variants are compared against. The key must exist in the experiment's variants and cannot be excluded from analysis. When omitted, analysis uses 'control' if present, otherwise the first configured variant. */
+ baseline_variant_key?: string | null;
+ bayesian?: ExperimentBayesianStatsConfig | null;
+ cuped?: ExperimentCupedStatsConfig | null;
+ frequentist?: ExperimentFrequentistStatsConfig | null;
+ method?: ExperimentStatsConfigMethod;
+ version?: number | null;
+ [key: string]: unknown;
+ }
+
export interface ExperimentToSavedMetric {
readonly id: number;
experiment: number;
@@ -31225,6 +31262,8 @@ export namespace Schemas {
parameters?: ExperimentParameters | null;
/** Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`. */
running_time_calculation?: ExperimentRunningTimeCalculation | null;
+ /** Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant. */
+ stats_config?: ExperimentStatsConfig | null;
/**
* Variant keys to exclude from metric result calculations. Excluded variants are still served to users but omitted from statistical analysis. The baseline variant and holdout pseudo-variants cannot be excluded. Canonical home for what historically lived in `parameters.excluded_variants`.
* @nullable
@@ -31256,7 +31295,6 @@ export namespace Schemas {
metrics?: _ExperimentApiMetricsList | null;
/** Secondary metrics for additional measurements. Same format as primary metrics. */
metrics_secondary?: _ExperimentApiMetricsList | null;
- stats_config?: unknown;
scheduling_config?: unknown;
/** Suppresses the validation that rejects metrics referencing events not yet ingested by this project. REQUIRES explicit user confirmation before being set to true — never flip this silently to retry a failed call. The default validation catches typo'd event names and missing instrumentation. Set this to true only when the user has confirmed the event is intentional (e.g. they are about to instrument it). */
allow_unknown_events?: boolean;
@@ -32294,6 +32332,8 @@ export namespace Schemas {
parameters?: ExperimentParameters | null;
/** Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`. */
running_time_calculation?: ExperimentRunningTimeCalculation | null;
+ /** Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant. */
+ stats_config?: ExperimentStatsConfig | null;
/**
* Variant keys to exclude from metric result calculations. Excluded variants are still served to users but omitted from statistical analysis. The baseline variant and holdout pseudo-variants cannot be excluded. Canonical home for what historically lived in `parameters.excluded_variants`.
* @nullable
@@ -32325,7 +32365,6 @@ export namespace Schemas {
metrics?: _ExperimentApiMetricsList | null;
/** Secondary metrics for additional measurements. Same format as primary metrics. */
metrics_secondary?: _ExperimentApiMetricsList | null;
- stats_config?: unknown;
scheduling_config?: unknown;
/** Suppresses the validation that rejects metrics referencing events not yet ingested by this project. REQUIRES explicit user confirmation before being set to true — never flip this silently to retry a failed call. The default validation catches typo'd event names and missing instrumentation. Set this to true only when the user has confirmed the event is intentional (e.g. they are about to instrument it). */
allow_unknown_events?: boolean;
@@ -56426,6 +56465,8 @@ export namespace Schemas {
parameters?: ExperimentParameters | null;
/** Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`. */
running_time_calculation?: ExperimentRunningTimeCalculation | null;
+ /** Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant. */
+ stats_config?: ExperimentStatsConfig | null;
/**
* Variant keys to exclude from metric result calculations. Excluded variants are still served to users but omitted from statistical analysis. The baseline variant and holdout pseudo-variants cannot be excluded. Canonical home for what historically lived in `parameters.excluded_variants`.
* @nullable
@@ -56457,7 +56498,6 @@ export namespace Schemas {
metrics?: _ExperimentApiMetricsList | null;
/** Secondary metrics for additional measurements. Same format as primary metrics. */
metrics_secondary?: _ExperimentApiMetricsList | null;
- stats_config?: unknown;
scheduling_config?: unknown;
/** Suppresses the validation that rejects metrics referencing events not yet ingested by this project. REQUIRES explicit user confirmation before being set to true — never flip this silently to retry a failed call. The default validation catches typo'd event names and missing instrumentation. Set this to true only when the user has confirmed the event is intentional (e.g. they are about to instrument it). */
allow_unknown_events?: boolean;
diff --git a/services/mcp/src/generated/experiments/api.ts b/services/mcp/src/generated/experiments/api.ts
index 735a75c1ac38..9257fa97c98e 100644
--- a/services/mcp/src/generated/experiments/api.ts
+++ b/services/mcp/src/generated/experiments/api.ts
@@ -1077,6 +1077,59 @@ export const ExperimentsCreateBody = /* @__PURE__ */ zod
.describe(
'Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`.'
),
+ stats_config: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ baseline_variant_key: zod
+ .union([zod.string(), zod.null()])
+ .optional()
+ .describe(
+ "Variant key all other variants are compared against. The key must exist in the experiment's variants and cannot be excluded from analysis. When omitted, analysis uses 'control' if present, otherwise the first configured variant."
+ ),
+ bayesian: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ ci_level: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ cuped: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ enabled: zod.union([zod.boolean(), zod.null()]).optional(),
+ lookback_days: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ frequentist: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ alpha: zod.union([zod.number(), zod.null()]).optional(),
+ sequential_testing_enabled: zod.union([zod.boolean(), zod.null()]).optional(),
+ sequential_tuning_parameter: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ method: zod.union([zod.enum(['bayesian', 'frequentist']), zod.null()]).optional(),
+ version: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional()
+ .describe(
+ 'Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant.'
+ ),
excluded_variants: zod
.array(zod.string())
.nullish()
@@ -5754,7 +5807,6 @@ export const ExperimentsCreateBody = /* @__PURE__ */ zod
])
.optional()
.describe('Secondary metrics for additional measurements. Same format as primary metrics.'),
- stats_config: zod.unknown().optional(),
scheduling_config: zod.unknown().optional(),
allow_unknown_events: zod
.boolean()
@@ -6166,6 +6218,59 @@ export const ExperimentsPartialUpdateBody = /* @__PURE__ */ zod
.describe(
'Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`.'
),
+ stats_config: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ baseline_variant_key: zod
+ .union([zod.string(), zod.null()])
+ .optional()
+ .describe(
+ "Variant key all other variants are compared against. The key must exist in the experiment's variants and cannot be excluded from analysis. When omitted, analysis uses 'control' if present, otherwise the first configured variant."
+ ),
+ bayesian: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ ci_level: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ cuped: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ enabled: zod.union([zod.boolean(), zod.null()]).optional(),
+ lookback_days: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ frequentist: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ alpha: zod.union([zod.number(), zod.null()]).optional(),
+ sequential_testing_enabled: zod.union([zod.boolean(), zod.null()]).optional(),
+ sequential_tuning_parameter: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ method: zod.union([zod.enum(['bayesian', 'frequentist']), zod.null()]).optional(),
+ version: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional()
+ .describe(
+ 'Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant.'
+ ),
excluded_variants: zod
.array(zod.string())
.nullish()
@@ -10848,7 +10953,6 @@ export const ExperimentsPartialUpdateBody = /* @__PURE__ */ zod
])
.optional()
.describe('Secondary metrics for additional measurements. Same format as primary metrics.'),
- stats_config: zod.unknown().optional(),
scheduling_config: zod.unknown().optional(),
allow_unknown_events: zod
.boolean()
@@ -11229,6 +11333,59 @@ export const ExperimentsDuplicateCreateBody = /* @__PURE__ */ zod
.describe(
'Running-time calculator state: `minimum_detectable_effect`, `recommended_running_time`, `recommended_sample_size`, and `exposure_estimate_config`. Canonical home for these keys, which historically lived in `parameters`.'
),
+ stats_config: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ baseline_variant_key: zod
+ .union([zod.string(), zod.null()])
+ .optional()
+ .describe(
+ "Variant key all other variants are compared against. The key must exist in the experiment's variants and cannot be excluded from analysis. When omitted, analysis uses 'control' if present, otherwise the first configured variant."
+ ),
+ bayesian: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ ci_level: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ cuped: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ enabled: zod.union([zod.boolean(), zod.null()]).optional(),
+ lookback_days: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ frequentist: zod
+ .union([
+ zod.record(zod.string(), zod.unknown()).and(
+ zod.object({
+ alpha: zod.union([zod.number(), zod.null()]).optional(),
+ sequential_testing_enabled: zod.union([zod.boolean(), zod.null()]).optional(),
+ sequential_tuning_parameter: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional(),
+ method: zod.union([zod.enum(['bayesian', 'frequentist']), zod.null()]).optional(),
+ version: zod.union([zod.number(), zod.null()]).optional(),
+ })
+ ),
+ zod.null(),
+ ])
+ .optional()
+ .describe(
+ 'Statistical analysis settings. Set `baseline_variant_key` to the variant all other variants are compared against. When omitted, analysis uses `control` if present, otherwise the first variant.'
+ ),
excluded_variants: zod
.array(zod.string())
.nullish()
@@ -15914,7 +16071,6 @@ export const ExperimentsDuplicateCreateBody = /* @__PURE__ */ zod
])
.optional()
.describe('Secondary metrics for additional measurements. Same format as primary metrics.'),
- stats_config: zod.unknown().optional(),
scheduling_config: zod.unknown().optional(),
allow_unknown_events: zod
.boolean()
diff --git a/services/mcp/src/tools/generated/experiments.ts b/services/mcp/src/tools/generated/experiments.ts
index b81bb2b8c7a6..1f52c29a76f6 100644
--- a/services/mcp/src/tools/generated/experiments.ts
+++ b/services/mcp/src/tools/generated/experiments.ts
@@ -251,12 +251,12 @@ const experimentCreate = (): ToolBase {
it('removes minLength from a schema with enum', () => {
@@ -201,3 +205,36 @@ describe('stripUuidFormat', () => {
expect(() => stripUuidFormat(undefined)).not.toThrow()
})
})
+
+describe('preserveAdditionalPropertiesForOrval', () => {
+ it('converts permissive named objects to the equivalent schema form', () => {
+ const statsConfigSchema: Record = {
+ type: 'object',
+ properties: { method: { type: 'string' } },
+ additionalProperties: true,
+ }
+ const schema = {
+ type: 'object',
+ properties: {
+ stats_config: statsConfigSchema,
+ },
+ }
+
+ preserveAdditionalPropertiesForOrval(schema)
+
+ expect(statsConfigSchema).not.toHaveProperty('additionalProperties')
+ expect(statsConfigSchema.allOf).toEqual([{ type: 'object', additionalProperties: {} }])
+ })
+
+ it('leaves pure records and closed objects unchanged', () => {
+ const schema = {
+ records: { type: 'object', additionalProperties: true },
+ closed: { type: 'object', properties: {}, additionalProperties: false },
+ }
+
+ preserveAdditionalPropertiesForOrval(schema)
+
+ expect(schema.records.additionalProperties).toBe(true)
+ expect(schema.closed.additionalProperties).toBe(false)
+ })
+})
diff --git a/services/mcp/tests/unit/schema.experiments.test.ts b/services/mcp/tests/unit/schema.experiments.test.ts
index e3decf842c4e..d7219967cea3 100644
--- a/services/mcp/tests/unit/schema.experiments.test.ts
+++ b/services/mcp/tests/unit/schema.experiments.test.ts
@@ -1,8 +1,9 @@
import { describe, expect, it } from 'vitest'
+import { ExperimentsPartialUpdateBody } from '@/generated/experiments/api'
import { ExperimentExposureQuerySchema } from '@/schema/experiments'
-describe('Experiment exposure query schema', () => {
+describe('Experiment schemas', () => {
// getExposures round-trips the stored exposure_criteria through this hand-written schema,
// and Zod 4 z.object strips unknown keys on parse, so a criteria field missing from the
// schema silently degrades the query the backend receives.
@@ -31,4 +32,16 @@ describe('Experiment exposure query schema', () => {
expect(parsed.exposure_criteria).toEqual(exposureCriteria)
})
+
+ it('preserves extension fields in stats config', () => {
+ const statsConfig = {
+ method: 'bayesian' as const,
+ migrated_from: 123,
+ bayesian: { ci_level: 0.95, extension_field: true },
+ }
+
+ const parsed = ExperimentsPartialUpdateBody.parse({ stats_config: statsConfig })
+
+ expect(parsed.stats_config).toEqual(statsConfig)
+ })
})
|