feat(experiments): support configurable baseline variants

frontend/src/scenes/experiments/ExperimentForm/VariantsPanelCreateFeatureFlag.test.tsx

@@ -38,6 +38,19 @@ jest.mock('scenes/feature-flags/JSONEditorInput', () => ({
     ),
 }))
 
+jest.mock('lib/lemon-ui/LemonSelect', () => ({
+    LemonSelect: ({ value, options, onChange, disabledReason }: any) => (
+        <button
+            aria-label="Baseline variant"
+            type="button"
+            disabled={!!disabledReason}
+            onClick={() => onChange?.((options.find((option: any) => option.value !== value) ?? options[0])?.value)}
+        >
+            Select baseline variant
+        </button>
+    ),
+}))
+
 describe('VariantsPanelCreateFeatureFlag', () => {
     const mockOnChange = jest.fn()
 
@@ -114,10 +127,21 @@ describe('VariantsPanelCreateFeatureFlag', () => {
             expect(mockOnChange).toHaveBeenCalledWith(
                 expect.objectContaining({
                     variants: [expect.objectContaining({ key: 'baseline' }), expect.objectContaining({ key: 'test' })],
+                    stats_config: expect.objectContaining({ baseline_variant_key: 'baseline' }),
                 })
             )
         })
 
+        it('updates the baseline when selecting a different variant', async () => {
+            renderComponent(defaultExperiment)
+
+            await userEvent.click(screen.getByLabelText('Baseline variant'))
+
+            expect(mockOnChange).toHaveBeenCalledWith({
+                stats_config: expect.objectContaining({ baseline_variant_key: 'test' }),
+            })
+        })
+
         it('locks the control variant key for web experiments', () => {
             renderComponent({ ...defaultExperiment, type: 'web' })
 
@@ -294,6 +318,24 @@ describe('VariantsPanelCreateFeatureFlag', () => {
             const deleteButton = controlRow.querySelector('[data-attr^="delete-prop-filter"]')
             expect(deleteButton).not.toBeInTheDocument()
         })
+
+        it('does not show a delete button for an explicitly selected baseline', () => {
+            const experimentWithThreeVariants = {
+                ...defaultExperiment,
+                stats_config: { baseline_variant_key: 'test' },
+                feature_flag_config: flagConfig([
+                    { key: 'control', rollout_percentage: 33 },
+                    { key: 'test', rollout_percentage: 33 },
+                    { key: 'test-2', rollout_percentage: 34 },
+                ]),
+            }
+
+            const { container } = renderComponent(experimentWithThreeVariants)
+            const rows = container.querySelectorAll('tbody tr')
+
+            expect(rows[0].querySelector('[data-attr^="delete-prop-filter"]')).toBeInTheDocument()
+            expect(rows[1].querySelector('[data-attr^="delete-prop-filter"]')).not.toBeInTheDocument()
+        })
     })
 
     describe('rollout distribution', () => {

frontend/src/scenes/experiments/ExperimentForm/VariantsPanelCreateFeatureFlag.tsx

@@ -9,6 +9,7 @@ import { LemonButton } from 'lib/lemon-ui/LemonButton'
 import { LemonCheckbox } from 'lib/lemon-ui/LemonCheckbox'
 import { LemonField } from 'lib/lemon-ui/LemonField'
 import { LemonInput } from 'lib/lemon-ui/LemonInput'
+import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
 import { LemonSlider } from 'lib/lemon-ui/LemonSlider'
 import { Lettermark, LettermarkColor } from 'lib/lemon-ui/Lettermark'
 import { Link } from 'lib/lemon-ui/Link/Link'
@@ -20,7 +21,7 @@ import { teamLogic } from 'scenes/teamLogic'
 import type { Experiment, MultivariateFlagVariant } from '~/types'
 
 import { NEW_EXPERIMENT } from '../constants'
-import { ensureIsPercent, isEvenlyDistributed } from '../utils'
+import { ensureIsPercent, getBaselineVariantKey, isEvenlyDistributed } from '../utils'
 import {
     computeUpdatedVariantSplit,
     distributeVariantsEvenly,
@@ -35,6 +36,7 @@ interface VariantsPanelCreateFeatureFlagProps {
         feature_flag_key?: string
         variants?: MultivariateFlagVariant[]
         rollout_percentage?: number
+        stats_config?: Experiment['stats_config']
         ensure_experience_continuity?: boolean
     }) => void
     disabled?: boolean
@@ -102,6 +104,7 @@ export const VariantsPanelCreateFeatureFlag = ({
         filters?.groups?.[0]?.rollout_percentage ??
         NEW_EXPERIMENT.feature_flag_config?.filters?.groups?.[0]?.rollout_percentage ??
         100
+    const baselineVariantKey = getBaselineVariantKey(experiment)
 
     const updateRolloutPercentage = (value: number): void => {
         onChange({
@@ -127,15 +130,26 @@ export const VariantsPanelCreateFeatureFlag = ({
 
     const updateVariant = (index: number, updates: Partial<MultivariateFlagVariant>): void => {
         const newVariants = [...variants]
+        const previousKey = newVariants[index].key
         newVariants[index] = { ...newVariants[index], ...updates }
-        updateVariants(newVariants)
+        updateVariants(
+            newVariants,
+            previousKey === baselineVariantKey && updates.key !== undefined ? updates.key : baselineVariantKey
+        )
     }
 
-    const updateVariants = (newVariants: MultivariateFlagVariant[]): void => {
+    const updateVariants = (
+        newVariants: MultivariateFlagVariant[],
+        nextBaselineVariantKey = baselineVariantKey
+    ): void => {
         onChange({
             variants: newVariants,
             ensure_experience_continuity: ensureExperienceContinuity,
             rollout_percentage: rolloutPercentage,
+            stats_config: {
+                ...experiment.stats_config,
+                baseline_variant_key: nextBaselineVariantKey,
+            },
         })
     }
 
@@ -151,12 +165,25 @@ export const VariantsPanelCreateFeatureFlag = ({
     }
 
     const removeVariant = (index: number): void => {
-        if (variants.length <= 2 || index === 0) {
+        if (
+            variants.length <= 2 ||
+            variants[index].key === baselineVariantKey ||
+            (experiment.type === 'web' && variants[index].key === 'control')
+        ) {
             return
         }
         updateVariants(distributeVariantsEvenly(variants.filter((_, i) => i !== index)))
     }
 
+    const updateBaselineVariant = (baseline_variant_key: string): void => {
+        onChange({
+            stats_config: {
+                ...experiment.stats_config,
+                baseline_variant_key,
+            },
+        })
+    }
+
     return (
         <div className="flex flex-col gap-4">
             <div className={`flex gap-4 ${layout === 'vertical' ? 'flex-col' : 'flex-row'}`}>
@@ -183,9 +210,24 @@ export const VariantsPanelCreateFeatureFlag = ({
                             {!disabled && !isEvenlyDistributed(variants) && (
                                 <LemonBanner type="warning" className="mb-3">
                                     In most cases, experiments work best with an equal split. If you want to limit
-                                    exposure to the test variant, adjust the rollout percentage instead.
+                                    exposure to a variant, adjust the rollout percentage instead.
                                 </LemonBanner>
                             )}
+                            <LemonField.Pure label="Baseline variant" className="mb-4">
+                                <LemonSelect
+                                    value={baselineVariantKey}
+                                    options={variants.map(({ key }) => ({ value: key, label: key }))}
+                                    onChange={updateBaselineVariant}
+                                    disabledReason={
+                                        disabled
+                                            ? 'Cannot edit the baseline variant in edit mode'
+                                            : experiment.type === 'web'
+                                              ? "Web experiments use 'control' as the baseline variant"
+                                              : undefined
+                                    }
+                                    data-attr="experiment-baseline-variant"
+                                />
+                            </LemonField.Pure>
                             <table className="w-full">
                                 <thead>
                                     <tr className="text-sm font-bold">
@@ -282,15 +324,18 @@ export const VariantsPanelCreateFeatureFlag = ({
                                                             })}
                                                         </div>
                                                     )}
-                                                    {!disabled && variants.length > 2 && index > 0 && (
-                                                        <LemonButton
-                                                            icon={<IconTrash />}
-                                                            data-attr={`delete-prop-filter-${index}`}
-                                                            noPadding
-                                                            onClick={() => removeVariant(index)}
-                                                            tooltipPlacement="top-end"
-                                                        />
-                                                    )}
+                                                    {!disabled &&
+                                                        variants.length > 2 &&
+                                                        variant.key !== baselineVariantKey &&
+                                                        !(experiment.type === 'web' && variant.key === 'control') && (
+                                                            <LemonButton
+                                                                icon={<IconTrash />}
+                                                                data-attr={`delete-prop-filter-${index}`}
+                                                                noPadding
+                                                                onClick={() => removeVariant(index)}
+                                                                tooltipPlacement="top-end"
+                                                            />
+                                                        )}
                                                 </div>
                                             </td>
                                         </tr>

frontend/src/scenes/experiments/ExperimentForm/VariantsPanelLinkFeatureFlag.test.tsx

@@ -214,7 +214,7 @@ describe('VariantsPanelLinkFeatureFlag', () => {
             expect(screen.getByText('test')).toBeInTheDocument()
         })
 
-        it('highlights control variant with primary tag', () => {
+        it('highlights the default baseline variant with primary tag', () => {
             render(
                 <VariantsPanelLinkFeatureFlag
                     linkedFeatureFlag={baseFeatureFlag}
@@ -226,6 +226,19 @@ describe('VariantsPanelLinkFeatureFlag', () => {
             expect(controlTag).toHaveClass('LemonTag--primary')
         })
 
+        it('highlights an explicitly selected non-control baseline variant', () => {
+            render(
+                <VariantsPanelLinkFeatureFlag
+                    linkedFeatureFlag={baseFeatureFlag}
+                    setShowFeatureFlagSelector={mockSetShowFeatureFlagSelector}
+                    baselineVariantKey="test"
+                />
+            )
+
+            expect(screen.getByText('test').closest('.LemonTag')).toHaveClass('LemonTag--primary')
+            expect(screen.getByText('control').closest('.LemonTag')).not.toHaveClass('LemonTag--primary')
+        })
+
         it('renders multiple variants', () => {
             const flagWithMultipleVariants: FeatureFlagType = {
                 ...baseFeatureFlag,

frontend/src/scenes/experiments/ExperimentForm/VariantsPanelLinkFeatureFlag.tsx

@@ -10,13 +10,14 @@ import { urls } from 'scenes/urls'
 
 import type { FeatureFlagType } from '~/types'
 
-import { getFlagVariants } from '../utils'
+import { getDefaultBaselineVariantKey, getFlagVariants } from '../utils'
 
 interface VariantsPanelLinkFeatureFlagProps {
     linkedFeatureFlag: FeatureFlagType | null
     setShowFeatureFlagSelector: () => void
     disabled?: boolean
     onRemove?: () => void
+    baselineVariantKey?: string
 }
 
 const getTargetingSummary = (flag: FeatureFlagType): string[] => {
@@ -75,6 +76,7 @@ export const VariantsPanelLinkFeatureFlag = ({
     setShowFeatureFlagSelector,
     disabled = false,
     onRemove,
+    baselineVariantKey,
 }: VariantsPanelLinkFeatureFlagProps): JSX.Element => {
     if (!linkedFeatureFlag) {
         if (disabled) {
@@ -101,6 +103,7 @@ export const VariantsPanelLinkFeatureFlag = ({
     }
 
     const variants = getFlagVariants(linkedFeatureFlag)
+    const effectiveBaselineVariantKey = baselineVariantKey ?? getDefaultBaselineVariantKey(variants)
 
     return (
         <div>
@@ -155,7 +158,7 @@ export const VariantsPanelLinkFeatureFlag = ({
                         <div className="text-xs uppercase tracking-wide font-semibold text-muted">Variants</div>
                         <div className="flex flex-wrap gap-1.5">
                             {variants.map(({ key }) => (
-                                <LemonTag key={key} type={key === 'control' ? 'primary' : 'default'}>
+                                <LemonTag key={key} type={key === effectiveBaselineVariantKey ? 'primary' : 'default'}>
                                     {key}
                                 </LemonTag>
                             ))}

frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts

@@ -32,6 +32,7 @@ import { visionScannersCreate } from 'products/replay_vision/frontend/generated/
 import type { ProductCrossSellProperties, ProductIntentProperties } from '../../../lib/utils/product-intents'
 import type { ExperimentMetricUnion } from '../../../queries/schema/schema-general'
 import type { FeatureFlagType } from '../../../types'
+import type { ExperimentStatsMethod } from '../../../types'
 import { NEW_EXPERIMENT } from '../constants'
 import { FORM_MODES, experimentLogic } from '../experimentLogic'
 import { experimentSceneLogic } from '../experimentSceneLogic'
@@ -211,12 +212,38 @@ export interface createExperimentLogicActions {
         ensure_experience_continuity?: boolean
         feature_flag_key?: string
         rollout_percentage?: number
+        stats_config?: Experiment['stats_config']
         variants?: MultivariateFlagVariant[]
     }) => {
         config: {
             ensure_experience_continuity?: boolean | undefined
             feature_flag_key?: string | undefined
             rollout_percentage?: number | undefined
+            stats_config?:
+                | {
+                      baseline_variant_key?: string | undefined
+                      bayesian?:
+                          | {
+                                ci_level?: number | undefined
+                            }
+                          | undefined
+                      cuped?:
+                          | {
+                                enabled?: boolean | undefined
+                                lookback_days?: number | undefined
+                            }
+                          | undefined
+                      frequentist?:
+                          | {
+                                alpha?: number | undefined
+                                sequential_testing_enabled?: boolean | undefined
+                                sequential_tuning_parameter?: number | undefined
+                            }
+                          | undefined
+                      method?: ExperimentStatsMethod | undefined
+                      version?: number | undefined
+                  }
+                | undefined
             variants?: MultivariateFlagVariant[] | undefined
         }
     }
@@ -297,6 +324,7 @@ export const createExperimentLogic = kea<createExperimentLogicType>([
             feature_flag_key?: string
             variants?: MultivariateFlagVariant[]
             rollout_percentage?: number
+            stats_config?: Experiment['stats_config']
             ensure_experience_continuity?: boolean
         }) => ({ config }),
         saveExperiment: true,
@@ -337,6 +365,7 @@ export const createExperimentLogic = kea<createExperimentLogicType>([
                         ...(config.feature_flag_key !== undefined && {
                             feature_flag_key: config.feature_flag_key,
                         }),
+                        ...(config.stats_config !== undefined && { stats_config: config.stats_config }),
                         feature_flag_config: {
                             ...currentConfig,
                             filters,

frontend/src/scenes/experiments/ExperimentMetricForm.tsx

@@ -326,7 +326,7 @@ export function ExperimentMetricForm({
                                 <div className="space-y-2">
                                     <p>
                                         Metrics are only counted for users who have been exposed to the experiment. This
-                                        ensures fair comparison between control and test groups.
+                                        ensures a fair comparison between baseline and comparison groups.
                                     </p>
                                     <p>
                                         {exposureCriteria

frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx

@@ -117,11 +117,11 @@ export function DistributionTable(): JSX.Element {
     const variants = getExperimentVariants(experiment)
 
     /**
-     * We use this check to disable the toggle if there's only one test variant left.
+     * We use this check to disable the toggle if there's only one comparison variant left.
      * - not the baseline variant
      * - not excluded
      */
-    const hasOnlyOneTestVariant =
+    const hasOnlyOneComparisonVariant =
         variants.filter(({ key }) => key !== baselineKey && !excludedVariants.includes(key)).length <= 1
 
     const onSelectElement = (variant: string): void => {
@@ -195,15 +195,15 @@ export function DistributionTable(): JSX.Element {
                            * - the variant is not excluded: we have to allow re-including it
                            * - there's only one variant left when we remove the baseline
                            */
-                          const disableToggle = !excluded && hasOnlyOneTestVariant
+                          const disableToggle = !excluded && hasOnlyOneComparisonVariant
                           return (
                               <div className="flex items-center gap-2">
                                   <LemonSwitch
                                       checked={!excluded}
                                       onChange={(checked) => setVariantExcluded(key, !checked)}
                                       disabledReason={
                                           disableToggle
-                                              ? 'At least one test variant must remain in analysis'
+                                              ? 'At least one comparison variant must remain in analysis'
                                               : undefined
                                       }
                                       loading={experimentUpdateLoading}
@@ -315,9 +315,9 @@ export function DistributionTable(): JSX.Element {
                     variants are modified to show their relative rollout percentage.
                 </LemonBanner>
             )}
-            {excludedVariants.length > 0 && hasOnlyOneTestVariant && (
+            {excludedVariants.length > 0 && hasOnlyOneComparisonVariant && (
                 <LemonBanner type="warning" className="mb-4">
-                    At least one test variant must remain in analysis. Re-include a variant to exclude others.
+                    At least one comparison variant must remain in analysis. Re-include a variant to exclude others.
                 </LemonBanner>
             )}
             <LemonTable