Add data-driven background param models: ABCD, SmoothABCD, ExtendedABCD, SmoothExtendedABCD

.github/workflows/main.yml

@@ -277,7 +277,7 @@ jobs:
           - name: linearized
             tensor: test_tensor_symmetric.hdf5
             fit_args: >-
-              --binByBinStatType normal-additive --allowNegativePOI
+              --binByBinStatType normal-additive --allowNegativeParam
           - name: bb_full_normal_additive
             tensor: test_tensor.hdf5
             fit_args: >-
@@ -400,12 +400,12 @@ jobs:
       - name: bsm limits
         run: >-
           rabbit_limit.py $RABBIT_OUTDIR/test_tensor_bsm.hdf5 -o $RABBIT_OUTDIR/ --postfix bsm
-          -t 0 --unblind --expectSignal bsm1 0 --expectSignal bsm2 1 --allowNegativePOI --asymptoticLimits bsm1 --levels 0.05
+          -t 0 --unblind --expectSignal bsm1 0 --expectSignal bsm2 1 --allowNegativeParam --asymptoticLimits bsm1 --levels 0.05
 
       - name: bsm limits on channel 
         run: >-
           rabbit_limit.py $RABBIT_OUTDIR/test_tensor_bsm.hdf5 -o $RABBIT_OUTDIR/ --postfix bsm
-          -t 0 --unblind --poiModel Mixture bsm1 sig --expectSignal bsm1_sig_mixing 0 --asymptoticLimits bsm1_sig_mixing --levels 0.05 --allowNegativePOI
+          -t 0 --unblind --paramModel Mixture bsm1 sig --expectSignal bsm1_sig_mixing 0 --asymptoticLimits bsm1_sig_mixing --levels 0.05 --allowNegativeParam
 
   plotting:
     runs-on: [self-hosted, linux, x64]
@@ -543,9 +543,81 @@ jobs:
           rabbit_plot_likelihood_scan2D.py $RABBIT_OUTDIR/fitresults_scans.hdf5 -o $WEB_DIR/$PLOT_DIR 
           --title Rabbit --subtitle Preliminary --params sig slope_signal --titlePos 0 --legPos 'lower right'
 
+  make-abcd-tensors:
+    runs-on: [self-hosted, linux, x64]
+    needs: setenv
+
+    steps:
+      - env:
+          RABBIT_OUTDIR: ${{ needs.setenv.outputs.RABBIT_OUTDIR }}
+          PYTHONPATH: ${{ needs.setenv.outputs.PYTHONPATH }}
+          PATH: ${{ needs.setenv.outputs.PATH }}
+        run: |
+          echo "RABBIT_OUTDIR=${RABBIT_OUTDIR}" >> $GITHUB_ENV
+          echo "PYTHONPATH=${PYTHONPATH}" >> $GITHUB_ENV
+          echo "PATH=${PATH}" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v5
+
+      - name: make ABCD tensor
+        run: >-
+          python tests/make_abcd_tensor.py -o $RABBIT_OUTDIR/
+
+      - name: make extended ABCD tensor
+        run: >-
+          python tests/make_extended_abcd_tensor.py -o $RABBIT_OUTDIR/
+
+  abcd-fits:
+    runs-on: [self-hosted, linux, x64]
+    needs: [setenv, make-abcd-tensors]
+
+    steps:
+      - env:
+          RABBIT_OUTDIR: ${{ needs.setenv.outputs.RABBIT_OUTDIR }}
+          PYTHONPATH: ${{ needs.setenv.outputs.PYTHONPATH }}
+          PATH: ${{ needs.setenv.outputs.PATH }}
+        run: |
+          echo "RABBIT_OUTDIR=${RABBIT_OUTDIR}" >> $GITHUB_ENV
+          echo "PYTHONPATH=${PYTHONPATH}" >> $GITHUB_ENV
+          echo "PATH=${PATH}" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v5
+
+      - name: ABCD fit
+        run: >-
+          rabbit_fit.py $RABBIT_OUTDIR/test_abcd_tensor/rabbit_input.hdf5
+          -o $RABBIT_OUTDIR/ --postfix abcd
+          -t 0 --paramModel Mu
+          --paramModel ABCD nonprompt ch_A ch_B ch_C ch_D
+          --saveHists
+
+      - name: SmoothABCD fit
+        run: >-
+          rabbit_fit.py $RABBIT_OUTDIR/test_abcd_tensor/rabbit_input.hdf5
+          -o $RABBIT_OUTDIR/ --postfix smooth_abcd
+          -t 0 --paramModel Mu
+          --paramModel SmoothABCD pt nonprompt ch_A ch_B ch_C ch_D
+          --saveHists
+
+      - name: ExtendedABCD fit
+        run: >-
+          rabbit_fit.py $RABBIT_OUTDIR/test_extended_abcd_tensor/rabbit_input.hdf5
+          -o $RABBIT_OUTDIR/ --postfix extended_abcd
+          -t 0 --paramModel Mu
+          --paramModel ExtendedABCD nonprompt ch_Ax ch_Bx ch_A ch_B ch_C ch_D
+          --saveHists
+
+      - name: SmoothExtendedABCD fit
+        run: >-
+          rabbit_fit.py $RABBIT_OUTDIR/test_extended_abcd_tensor/rabbit_input.hdf5
+          -o $RABBIT_OUTDIR/ --postfix smooth_extended_abcd
+          -t 0 --paramModel Mu
+          --paramModel SmoothExtendedABCD pt nonprompt ch_Ax ch_Bx ch_A ch_B ch_C ch_D
+          --saveHists
+
   copy-clean:
     runs-on: [self-hosted, linux, x64]
-    needs: [setenv, symmerizations, sparse-fits, alternative-fits, regularization, bsm, plotting, likelihoodscans]
+    needs: [setenv, symmerizations, sparse-fits, alternative-fits, regularization, bsm, plotting, likelihoodscans, abcd-fits]
     if: always() 
     steps:
       - env:

CLAUDE.md

@@ -65,26 +65,40 @@ The workflow has three stages: **write input tensor → run fit → post-process
 Supports dense and sparse tensor representations, symmetric/asymmetric systematics, and log-normal or normal systematic types.
 
 ### 2. Fit: `rabbit/fitter.py`
-`Fitter` takes a `FitInputData` object (loaded from HDF5 by `rabbit/inputdata.py`), a `POIModel`, and options. It builds a differentiable negative log-likelihood using TensorFlow and minimizes it via SciPy. Results are written through a `Workspace`.
+`Fitter` takes a `FitInputData` object (loaded from HDF5 by `rabbit/inputdata.py`), a `ParamModel`, and options. It builds a differentiable negative log-likelihood using TensorFlow and minimizes it via SciPy. Results are written through a `Workspace`.
 
 ### 3. Output: `rabbit/workspace.py`
 `Workspace` collects post-fit histograms, covariance matrices, impacts, and likelihood scans into an HDF5 output file using `hist.Hist` objects (via the `wums` library).
 
-### POI Models: `rabbit/poi_models/poi_model.py`
-Base class `POIModel` defines `compute(poi)` which returns a `[1, nproc]` tensor scaling signal processes. Built-in models: `Mu` (default), `Ones`, `Mixture`. Custom models are loaded by providing a dotted Python path (e.g. `--poiModel mymod.MyModel`); the module must be on `$PYTHONPATH` with an `__init__.py`.
+### Param Models: `rabbit/param_models/param_model.py`
+Base class `ParamModel` defines `compute(param)` which returns a `[1, nproc]` tensor scaling process yields. Each model declares:
+- `nparams`: total parameters (POIs + model nuisances)
+- `npoi`: true parameters of interest (first `npoi` entries; reported as POIs in outputs)
+- `npou`: model nuisance parameters (`nparams - npoi`; reported as nuisances in outputs)
+
+Built-in models:
+- `Mu` (default): one POI per signal process
+- `Ones`: no parameters (all yields fixed to MC)
+- `Mixture`: mixing POIs between pairs of processes
+- `ABCD`: data-driven background estimation using four regions; `npoi=0`, `npou=3*n_bins`. CLI: `--paramModel ABCD <process> <ch_A> [ax:val ...] <ch_B> [ax:val ...] <ch_C> [ax:val ...] <ch_D> [ax:val ...]` where `ax:val` pairs optionally select a single bin along a named axis (e.g. `iso:0`). Regions A, B, C have free parameters; D is predicted as `a*c/b` times an MC correction factor.
+- `SmoothABCD`: like ABCD but one axis is parameterised with an exponential polynomial `val(x)=exp(p_0+p_1·x̃+...)` instead of per-bin free parameters. CLI: `--paramModel SmoothABCD <axis> [order:N] <process> <ch_A> ... <ch_D>`. Default order=1 (log-linear). Reduces parameters from `3·n_bins` to `3·n_outer·(order+1)`.
+- `ExtendedABCD`: 6-region ABCD using two sideband bins in the x direction (Ax, Bx further from signal, A/B in the middle). Fake rate is log-linearly extrapolated: `D = C·Ax·B² / (Bx·A²)`. `npoi=0`, `npou=5·n_bins`. CLI: `--paramModel ExtendedABCD <process> <ch_Ax> [ax:val ...] <ch_Bx> [ax:val ...] <ch_A> [ax:val ...] <ch_B> [ax:val ...] <ch_C> [ax:val ...] <ch_D> [ax:val ...]`.
+- `SmoothExtendedABCD`: like `ExtendedABCD` but all five free-parameter regions (A, B, C, Ax, Bx) are parameterised with an exponential polynomial along one smoothing axis. `npoi=0`, `npou=5·n_outer·(order+1)`. CLI: `--paramModel SmoothExtendedABCD <axis> [order:N] <process> <ch_Ax> [ax:val ...] <ch_Bx> [ax:val ...] <ch_A> [ax:val ...] <ch_B> [ax:val ...] <ch_C> [ax:val ...] <ch_D> [ax:val ...]`.
+
+Custom models are loaded by providing a dotted Python path (e.g. `--paramModel mymod.MyModel`); the module must be on `$PYTHONPATH` with an `__init__.py`.
 
 ### Mappings: `rabbit/mappings/`
-Base class `Mapping` in `mapping.py` defines `compute_flat(params, observables)`, which is a differentiable transformation of the flat bin vector. The framework propagates uncertainties through it via automatic differentiation (`tf.GradientTape`). Built-in mappings (`Select`, `Project`, `Normalize`, `Ratio`, `Normratio`) live in `project.py` and `ratio.py`. Custom mappings follow the same pattern as POI models.
+Base class `Mapping` in `mapping.py` defines `compute_flat(params, observables)`, which is a differentiable transformation of the flat bin vector. The framework propagates uncertainties through it via automatic differentiation (`tf.GradientTape`). Built-in mappings (`Select`, `Project`, `Normalize`, `Ratio`, `Normratio`) live in `project.py` and `ratio.py`. Custom mappings follow the same pattern as param models.
 
 ### Bin scripts: `bin/`
 Entry points registered in `pyproject.toml`. The main one is `rabbit_fit.py`; others are diagnostic/plotting scripts. All use `rabbit/parsing.py` for shared CLI arguments.
 
 ## Custom Extensions
 
-Custom mappings and POI models must:
-1. Subclass `Mapping` or `POIModel` respectively
-2. Implement `compute_flat` (mapping) or `compute` (POI model) as TF-differentiable functions
+Custom mappings and param models must:
+1. Subclass `Mapping` or `ParamModel` respectively
+2. Implement `compute_flat` (mapping) or `compute` (param model) as TF-differentiable functions
 3. Be importable from `$PYTHONPATH` (directory needs `__init__.py`)
-4. Be referenced with dotted module path: `-m my_package.MyMapping` or `--poiModel my_package.MyModel`
+4. Be referenced with dotted module path: `-m my_package.MyMapping` or `--paramModel my_package.MyModel`
 
 See `tests/my_custom_mapping.py` and `tests/my_custom_model.py` for examples.

bin/rabbit_fit.py

@@ -15,8 +15,8 @@
 from rabbit.mappings import helpers as mh
 from rabbit.mappings import mapping as mp
 from rabbit.mappings import project
-from rabbit.poi_models import helpers as ph
-from rabbit.poi_models import poi_model
+from rabbit.param_models import helpers as ph
+from rabbit.param_models import param_model
 from rabbit.regularization import helpers as rh
 from rabbit.regularization.lcurve import l_curve_optimize_tau, l_curve_scan_tau
 from rabbit.tfhelpers import edmval_cov
@@ -302,11 +302,11 @@ def save_hists(args, mappings, fitter, ws, prefit=True, profile=False):
             ):
                 # saturated likelihood test
 
-                saturated_model = poi_model.SaturatedProjectModel(
+                saturated_model = param_model.SaturatedProjectModel(
                     fitter.indata, mapping.channel_info
                 )
-                composite_model = poi_model.CompositePOIModel(
-                    [fitter.poi_model, saturated_model]
+                composite_model = param_model.CompositeParamModel(
+                    [fitter.param_model, saturated_model]
                 )
 
                 fitter_saturated = copy.deepcopy(fitter)
@@ -449,7 +449,9 @@ def fit(args, fitter, ws, dofit=True):
     nllvalreduced = fitter.reduced_nll().numpy()
 
     ndfsat = (
-        tf.size(fitter.nobs) - fitter.poi_model.npoi - fitter.indata.nsystnoconstraint
+        tf.size(fitter.nobs)
+        - fitter.param_model.nparams
+        - fitter.indata.nsystnoconstraint
     ).numpy()
 
     chi2_val = 2.0 * nllvalreduced
@@ -574,12 +576,12 @@ def main():
 
     indata = inputdata.FitInputData(args.filename, args.pseudoData)
 
-    margs = args.poiModel
-    poi_model = ph.load_model(margs[0], indata, *margs[1:], **vars(args))
+    model_specs = args.paramModel or [["Mu"]]
+    param_model = ph.load_models(model_specs, indata, **vars(args))
 
     ifitter = fitter.Fitter(
         indata,
-        poi_model,
+        param_model,
         args,
         do_blinding=any(blinded_fits),
         globalImpactsFromJVP=not args.globalImpactsDisableJVP,
@@ -615,8 +617,8 @@ def main():
         "meta_info": output_tools.make_meta_info_dict(args=args),
         "meta_info_input": ifitter.indata.metadata,
         "procs": ifitter.indata.procs,
-        "pois": ifitter.poi_model.pois,
-        "nois": ifitter.parms[ifitter.poi_model.npoi :][indata.noiidxs],
+        "pois": ifitter.param_model.params[: ifitter.param_model.npoi],
+        "nois": ifitter.parms[ifitter.param_model.nparams :][indata.noiidxs],
     }
 
     with workspace.Workspace(

bin/rabbit_limit.py

@@ -13,7 +13,7 @@
 
 from rabbit import asymptotic_limits, fitter, inputdata, parsing, workspace
 from rabbit.mappings import helpers as mh
-from rabbit.poi_models import helpers as ph
+from rabbit.param_models import helpers as ph
 from rabbit.tfhelpers import edmval_cov
 
 from wums import output_tools, logging  # isort: skip
@@ -136,14 +136,16 @@ def do_asymptotic_limits(args, fitter, ws, data_values, mapping=None, fit_data=F
             fun = None
 
             idx = np.where(fitter.parms.astype(str) == key)[0][0]
-            if key in fitter.poi_model.pois.astype(str):
+            if key in fitter.param_model.params[: fitter.param_model.npoi].astype(str):
                 is_poi = True
                 xbest = fitter_asimov.get_poi()[idx]
                 xobs = fitter.get_poi()[idx]
             elif key in fitter.parms.astype(str):
                 is_poi = False
-                xbest = fitter_asimov.get_theta()[idx - fitter_asimov.poi_model.npoi]
-                xobs = fitter.get_theta()[idx - fitter.poi_model.npoi]
+                xbest = fitter_asimov.get_theta()[
+                    idx - fitter_asimov.param_model.nparams
+                ]
+                xobs = fitter.get_theta()[idx - fitter.param_model.nparams]
 
             xerr = fitter_asimov.cov[idx, idx] ** 0.5
 
@@ -156,7 +158,7 @@ def do_asymptotic_limits(args, fitter, ws, data_values, mapping=None, fit_data=F
             fitter.cov.assign(cov)
             xobs_err = fitter.cov[idx, idx] ** 0.5
 
-            if is_poi and not args.allowNegativePOI:
+            if is_poi and not args.allowNegativeParam:
                 xerr = 2 * xerr * xbest**0.5
                 xobs_err = 2 * xobs_err * xobs**0.5
 
@@ -262,10 +264,10 @@ def main():
 
     indata = inputdata.FitInputData(args.filename, args.pseudoData)
 
-    margs = args.poiModel
-    poi_model = ph.load_model(margs[0], indata, *margs[1:], **vars(args))
+    model_specs = args.paramModel or [["Mu"]]
+    param_model = ph.load_models(model_specs, indata, **vars(args))
 
-    ifitter = fitter.Fitter(indata, poi_model, args, do_blinding=any(blinded_fits))
+    ifitter = fitter.Fitter(indata, param_model, args, do_blinding=any(blinded_fits))
 
     # mapping for observables and parameters
     if len(args.mapping) > 0:
@@ -282,8 +284,8 @@ def main():
         "meta_info": output_tools.make_meta_info_dict(args=args),
         "meta_info_input": ifitter.indata.metadata,
         "procs": ifitter.indata.procs,
-        "pois": ifitter.poi_model.pois,
-        "nois": ifitter.parms[ifitter.poi_model.npoi :][indata.noiidxs],
+        "pois": ifitter.param_model.params[: ifitter.param_model.npoi],
+        "nois": ifitter.parms[ifitter.param_model.nparams :][indata.noiidxs],
     }
 
     with workspace.Workspace(