Add optimizer from bayesian-optimization

.tools/envs/testenv-linux.yml

@@ -29,6 +29,7 @@ dependencies:
   - annotated-types  # dev, tests
   - iminuit  # dev, tests
   - pip:  # dev, tests, docs
+      - bayesian-optimization>=2.0.4  # dev, tests
       - nevergrad  # dev, tests
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests

.tools/envs/testenv-numpy.yml

@@ -27,6 +27,7 @@ dependencies:
   - annotated-types  # dev, tests
   - iminuit  # dev, tests
   - pip:  # dev, tests, docs
+      - bayesian-optimization>=2.0.4  # dev, tests
       - nevergrad  # dev, tests
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests

.tools/envs/testenv-others.yml

@@ -27,6 +27,7 @@ dependencies:
   - annotated-types  # dev, tests
   - iminuit  # dev, tests
   - pip:  # dev, tests, docs
+      - bayesian-optimization>=2.0.4  # dev, tests
       - nevergrad  # dev, tests
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests

.tools/envs/testenv-pandas.yml

@@ -27,6 +27,7 @@ dependencies:
   - annotated-types  # dev, tests
   - iminuit  # dev, tests
   - pip:  # dev, tests, docs
+      - bayesian-optimization>=2.0.4  # dev, tests
       - nevergrad  # dev, tests
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests

.tools/envs/testenv-plotly.yml

@@ -27,6 +27,7 @@ dependencies:
   - annotated-types  # dev, tests
   - iminuit  # dev, tests
   - pip:  # dev, tests, docs
+      - bayesian-optimization>=2.0.4  # dev, tests
       - nevergrad  # dev, tests
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests

docs/source/algorithms.md

@@ -4043,6 +4043,87 @@ these optimizers, you need to have
       initialization for speed. Default is False.
 ```
 
+## Bayesian Optimization
+
+We wrap the
+[BayesianOptimization](https://github.com/bayesian-optimization/BayesianOptimization)
+package. To use it, you need to have
+[bayesian-optimization](https://pypi.org/project/bayesian-optimization/) installed.
+
+```{eval-rst}
+.. dropdown::  bayes_opt
+
+    .. code-block::
+
+        "bayes_opt"
+
+    Minimize a scalar function using Bayesian Optimization with Gaussian Processes.
+
+    This optimizer wraps the BayesianOptimization package (:cite:`Nogueira2014`), which 
+    implements Bayesian optimization using Gaussian processes to build probabilistic 
+    models of the objective function. Bayesian optimization is particularly effective 
+    for expensive black-box functions where gradient information is not available.
+
+    The algorithm requires finite bounds for all parameters.
+
+    The bayes_opt wrapper preserves the default parameter values from the underlying 
+    BayesianOptimization package where appropriate.
+
+    bayes_opt supports the following options:
+
+    - **init_points** (PositiveInt): Number of random exploration points to evaluate before 
+      starting optimization. Default is 5.
+
+    - **n_iter** (PositiveInt): Number of Bayesian optimization iterations to perform after 
+      the initial random exploration. Default is 25.
+
+    - **verbose** (Literal[0, 1, 2]): Verbosity level from 0 (silent) to 2 (most verbose). Default is 0.
+
+    - **kappa** (NonNegativeFloat): Parameter to balance exploration versus exploitation trade-off 
+      for the Upper Confidence Bound acquisition function. Higher values mean more exploration. 
+      This parameter is only used if the acquisition function is set to "ucb" or "upper_confidence_bound" 
+      and when a configured instance of an AcquisitionFunction object is not passed. Default is 2.576.
+
+    - **xi** (PositiveFloat): Parameter to balance exploration versus exploitation trade-off 
+      for the Expected Improvement or Probability of Improvement acquisition functions. 
+      Higher values mean more exploration. This parameter is only used if the acquisition function 
+      is set to "ei", "expected_improvement", "poi", or "probability_of_improvement" 
+      and when a configured instance of an AcquisitionFunction object is not passed. Default is 0.01.
+
+    - **exploration_decay** (float | None): Rate at which exploration decays over time.
+      Default is None (no decay).
+
+    - **exploration_decay_delay** (NonNegativeInt | None): Delay for decay. If None, 
+      decay is applied from the start. Default is None.
+
+    - **random_state** (int | None): Random seed for reproducible results. Default is None.
+
+    - **acquisition_function** (str | AcquisitionFunction | Type[AcquisitionFunction] | None): Strategy for selecting 
+      the next evaluation point. Options include:
+      - "ucb" or "upper_confidence_bound": Upper Confidence Bound 
+      - "ei" or "expected_improvement": Expected Improvement 
+      - "poi" or "probability_of_improvement": Probability of Improvement
+      Default is None (uses package default).
+
+    - **allow_duplicate_points** (bool): Whether to allow re-evaluation of the same point.
+      Default is False.
+
+    - **enable_sdr** (bool): Enable Sequential Domain Reduction, which progressively 
+      narrows the search space around promising regions. Default is False.
+
+    - **sdr_gamma_osc** (float): Oscillation parameter for SDR. Default is 0.7.
+
+    - **sdr_gamma_pan** (float): Panning parameter for SDR. Default is 1.0.
+
+    - **sdr_eta** (float): Zooming parameter for SDR. Default is 0.9.
+
+    - **sdr_minimum_window** (NonNegativeFloat): Minimum window size for SDR. Default is 0.0.
+
+    - **alpha** (float): Noise parameter for the Gaussian Process. Default is 1e-6.
+
+    - **n_restarts** (int): Number of times to restart the optimizer. Default is 1.
+```
+
 ## References
 
 ```{eval-rst}

docs/source/refs.bib

@@ -927,4 +927,31 @@ @InProceedings{Zambrano2013
   doi = {10.1109/CEC.2013.6557848},
 }
 
+@Misc{Nogueira2014,
+  author={Fernando Nogueira},
+  title={{Bayesian Optimization}: Open source constrained global optimization tool for {Python}},
+  year={2014--},
+  url="https://github.com/bayesian-optimization/BayesianOptimization"
+}
+
+@article{Stander2002,
+  author={Stander, Nielen and Craig, Kenneth},
+  year={2002},
+  month={06},
+  pages={},
+  title={On the robustness of a simple domain reduction scheme for simulation-based optimization},
+  volume={19},
+  journal={International Journal for Computer-Aided Engineering and Software (Eng. Comput.)},
+  doi={10.1108/02644400210430190}
+}
+
+@inproceedings{gardner2014bayesian,
+  title={Bayesian optimization with inequality constraints.},
+  author={Gardner, Jacob R and Kusner, Matt J and Xu, Zhixiang Eddie and Weinberger, Kilian Q and Cunningham, John P},
+  booktitle={ICML},
+  volume={2014},
+  pages={937--945},
+  year={2014}
+}
+
 @Comment{jabref-meta: databaseType:bibtex;}

environment.yml

@@ -39,6 +39,7 @@ dependencies:
   - annotated-types  # dev, tests
   - iminuit  # dev, tests
   - pip:  # dev, tests, docs
+      - bayesian-optimization>=2.0.4  # dev, tests
       - nevergrad  # dev, tests
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests

pyproject.toml

@@ -16,7 +16,6 @@ dependencies = [
     "sqlalchemy>=1.3",
     "annotated-types",
     "typing-extensions",
-    "iminuit",
 ]
 dynamic = ["version"]
 keywords = [

src/optimagic/algorithms.py

@@ -12,6 +12,7 @@
 from typing import Type, cast
 
 from optimagic.optimization.algorithm import Algorithm
+from optimagic.optimizers.bayesian_optimizer import BayesOpt
 from optimagic.optimizers.bhhh import BHHH
 from optimagic.optimizers.fides import Fides
 from optimagic.optimizers.iminuit_migrad import IminuitMigrad
@@ -366,6 +367,7 @@ def Scalar(self) -> BoundedGlobalGradientFreeNonlinearConstrainedScalarAlgorithm
 
 @dataclass(frozen=True)
 class BoundedGlobalGradientFreeScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -1032,6 +1034,7 @@ def Local(self) -> GradientBasedLocalNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedGlobalGradientFreeAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -1096,6 +1099,7 @@ def Scalar(self) -> GlobalGradientFreeNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class GlobalGradientFreeScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -1305,6 +1309,7 @@ def Scalar(self) -> BoundedGradientFreeNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedGradientFreeScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA
@@ -1529,6 +1534,7 @@ def Scalar(self) -> BoundedGlobalNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedGlobalScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -2141,6 +2147,7 @@ def Local(self) -> GradientBasedLikelihoodLocalAlgorithms:
 
 @dataclass(frozen=True)
 class GlobalGradientFreeAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -2227,6 +2234,7 @@ def Scalar(self) -> GradientFreeLocalScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedGradientFreeAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nag_dfols: Type[NagDFOLS] = NagDFOLS
     nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
@@ -2324,6 +2332,7 @@ def Scalar(self) -> GradientFreeNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class GradientFreeScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA
     neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
@@ -2447,6 +2456,7 @@ def Scalar(self) -> GradientFreeParallelScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedGlobalAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -2529,6 +2539,7 @@ def Scalar(self) -> GlobalNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class GlobalScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -2843,6 +2854,7 @@ def Scalar(self) -> BoundedNonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     fides: Type[Fides] = Fides
     iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad
     ipopt: Type[Ipopt] = Ipopt
@@ -3155,6 +3167,7 @@ def Scalar(self) -> GradientBasedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class GradientFreeAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nag_dfols: Type[NagDFOLS] = NagDFOLS
     nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA
     neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel
@@ -3229,6 +3242,7 @@ def Scalar(self) -> GradientFreeScalarAlgorithms:
 
 @dataclass(frozen=True)
 class GlobalAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     nevergrad_pso: Type[NevergradPSO] = NevergradPSO
     nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM
     nlopt_direct: Type[NloptDirect] = NloptDirect
@@ -3358,6 +3372,7 @@ def Scalar(self) -> LocalScalarAlgorithms:
 
 @dataclass(frozen=True)
 class BoundedAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     fides: Type[Fides] = Fides
     iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad
     ipopt: Type[Ipopt] = Ipopt
@@ -3495,6 +3510,7 @@ def Scalar(self) -> NonlinearConstrainedScalarAlgorithms:
 
 @dataclass(frozen=True)
 class ScalarAlgorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     fides: Type[Fides] = Fides
     iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad
     ipopt: Type[Ipopt] = Ipopt
@@ -3671,6 +3687,7 @@ def Scalar(self) -> ParallelScalarAlgorithms:
 
 @dataclass(frozen=True)
 class Algorithms(AlgoSelection):
+    bayes_opt: Type[BayesOpt] = BayesOpt
     bhhh: Type[BHHH] = BHHH
     fides: Type[Fides] = Fides
     iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad

src/optimagic/config.py

@@ -108,6 +108,14 @@
     IS_NEVERGRAD_INSTALLED = True
 
 
+try:
+    from bayes_opt import BayesianOptimization  # noqa: F401
+except ImportError:
+    IS_BAYESOPT_INSTALLED = False
+else:
+    IS_BAYESOPT_INSTALLED = True
+
+
 # ======================================================================================
 # Check if pandas version is newer or equal to version 2.1.0
 # ======================================================================================