Add CMAES optimizer from nevergrad

.pre-commit-config.yaml

@@ -29,6 +29,7 @@ repos:
         additional_dependencies:
           - hatchling
           - ruff
+          - iminuit
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:

.tools/envs/testenv-linux.yml

@@ -33,6 +33,7 @@ dependencies:
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
+      - nevergrad  # dev, tests
       - kaleido  # dev, tests
       - pandas-stubs  # dev, tests
       - types-cffi  # dev, tests

.tools/envs/testenv-numpy.yml

@@ -31,6 +31,7 @@ dependencies:
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
+      - nevergrad  # dev, tests
       - kaleido  # dev, tests
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests

.tools/envs/testenv-others.yml

@@ -31,6 +31,7 @@ dependencies:
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
+      - nevergrad  # dev, tests
       - kaleido  # dev, tests
       - pandas-stubs  # dev, tests
       - types-cffi  # dev, tests

.tools/envs/testenv-pandas.yml

@@ -31,6 +31,7 @@ dependencies:
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
+      - nevergrad  # dev, tests
       - kaleido  # dev, tests
       - types-cffi  # dev, tests
       - types-openpyxl  # dev, tests

docs/source/algorithms.md

@@ -3984,8 +3984,6 @@ iminuit).
       - Values greater than 1 specify the maximum number of restart attempts.  
 ```
 
-(nevergrad-algorithms)=
-
 ## Nevergrad Optimizers
 
 optimagic supports some algorithms from the
@@ -4041,6 +4039,68 @@ these optimizers, you need to have
       for speed. Default is False.
     - **special_speed_quasi_opp_init** (bool): Whether to use special quasi-opposition
       initialization for speed. Default is False.
+
+```
+
+```{eval-rst}
+.. dropdown::  nevegrad_cmaes
+
+    .. code-block::
+
+        "nevergrad_cmaes"
+
+    The Covariance Matrix Adaptation Evolution Strategy (CMA-ES) is a stochastic derivative-free numerical optimization algorithm for difficult (non-convex, ill-conditioned, multi-modal, rugged, noisy) optimization problems in continuous search spaces.
+
+    The version available through nevergrad wraps an external implementation `pycma <https://github.com/CMA-ES/pycma>`_.
+
+    The original method can be found in
+    :cite:`Hansen2023`.
+
+    The fast implementation relies on fcmaes which can be installed with `pip install fcmaes`.
+
+    **Optimizer Parameters:**
+
+    - **scale** (float): Scale of the search.
+
+    - **elitist** (bool): Whether to switch to elitist mode, i.e., `+` mode instead of `,` mode, where the best point in the population is always retained.
+    - **population_size** (Optional[int]): Population size. Should be
+
+    .. math::
+
+        \text{population_size} = n \times \text{num_workers} \quad \text{for integer } n \geq 1
+
+    Default is
+
+    .. math::
+
+        \max\left( \text{num_workers},\ 4 + \left\lfloor 3 \cdot \log(\text{dimension}) \right\rfloor \right)
+
+    - **popsize_factor** (float): Factor used in the formula for computing the population size.  
+      - Default is `3.0`.
+
+    - **diagonal** (bool): Use the diagonal version of CMA, which is advised for high-dimensional problems.
+
+    - **high_speed** (bool): Use a metamodel for recommendation to speed up optimization.
+
+    - **use_fast_implementation** (bool): Use the fast CMA-ES implementation.  
+      - Cannot be used with `diagonal=True`.  
+      - Produces equivalent results and is preferable for high dimensions or when objective function evaluations are fast.
+
+    - **stopping.maxfun** (int): The maximum number of criterion
+      evaluations.
+    - **stopping.maxiter** (int): The maximum number of iterations.
+    - **convergence_ftol_abs** (float): stopping criteria on the f tolerance.
+    - **convergence_ftol_rel** (float): relative stopping criteria on the f tolerance.
+    - **convergence_xtol_abs** (float): stopping criteria on the x tolerance.
+
+    - **seed** (int): seed used by the internal random number generator.
+
+    - **learning_rate_rank_one_update** (float): Multiplier for the rank-one update learning rate of the covariance matrix.  
+      - Default is `1.0`.
+
+    - **learning_rate_rank_mu_update** (float): Multiplier for the rank-mu update learning rate of the covariance matrix.  
+      - Default is `1.0`.
+
 ```
 
 ## References

docs/source/refs.bib

@@ -906,6 +906,17 @@ @article{JAMES1975343
 author = {F. James and M. Roos}
 }
 
+
+@misc{Hansen2023,
+title={The CMA Evolution Strategy: A Tutorial}, 
+author={Nikolaus Hansen},
+year={2023},
+eprint={1604.00772},
+archivePrefix={arXiv},
+primaryClass={cs.LG},
+url={https://arxiv.org/abs/1604.00772}, 
+}
+
 @InProceedings{Kennedy1995,
   author={Kennedy, J. and Eberhart, R.},
   booktitle={Proceedings of ICNN'95 - International Conference on Neural Networks}, 

environment.yml

@@ -43,6 +43,7 @@ dependencies:
       - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
+      - nevergrad  # dev, tests
       - kaleido  # dev, tests
       - pre-commit>=4  # dev
       - -e .  # dev

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
     "annotated-types",
     "typing-extensions",
     "iminuit",
+    "nevergrad",
 ]
 dynamic = ["version"]
 keywords = [
@@ -350,6 +351,7 @@ module = [
     "cyipopt",
     "nlopt",
     "bokeh",
+    "nevergrad",
     "bokeh.layouts",
     "bokeh.models",
     "bokeh.plotting",