add almost every nevergrad optimizer
This commit is contained in:
parent
5118c93097
commit
3353b27884
3 changed files with 385 additions and 0 deletions
|
|
@ -1,5 +1,8 @@
|
|||
hey
|
||||
|
||||
**note:** i have a personal bias against bayesian optimizers.
|
||||
these results may not represent them fairly (or at all).
|
||||
|
||||
```
|
||||
python3 go_benchmark_it.py baseline -2 80
|
||||
python3 go_benchmark_it.py baseline -3 130
|
||||
|
|
|
|||
381
nevergradcube2.py
Normal file
381
nevergradcube2.py
Normal file
|
|
@ -0,0 +1,381 @@
|
|||
from nevergrad.common.errors import InefficientSettingsWarning
|
||||
import logging
|
||||
import nevergrad as ng
|
||||
import nevergrad.optimization.experimentalvariants as ev
|
||||
import nevergrad.optimization.optimizerlib as ol
|
||||
import warnings
|
||||
|
||||
logging.getLogger("nevergrad").setLevel(logging.WARNING)
|
||||
warnings.filterwarnings("ignore", category=InefficientSettingsWarning)
|
||||
|
||||
optimizers = [
|
||||
ol.ASCMADEthird,
|
||||
ol.AdaptiveDiscreteOnePlusOne,
|
||||
ol.AlmostRotationInvariantDE,
|
||||
ol.AnisotropicAdaptiveDiscreteOnePlusOne,
|
||||
ol.AvgMetaRecenteringNoHull,
|
||||
# ol.BO, # slow
|
||||
# ol.BOSplit, # slow
|
||||
# ol.BayesOptimBO, # ModuleNotFoundError: No module named 'bayes_optim'
|
||||
ol.CMA,
|
||||
ol.CMAbounded,
|
||||
ol.CMApara,
|
||||
ol.CMAsmall,
|
||||
ol.CMAstd,
|
||||
ol.CMAtuning,
|
||||
ol.CauchyLHSSearch,
|
||||
ol.CauchyOnePlusOne,
|
||||
ol.CauchyScrHammersleySearch,
|
||||
ol.ChainCMAPowell,
|
||||
ol.ChainDiagonalCMAPowell,
|
||||
ol.ChainMetaModelPowell,
|
||||
ol.ChainMetaModelSQP,
|
||||
ol.ChainNaiveTBPSACMAPowell,
|
||||
ol.ChainNaiveTBPSAPowell,
|
||||
ol.CmaFmin2,
|
||||
ol.Cobyla,
|
||||
ol.DE,
|
||||
ol.DiagonalCMA,
|
||||
ol.DiscreteBSOOnePlusOne,
|
||||
ol.DiscreteDE,
|
||||
ol.DiscreteDoerrOnePlusOne,
|
||||
ol.DiscreteLenglerOnePlusOne,
|
||||
ol.DiscreteLenglerOnePlusOneT,
|
||||
ol.DiscreteOnePlusOne,
|
||||
ol.DiscreteOnePlusOneT,
|
||||
ol.DoubleFastGADiscreteOnePlusOne,
|
||||
ol.ES,
|
||||
ol.FCMA,
|
||||
ol.GeneticDE,
|
||||
ol.HaltonSearch,
|
||||
ol.HaltonSearchPlusMiddlePoint,
|
||||
ol.HammersleySearch,
|
||||
ol.HammersleySearchPlusMiddlePoint,
|
||||
ol.HullAvgMetaRecentering,
|
||||
ol.HullAvgMetaTuneRecentering,
|
||||
# ol.HyperOpt, # fairly slow with larger problems (NOTE: newly removed)
|
||||
ol.LHSSearch,
|
||||
ol.LargeHaltonSearch,
|
||||
ol.LhsDE,
|
||||
ol.MetaModel,
|
||||
ol.MetaModelOnePlusOne,
|
||||
ol.MetaRecentering,
|
||||
ol.MetaTuneRecentering,
|
||||
ol.MixES,
|
||||
ol.MultiCMA,
|
||||
ol.MultiScaleCMA,
|
||||
ol.MutDE,
|
||||
# ol.NEWUOA, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.
|
||||
# ol.NLOPT, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.
|
||||
ol.NaiveIsoEMNA,
|
||||
ol.NaiveTBPSA,
|
||||
ol.NelderMead,
|
||||
ol.NoisyDE,
|
||||
ol.NoisyDiscreteOnePlusOne,
|
||||
ol.NoisyOnePlusOne,
|
||||
ol.NonNSGAIIES,
|
||||
ol.ORandomSearch,
|
||||
ol.OScrHammersleySearch,
|
||||
ol.OldCMA,
|
||||
ol.OnePlusOne,
|
||||
ol.OptimisticDiscreteOnePlusOne,
|
||||
ol.OptimisticNoisyOnePlusOne,
|
||||
# ol.PCABO, # ModuleNotFoundError: No module named 'bayes_optim'
|
||||
ol.PSO,
|
||||
ol.ParaPortfolio,
|
||||
ol.PolyCMA,
|
||||
ol.PortfolioDiscreteOnePlusOne,
|
||||
ol.PortfolioDiscreteOnePlusOneT,
|
||||
ol.Powell,
|
||||
# ol.PymooBatchNSGA2, # NotImplementedError: This optimizer isn't supported by the way minimize works by default.
|
||||
# ol.PymooNSGA2, # RuntimeError: Recast optimizer raised an error: The method `get_algorithm` has been deprecated since 0.6.0 Please use the object-oriented interface.
|
||||
ol.QORandomSearch,
|
||||
ol.QOScrHammersleySearch,
|
||||
ol.QrDE,
|
||||
ol.RCobyla,
|
||||
ol.RLSOnePlusOne,
|
||||
ol.RPowell,
|
||||
# ol.RSLSQP,
|
||||
ol.RSQP,
|
||||
ol.RandomSearch,
|
||||
ol.RandomSearchPlusMiddlePoint,
|
||||
ol.RealSpacePSO,
|
||||
ol.RecES,
|
||||
ol.RecMixES,
|
||||
ol.RecMutDE,
|
||||
ol.RecombiningPortfolioDiscreteOnePlusOne,
|
||||
ol.RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne,
|
||||
ol.RescaledCMA,
|
||||
ol.RotatedTwoPointsDE,
|
||||
ol.RotationInvariantDE,
|
||||
ol.SADiscreteLenglerOnePlusOneExp09,
|
||||
ol.SADiscreteLenglerOnePlusOneExp099,
|
||||
ol.SADiscreteLenglerOnePlusOneExp09Auto,
|
||||
ol.SADiscreteLenglerOnePlusOneLin1,
|
||||
ol.SADiscreteLenglerOnePlusOneLin100,
|
||||
ol.SADiscreteLenglerOnePlusOneLinAuto,
|
||||
ol.SADiscreteOnePlusOneExp09,
|
||||
ol.SADiscreteOnePlusOneExp099,
|
||||
ol.SADiscreteOnePlusOneLin100,
|
||||
# ol.SLSQP,
|
||||
ol.SQP,
|
||||
ol.ScrHaltonSearch,
|
||||
ol.ScrHaltonSearchPlusMiddlePoint,
|
||||
ol.ScrHammersleySearch,
|
||||
ol.ScrHammersleySearchPlusMiddlePoint,
|
||||
ol.SparseDoubleFastGADiscreteOnePlusOne,
|
||||
ol.TBPSA,
|
||||
ol.TripleCMA,
|
||||
ol.TwoPointsDE,
|
||||
ol.discretememetic,
|
||||
]
|
||||
|
||||
more_optimizers = [
|
||||
ev.AlmostRotationInvariantDEAndBigPop,
|
||||
ev.AnisoEMNA,
|
||||
ev.AnisoEMNATBPSA,
|
||||
ev.AvgHammersleySearch,
|
||||
ev.AvgHammersleySearchPlusMiddlePoint,
|
||||
ev.AvgRandomSearch,
|
||||
# ev.BO, # slow
|
||||
ev.BPRotationInvariantDE,
|
||||
ev.CMA,
|
||||
ev.CauchyRandomSearch,
|
||||
# ev.ChainBOwithLHS, # slow
|
||||
# ev.ChainBOwithLHS30, # slow
|
||||
# ev.ChainBOwithLHSdim, # slow
|
||||
# ev.ChainBOwithLHSsqrt, # slow
|
||||
# ev.ChainBOwithMetaRecentering, # slow
|
||||
# ev.ChainBOwithMetaRecentering30, # slow
|
||||
# ev.ChainBOwithMetaRecenteringdim, # slow
|
||||
# ev.ChainBOwithMetaRecenteringsqrt, # slow
|
||||
# ev.ChainBOwithMetaTuneRecentering, # slow
|
||||
# ev.ChainBOwithMetaTuneRecentering30, # slow
|
||||
# ev.ChainBOwithMetaTuneRecenteringdim, # slow
|
||||
# ev.ChainBOwithMetaTuneRecenteringsqrt, # slow
|
||||
# ev.ChainBOwithR, # slow
|
||||
# ev.ChainBOwithR30, # slow
|
||||
# ev.ChainBOwithRdim, # slow
|
||||
# ev.ChainBOwithRsqrt, # slow
|
||||
ev.ChainCMASQP,
|
||||
ev.ChainCMAwithLHS,
|
||||
ev.ChainCMAwithLHS30,
|
||||
ev.ChainCMAwithLHSdim,
|
||||
ev.ChainCMAwithLHSsqrt,
|
||||
ev.ChainCMAwithMetaRecentering,
|
||||
ev.ChainCMAwithMetaRecentering30,
|
||||
ev.ChainCMAwithMetaRecenteringdim,
|
||||
ev.ChainCMAwithMetaRecenteringsqrt,
|
||||
ev.ChainCMAwithR,
|
||||
ev.ChainCMAwithR30,
|
||||
ev.ChainCMAwithRdim,
|
||||
ev.ChainCMAwithRsqrt,
|
||||
ev.ChainDEwithLHS,
|
||||
ev.ChainDEwithLHS30,
|
||||
ev.ChainDEwithLHSdim,
|
||||
ev.ChainDEwithLHSsqrt,
|
||||
ev.ChainDEwithMetaRecentering,
|
||||
ev.ChainDEwithMetaRecentering30,
|
||||
ev.ChainDEwithMetaRecenteringdim,
|
||||
ev.ChainDEwithMetaRecenteringsqrt,
|
||||
ev.ChainDEwithMetaTuneRecentering,
|
||||
ev.ChainDEwithMetaTuneRecentering30,
|
||||
ev.ChainDEwithMetaTuneRecenteringdim,
|
||||
ev.ChainDEwithMetaTuneRecenteringsqrt,
|
||||
ev.ChainDEwithR,
|
||||
ev.ChainDEwithR30,
|
||||
ev.ChainDEwithRdim,
|
||||
ev.ChainDEwithRsqrt,
|
||||
ev.ChainPSOwithLHS,
|
||||
ev.ChainPSOwithLHS30,
|
||||
ev.ChainPSOwithLHSdim,
|
||||
ev.ChainPSOwithLHSsqrt,
|
||||
ev.ChainPSOwithMetaRecentering,
|
||||
ev.ChainPSOwithMetaRecentering30,
|
||||
ev.ChainPSOwithMetaRecenteringdim,
|
||||
ev.ChainPSOwithMetaRecenteringsqrt,
|
||||
ev.ChainPSOwithR,
|
||||
ev.ChainPSOwithR30,
|
||||
ev.ChainPSOwithRdim,
|
||||
ev.ChainPSOwithRsqrt,
|
||||
ev.CmaFmin2,
|
||||
ev.DE,
|
||||
ev.DiagonalCMA,
|
||||
ev.DiscreteNoisy13Splits,
|
||||
ev.DiscreteNoisyInfSplits,
|
||||
ev.DoubleFastGAOptimisticNoisyDiscreteOnePlusOne,
|
||||
ev.ECMA,
|
||||
ev.FCMAp13,
|
||||
ev.FCMAs03,
|
||||
ev.FastGADiscreteOnePlusOne,
|
||||
ev.FastGANoisyDiscreteOnePlusOne,
|
||||
ev.FastGAOptimisticNoisyDiscreteOnePlusOne,
|
||||
ev.GeneticDE,
|
||||
ev.HSCMA,
|
||||
ev.HSDE,
|
||||
ev.HSMetaModel,
|
||||
ev.HullCenterHullAvgCauchyLHSSearch,
|
||||
ev.HullCenterHullAvgCauchyScrHammersleySearch,
|
||||
ev.HullCenterHullAvgLHSSearch,
|
||||
ev.HullCenterHullAvgLargeHammersleySearch,
|
||||
ev.HullCenterHullAvgRandomSearch,
|
||||
ev.HullCenterHullAvgScrHaltonSearch,
|
||||
ev.HullCenterHullAvgScrHaltonSearchPlusMiddlePoint,
|
||||
ev.HullCenterHullAvgScrHammersleySearch,
|
||||
ev.HullCenterHullAvgScrHammersleySearchPlusMiddlePoint,
|
||||
ev.IsoEMNA,
|
||||
ev.IsoEMNATBPSA,
|
||||
# ev.LBO, # slow
|
||||
ev.LHSSearch,
|
||||
ev.LhsHSDE,
|
||||
ev.MetaCauchyRecentering,
|
||||
ev.MetaModelDiagonalCMA,
|
||||
ev.MetaModelFmin2,
|
||||
# ev.MetaNGOpt10, # hangs: Optimizing the schwefel function with the ngx_metangopt10 optimizer
|
||||
ev.MetaRecentering,
|
||||
ev.MetaTuneRecentering,
|
||||
ev.MicroCMA,
|
||||
# ev.MidQRBO, # slow
|
||||
ev.MilliCMA,
|
||||
ev.MiniDE,
|
||||
ev.MiniLhsDE,
|
||||
ev.MiniQrDE,
|
||||
ev.MixDeterministicRL, # InefficientSettingsWarning: DE algorithms are inefficient with budget < 60
|
||||
# ev.NGOptSingle16, # slow
|
||||
# ev.NGOptSingle25, # slow
|
||||
# ev.NGOptSingle9, # slow
|
||||
ev.NaiveAnisoEMNA,
|
||||
ev.NaiveAnisoEMNATBPSA,
|
||||
ev.NaiveIsoEMNATBPSA,
|
||||
ev.Noisy13Splits,
|
||||
ev.NoisyInfSplits,
|
||||
ev.NoisyOnePlusOne,
|
||||
ev.NoisyRL1, # note: inefficiency warnings
|
||||
ev.NoisyRL2, # note: inefficiency warnings
|
||||
ev.NoisyRL3, # note: inefficiency warnings
|
||||
ev.OnePointDE,
|
||||
ev.OptimisticNoisyOnePlusOne,
|
||||
# ev.PCABO80, # ModuleNotFoundError: No module named 'bayes_optim'
|
||||
# ev.PCABO95DoE20, # ModuleNotFoundError: No module named 'bayes_optim'
|
||||
ev.PSO,
|
||||
ev.ParametrizationDE,
|
||||
ev.PortfolioNoisyDiscreteOnePlusOne,
|
||||
ev.PortfolioOptimisticNoisyDiscreteOnePlusOne,
|
||||
# ev.QRBO, # slow
|
||||
# ev.RBO, # slow
|
||||
ev.RandomScaleRandomSearch,
|
||||
ev.RandomScaleRandomSearchPlusMiddlePoint,
|
||||
ev.RandomSearch,
|
||||
ev.RecombiningGA,
|
||||
ev.RecombiningOptimisticNoisyDiscreteOnePlusOne,
|
||||
ev.RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne,
|
||||
ev.RescaleScrHammersleySearch,
|
||||
ev.RotatedRecombiningGA,
|
||||
ev.SQP,
|
||||
ev.SmoothAdaptiveDiscreteOnePlusOne,
|
||||
ev.SmoothDiscreteLenglerOnePlusOne,
|
||||
ev.SmoothDiscreteOnePlusOne,
|
||||
ev.SmoothPortfolioDiscreteOnePlusOne,
|
||||
ev.SparseDiscreteOnePlusOne,
|
||||
ev.SpecialRL, # note: inefficiency warnings
|
||||
ev.StupidRandom,
|
||||
ev.TBPSA,
|
||||
ev.TEAvgCauchyLHSSearch,
|
||||
ev.TEAvgCauchyScrHammersleySearch,
|
||||
ev.TEAvgLHSSearch,
|
||||
ev.TEAvgRandomSearch,
|
||||
ev.TEAvgScrHammersleySearch,
|
||||
ev.TEAvgScrHammersleySearchPlusMiddlePoint,
|
||||
ev.Zero,
|
||||
]
|
||||
|
||||
bayes_optimizers = [
|
||||
ol.BO,
|
||||
ev.RBO,
|
||||
ev.QRBO,
|
||||
ev.MidQRBO,
|
||||
ev.LBO,
|
||||
ol.ParametrizedBO(utility_kind="ei").set_name("BOEI"),
|
||||
ol.ParametrizedBO(utility_kind="poi").set_name("BOPOI"),
|
||||
]
|
||||
|
||||
assert ol.RSLSQP is ol.RSQP, "weirdness is gone, please adjust accordingly"
|
||||
assert ol.SLSQP is ol.SQP, "weirdness is gone, please adjust accordingly"
|
||||
|
||||
|
||||
def nevergrad_cube_factory(optimizer, objective, n_trials, n_dim, with_count):
|
||||
instrument = ng.p.Array(lower=0, upper=1, shape=(n_dim,)) # better sigma
|
||||
# ev.RBO, ev.QRBO, ev.MidQRBO, and ev.LBO still complain, though:
|
||||
# /home/py/.local/lib/python3.10/site-packages/nevergrad/parametrization/_datalayers.py:107: NevergradRuntimeWarning: Bounds are 1.0 sigma away from each other at the closest, you should aim for at least 3 for better quality.
|
||||
if optimizer in (ev.RBO, ev.QRBO, ev.MidQRBO, ev.LBO):
|
||||
instrument.set_mutation(sigma=0.3)
|
||||
|
||||
opt = optimizer
|
||||
assert opt is not None, optimizer
|
||||
|
||||
optimizer = opt(parametrization=instrument, budget=n_trials, num_workers=1)
|
||||
|
||||
feval_count = 0
|
||||
|
||||
def cube_objective(us):
|
||||
nonlocal feval_count
|
||||
feval_count += 1
|
||||
return objective(us)
|
||||
|
||||
recommendation = optimizer.minimize(cube_objective)
|
||||
best_x = recommendation.value
|
||||
best_val = cube_objective(best_x) # don't trust recommendation.loss
|
||||
|
||||
return (best_val, best_x, feval_count) if with_count else (best_val, best_x)
|
||||
|
||||
|
||||
def named_optimizer(optimizer, experimental=False):
|
||||
def f(*args, **kwargs):
|
||||
return nevergrad_cube_factory(optimizer, *args, **kwargs)
|
||||
|
||||
name = optimizer.name if hasattr(optimizer, "name") else optimizer.__name__
|
||||
|
||||
# TODO: make these names less awful.
|
||||
new_name = ("ngx_" if experimental else "ng_") + name + "_cube"
|
||||
new_name = new_name.replace("OnePlusOne", "_1p1")
|
||||
new_name = new_name.replace("PlusMiddlePoint", "_pmp")
|
||||
new_name = new_name.replace("Search", "")
|
||||
while "__" in new_name:
|
||||
new_name = new_name.replace("__", "_")
|
||||
f.__name__ = new_name.lower()
|
||||
return f
|
||||
|
||||
|
||||
# some selections:
|
||||
ng_1p1_cube = named_optimizer(ol.OnePlusOne)
|
||||
ng_bo_cube = named_optimizer(ol.BO)
|
||||
ng_cauchy_1p1_cube = named_optimizer(ol.CauchyOnePlusOne)
|
||||
ng_chaincmapowell_cube = named_optimizer(ol.ChainCMAPowell)
|
||||
ng_chaindiagonalcmapowell_cube = named_optimizer(ol.ChainDiagonalCMAPowell)
|
||||
ng_chainmetamodelpowell_cube = named_optimizer(ol.ChainMetaModelPowell)
|
||||
ng_chainmetamodelsqp_cube = named_optimizer(ol.ChainMetaModelSQP)
|
||||
ng_chainnaivetbpsapowell_cube = named_optimizer(ol.ChainNaiveTBPSAPowell)
|
||||
ng_cma_cube = named_optimizer(ol.CMA)
|
||||
ng_cobyla_cube = named_optimizer(ol.Cobyla)
|
||||
ng_diagonalcma_cube = named_optimizer(ol.DiagonalCMA)
|
||||
ng_ecma_cube = named_optimizer(ev.ECMA)
|
||||
ng_fcma_cube = named_optimizer(ol.FCMA)
|
||||
ng_fcmap13_cube = named_optimizer(ev.FCMAp13)
|
||||
ng_fcmas03_cube = named_optimizer(ev.FCMAs03)
|
||||
ng_metamodel_1p1_cube = named_optimizer(ol.MetaModelOnePlusOne)
|
||||
ng_metamodel_cube = named_optimizer(ol.MetaModel)
|
||||
ng_neldermead_cube = named_optimizer(ol.NelderMead)
|
||||
ng_parametrizationde_cube = named_optimizer(ev.ParametrizationDE)
|
||||
ng_powell_cube = named_optimizer(ol.Powell)
|
||||
ng_rescaledcma_cube = named_optimizer(ol.RescaledCMA)
|
||||
ng_rpowell_cube = named_optimizer(ol.RPowell)
|
||||
ng_rsqp_cube = named_optimizer(ol.RSQP)
|
||||
ng_sqp_cube = named_optimizer(ol.SQP)
|
||||
ngx_chaincmasqp_cube = named_optimizer(ev.ChainCMASQP)
|
||||
ngx_chaincmawithmetarecenteringdim_cube = named_optimizer(ev.ChainCMAwithMetaRecenteringdim)
|
||||
|
||||
NEVERGRAD2_OPTIMIZERS = list(map(named_optimizer, optimizers)) + list(
|
||||
map(lambda o: named_optimizer(o, experimental=True), more_optimizers)
|
||||
)
|
||||
|
||||
BAYES_OPTIMIZERS = list(map(lambda o: named_optimizer(o, experimental=o != ol.BO), bayes_optimizers))
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
from dlibcube2 import dlib_cube
|
||||
from nevergradcube2 import NEVERGRAD2_OPTIMIZERS
|
||||
from nloptcube2 import nlopt_neldermead_cube
|
||||
from notwacube2 import make_birect, make_mercury, make_soo
|
||||
from randomcube2 import another_random_cube, quasirandom_cube
|
||||
|
|
|
|||
Loading…
Reference in a new issue