diff --git a/README.md b/README.md index 4d9d4f5..84e23ee 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ hey +**note:** i have a personal bias against bayesian optimizers. +these results may not represent them fairly (or at all). + ``` python3 go_benchmark_it.py baseline -2 80 python3 go_benchmark_it.py baseline -3 130 diff --git a/nevergradcube2.py b/nevergradcube2.py new file mode 100644 index 0000000..3bd0cbf --- /dev/null +++ b/nevergradcube2.py @@ -0,0 +1,381 @@ +from nevergrad.common.errors import InefficientSettingsWarning +import logging +import nevergrad as ng +import nevergrad.optimization.experimentalvariants as ev +import nevergrad.optimization.optimizerlib as ol +import warnings + +logging.getLogger("nevergrad").setLevel(logging.WARNING) +warnings.filterwarnings("ignore", category=InefficientSettingsWarning) + +optimizers = [ + ol.ASCMADEthird, + ol.AdaptiveDiscreteOnePlusOne, + ol.AlmostRotationInvariantDE, + ol.AnisotropicAdaptiveDiscreteOnePlusOne, + ol.AvgMetaRecenteringNoHull, + # ol.BO, # slow + # ol.BOSplit, # slow + # ol.BayesOptimBO, # ModuleNotFoundError: No module named 'bayes_optim' + ol.CMA, + ol.CMAbounded, + ol.CMApara, + ol.CMAsmall, + ol.CMAstd, + ol.CMAtuning, + ol.CauchyLHSSearch, + ol.CauchyOnePlusOne, + ol.CauchyScrHammersleySearch, + ol.ChainCMAPowell, + ol.ChainDiagonalCMAPowell, + ol.ChainMetaModelPowell, + ol.ChainMetaModelSQP, + ol.ChainNaiveTBPSACMAPowell, + ol.ChainNaiveTBPSAPowell, + ol.CmaFmin2, + ol.Cobyla, + ol.DE, + ol.DiagonalCMA, + ol.DiscreteBSOOnePlusOne, + ol.DiscreteDE, + ol.DiscreteDoerrOnePlusOne, + ol.DiscreteLenglerOnePlusOne, + ol.DiscreteLenglerOnePlusOneT, + ol.DiscreteOnePlusOne, + ol.DiscreteOnePlusOneT, + ol.DoubleFastGADiscreteOnePlusOne, + ol.ES, + ol.FCMA, + ol.GeneticDE, + ol.HaltonSearch, + ol.HaltonSearchPlusMiddlePoint, + ol.HammersleySearch, + ol.HammersleySearchPlusMiddlePoint, + ol.HullAvgMetaRecentering, + ol.HullAvgMetaTuneRecentering, + # ol.HyperOpt, # fairly slow with larger problems (NOTE: newly removed) + ol.LHSSearch, + ol.LargeHaltonSearch, + ol.LhsDE, + ol.MetaModel, + ol.MetaModelOnePlusOne, + ol.MetaRecentering, + ol.MetaTuneRecentering, + ol.MixES, + ol.MultiCMA, + ol.MultiScaleCMA, + ol.MutDE, + # ol.NEWUOA, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part. + # ol.NLOPT, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part. + ol.NaiveIsoEMNA, + ol.NaiveTBPSA, + ol.NelderMead, + ol.NoisyDE, + ol.NoisyDiscreteOnePlusOne, + ol.NoisyOnePlusOne, + ol.NonNSGAIIES, + ol.ORandomSearch, + ol.OScrHammersleySearch, + ol.OldCMA, + ol.OnePlusOne, + ol.OptimisticDiscreteOnePlusOne, + ol.OptimisticNoisyOnePlusOne, + # ol.PCABO, # ModuleNotFoundError: No module named 'bayes_optim' + ol.PSO, + ol.ParaPortfolio, + ol.PolyCMA, + ol.PortfolioDiscreteOnePlusOne, + ol.PortfolioDiscreteOnePlusOneT, + ol.Powell, + # ol.PymooBatchNSGA2, # NotImplementedError: This optimizer isn't supported by the way minimize works by default. + # ol.PymooNSGA2, # RuntimeError: Recast optimizer raised an error: The method `get_algorithm` has been deprecated since 0.6.0 Please use the object-oriented interface. + ol.QORandomSearch, + ol.QOScrHammersleySearch, + ol.QrDE, + ol.RCobyla, + ol.RLSOnePlusOne, + ol.RPowell, + # ol.RSLSQP, + ol.RSQP, + ol.RandomSearch, + ol.RandomSearchPlusMiddlePoint, + ol.RealSpacePSO, + ol.RecES, + ol.RecMixES, + ol.RecMutDE, + ol.RecombiningPortfolioDiscreteOnePlusOne, + ol.RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne, + ol.RescaledCMA, + ol.RotatedTwoPointsDE, + ol.RotationInvariantDE, + ol.SADiscreteLenglerOnePlusOneExp09, + ol.SADiscreteLenglerOnePlusOneExp099, + ol.SADiscreteLenglerOnePlusOneExp09Auto, + ol.SADiscreteLenglerOnePlusOneLin1, + ol.SADiscreteLenglerOnePlusOneLin100, + ol.SADiscreteLenglerOnePlusOneLinAuto, + ol.SADiscreteOnePlusOneExp09, + ol.SADiscreteOnePlusOneExp099, + ol.SADiscreteOnePlusOneLin100, + # ol.SLSQP, + ol.SQP, + ol.ScrHaltonSearch, + ol.ScrHaltonSearchPlusMiddlePoint, + ol.ScrHammersleySearch, + ol.ScrHammersleySearchPlusMiddlePoint, + ol.SparseDoubleFastGADiscreteOnePlusOne, + ol.TBPSA, + ol.TripleCMA, + ol.TwoPointsDE, + ol.discretememetic, +] + +more_optimizers = [ + ev.AlmostRotationInvariantDEAndBigPop, + ev.AnisoEMNA, + ev.AnisoEMNATBPSA, + ev.AvgHammersleySearch, + ev.AvgHammersleySearchPlusMiddlePoint, + ev.AvgRandomSearch, + # ev.BO, # slow + ev.BPRotationInvariantDE, + ev.CMA, + ev.CauchyRandomSearch, + # ev.ChainBOwithLHS, # slow + # ev.ChainBOwithLHS30, # slow + # ev.ChainBOwithLHSdim, # slow + # ev.ChainBOwithLHSsqrt, # slow + # ev.ChainBOwithMetaRecentering, # slow + # ev.ChainBOwithMetaRecentering30, # slow + # ev.ChainBOwithMetaRecenteringdim, # slow + # ev.ChainBOwithMetaRecenteringsqrt, # slow + # ev.ChainBOwithMetaTuneRecentering, # slow + # ev.ChainBOwithMetaTuneRecentering30, # slow + # ev.ChainBOwithMetaTuneRecenteringdim, # slow + # ev.ChainBOwithMetaTuneRecenteringsqrt, # slow + # ev.ChainBOwithR, # slow + # ev.ChainBOwithR30, # slow + # ev.ChainBOwithRdim, # slow + # ev.ChainBOwithRsqrt, # slow + ev.ChainCMASQP, + ev.ChainCMAwithLHS, + ev.ChainCMAwithLHS30, + ev.ChainCMAwithLHSdim, + ev.ChainCMAwithLHSsqrt, + ev.ChainCMAwithMetaRecentering, + ev.ChainCMAwithMetaRecentering30, + ev.ChainCMAwithMetaRecenteringdim, + ev.ChainCMAwithMetaRecenteringsqrt, + ev.ChainCMAwithR, + ev.ChainCMAwithR30, + ev.ChainCMAwithRdim, + ev.ChainCMAwithRsqrt, + ev.ChainDEwithLHS, + ev.ChainDEwithLHS30, + ev.ChainDEwithLHSdim, + ev.ChainDEwithLHSsqrt, + ev.ChainDEwithMetaRecentering, + ev.ChainDEwithMetaRecentering30, + ev.ChainDEwithMetaRecenteringdim, + ev.ChainDEwithMetaRecenteringsqrt, + ev.ChainDEwithMetaTuneRecentering, + ev.ChainDEwithMetaTuneRecentering30, + ev.ChainDEwithMetaTuneRecenteringdim, + ev.ChainDEwithMetaTuneRecenteringsqrt, + ev.ChainDEwithR, + ev.ChainDEwithR30, + ev.ChainDEwithRdim, + ev.ChainDEwithRsqrt, + ev.ChainPSOwithLHS, + ev.ChainPSOwithLHS30, + ev.ChainPSOwithLHSdim, + ev.ChainPSOwithLHSsqrt, + ev.ChainPSOwithMetaRecentering, + ev.ChainPSOwithMetaRecentering30, + ev.ChainPSOwithMetaRecenteringdim, + ev.ChainPSOwithMetaRecenteringsqrt, + ev.ChainPSOwithR, + ev.ChainPSOwithR30, + ev.ChainPSOwithRdim, + ev.ChainPSOwithRsqrt, + ev.CmaFmin2, + ev.DE, + ev.DiagonalCMA, + ev.DiscreteNoisy13Splits, + ev.DiscreteNoisyInfSplits, + ev.DoubleFastGAOptimisticNoisyDiscreteOnePlusOne, + ev.ECMA, + ev.FCMAp13, + ev.FCMAs03, + ev.FastGADiscreteOnePlusOne, + ev.FastGANoisyDiscreteOnePlusOne, + ev.FastGAOptimisticNoisyDiscreteOnePlusOne, + ev.GeneticDE, + ev.HSCMA, + ev.HSDE, + ev.HSMetaModel, + ev.HullCenterHullAvgCauchyLHSSearch, + ev.HullCenterHullAvgCauchyScrHammersleySearch, + ev.HullCenterHullAvgLHSSearch, + ev.HullCenterHullAvgLargeHammersleySearch, + ev.HullCenterHullAvgRandomSearch, + ev.HullCenterHullAvgScrHaltonSearch, + ev.HullCenterHullAvgScrHaltonSearchPlusMiddlePoint, + ev.HullCenterHullAvgScrHammersleySearch, + ev.HullCenterHullAvgScrHammersleySearchPlusMiddlePoint, + ev.IsoEMNA, + ev.IsoEMNATBPSA, + # ev.LBO, # slow + ev.LHSSearch, + ev.LhsHSDE, + ev.MetaCauchyRecentering, + ev.MetaModelDiagonalCMA, + ev.MetaModelFmin2, + # ev.MetaNGOpt10, # hangs: Optimizing the schwefel function with the ngx_metangopt10 optimizer + ev.MetaRecentering, + ev.MetaTuneRecentering, + ev.MicroCMA, + # ev.MidQRBO, # slow + ev.MilliCMA, + ev.MiniDE, + ev.MiniLhsDE, + ev.MiniQrDE, + ev.MixDeterministicRL, # InefficientSettingsWarning: DE algorithms are inefficient with budget < 60 + # ev.NGOptSingle16, # slow + # ev.NGOptSingle25, # slow + # ev.NGOptSingle9, # slow + ev.NaiveAnisoEMNA, + ev.NaiveAnisoEMNATBPSA, + ev.NaiveIsoEMNATBPSA, + ev.Noisy13Splits, + ev.NoisyInfSplits, + ev.NoisyOnePlusOne, + ev.NoisyRL1, # note: inefficiency warnings + ev.NoisyRL2, # note: inefficiency warnings + ev.NoisyRL3, # note: inefficiency warnings + ev.OnePointDE, + ev.OptimisticNoisyOnePlusOne, + # ev.PCABO80, # ModuleNotFoundError: No module named 'bayes_optim' + # ev.PCABO95DoE20, # ModuleNotFoundError: No module named 'bayes_optim' + ev.PSO, + ev.ParametrizationDE, + ev.PortfolioNoisyDiscreteOnePlusOne, + ev.PortfolioOptimisticNoisyDiscreteOnePlusOne, + # ev.QRBO, # slow + # ev.RBO, # slow + ev.RandomScaleRandomSearch, + ev.RandomScaleRandomSearchPlusMiddlePoint, + ev.RandomSearch, + ev.RecombiningGA, + ev.RecombiningOptimisticNoisyDiscreteOnePlusOne, + ev.RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne, + ev.RescaleScrHammersleySearch, + ev.RotatedRecombiningGA, + ev.SQP, + ev.SmoothAdaptiveDiscreteOnePlusOne, + ev.SmoothDiscreteLenglerOnePlusOne, + ev.SmoothDiscreteOnePlusOne, + ev.SmoothPortfolioDiscreteOnePlusOne, + ev.SparseDiscreteOnePlusOne, + ev.SpecialRL, # note: inefficiency warnings + ev.StupidRandom, + ev.TBPSA, + ev.TEAvgCauchyLHSSearch, + ev.TEAvgCauchyScrHammersleySearch, + ev.TEAvgLHSSearch, + ev.TEAvgRandomSearch, + ev.TEAvgScrHammersleySearch, + ev.TEAvgScrHammersleySearchPlusMiddlePoint, + ev.Zero, +] + +bayes_optimizers = [ + ol.BO, + ev.RBO, + ev.QRBO, + ev.MidQRBO, + ev.LBO, + ol.ParametrizedBO(utility_kind="ei").set_name("BOEI"), + ol.ParametrizedBO(utility_kind="poi").set_name("BOPOI"), +] + +assert ol.RSLSQP is ol.RSQP, "weirdness is gone, please adjust accordingly" +assert ol.SLSQP is ol.SQP, "weirdness is gone, please adjust accordingly" + + +def nevergrad_cube_factory(optimizer, objective, n_trials, n_dim, with_count): + instrument = ng.p.Array(lower=0, upper=1, shape=(n_dim,)) # better sigma + # ev.RBO, ev.QRBO, ev.MidQRBO, and ev.LBO still complain, though: + # /home/py/.local/lib/python3.10/site-packages/nevergrad/parametrization/_datalayers.py:107: NevergradRuntimeWarning: Bounds are 1.0 sigma away from each other at the closest, you should aim for at least 3 for better quality. + if optimizer in (ev.RBO, ev.QRBO, ev.MidQRBO, ev.LBO): + instrument.set_mutation(sigma=0.3) + + opt = optimizer + assert opt is not None, optimizer + + optimizer = opt(parametrization=instrument, budget=n_trials, num_workers=1) + + feval_count = 0 + + def cube_objective(us): + nonlocal feval_count + feval_count += 1 + return objective(us) + + recommendation = optimizer.minimize(cube_objective) + best_x = recommendation.value + best_val = cube_objective(best_x) # don't trust recommendation.loss + + return (best_val, best_x, feval_count) if with_count else (best_val, best_x) + + +def named_optimizer(optimizer, experimental=False): + def f(*args, **kwargs): + return nevergrad_cube_factory(optimizer, *args, **kwargs) + + name = optimizer.name if hasattr(optimizer, "name") else optimizer.__name__ + + # TODO: make these names less awful. + new_name = ("ngx_" if experimental else "ng_") + name + "_cube" + new_name = new_name.replace("OnePlusOne", "_1p1") + new_name = new_name.replace("PlusMiddlePoint", "_pmp") + new_name = new_name.replace("Search", "") + while "__" in new_name: + new_name = new_name.replace("__", "_") + f.__name__ = new_name.lower() + return f + + +# some selections: +ng_1p1_cube = named_optimizer(ol.OnePlusOne) +ng_bo_cube = named_optimizer(ol.BO) +ng_cauchy_1p1_cube = named_optimizer(ol.CauchyOnePlusOne) +ng_chaincmapowell_cube = named_optimizer(ol.ChainCMAPowell) +ng_chaindiagonalcmapowell_cube = named_optimizer(ol.ChainDiagonalCMAPowell) +ng_chainmetamodelpowell_cube = named_optimizer(ol.ChainMetaModelPowell) +ng_chainmetamodelsqp_cube = named_optimizer(ol.ChainMetaModelSQP) +ng_chainnaivetbpsapowell_cube = named_optimizer(ol.ChainNaiveTBPSAPowell) +ng_cma_cube = named_optimizer(ol.CMA) +ng_cobyla_cube = named_optimizer(ol.Cobyla) +ng_diagonalcma_cube = named_optimizer(ol.DiagonalCMA) +ng_ecma_cube = named_optimizer(ev.ECMA) +ng_fcma_cube = named_optimizer(ol.FCMA) +ng_fcmap13_cube = named_optimizer(ev.FCMAp13) +ng_fcmas03_cube = named_optimizer(ev.FCMAs03) +ng_metamodel_1p1_cube = named_optimizer(ol.MetaModelOnePlusOne) +ng_metamodel_cube = named_optimizer(ol.MetaModel) +ng_neldermead_cube = named_optimizer(ol.NelderMead) +ng_parametrizationde_cube = named_optimizer(ev.ParametrizationDE) +ng_powell_cube = named_optimizer(ol.Powell) +ng_rescaledcma_cube = named_optimizer(ol.RescaledCMA) +ng_rpowell_cube = named_optimizer(ol.RPowell) +ng_rsqp_cube = named_optimizer(ol.RSQP) +ng_sqp_cube = named_optimizer(ol.SQP) +ngx_chaincmasqp_cube = named_optimizer(ev.ChainCMASQP) +ngx_chaincmawithmetarecenteringdim_cube = named_optimizer(ev.ChainCMAwithMetaRecenteringdim) + +NEVERGRAD2_OPTIMIZERS = list(map(named_optimizer, optimizers)) + list( + map(lambda o: named_optimizer(o, experimental=True), more_optimizers) +) + +BAYES_OPTIMIZERS = list(map(lambda o: named_optimizer(o, experimental=o != ol.BO), bayes_optimizers)) diff --git a/notwacube.py b/notwacube.py index 782ce20..ebe4ad4 100644 --- a/notwacube.py +++ b/notwacube.py @@ -1,4 +1,5 @@ from dlibcube2 import dlib_cube +from nevergradcube2 import NEVERGRAD2_OPTIMIZERS from nloptcube2 import nlopt_neldermead_cube from notwacube2 import make_birect, make_mercury, make_soo from randomcube2 import another_random_cube, quasirandom_cube