382 lines
13 KiB
Python
382 lines
13 KiB
Python
from nevergrad.common.errors import InefficientSettingsWarning
|
|
import logging
|
|
import nevergrad as ng
|
|
import nevergrad.optimization.experimentalvariants as ev
|
|
import nevergrad.optimization.optimizerlib as ol
|
|
import warnings
|
|
|
|
logging.getLogger("nevergrad").setLevel(logging.WARNING)
|
|
warnings.filterwarnings("ignore", category=InefficientSettingsWarning)
|
|
|
|
optimizers = [
|
|
ol.ASCMADEthird,
|
|
ol.AdaptiveDiscreteOnePlusOne,
|
|
ol.AlmostRotationInvariantDE,
|
|
ol.AnisotropicAdaptiveDiscreteOnePlusOne,
|
|
ol.AvgMetaRecenteringNoHull,
|
|
# ol.BO, # slow
|
|
# ol.BOSplit, # slow
|
|
# ol.BayesOptimBO, # ModuleNotFoundError: No module named 'bayes_optim'
|
|
ol.CMA,
|
|
ol.CMAbounded,
|
|
ol.CMApara,
|
|
ol.CMAsmall,
|
|
ol.CMAstd,
|
|
ol.CMAtuning,
|
|
ol.CauchyLHSSearch,
|
|
ol.CauchyOnePlusOne,
|
|
ol.CauchyScrHammersleySearch,
|
|
ol.ChainCMAPowell,
|
|
ol.ChainDiagonalCMAPowell,
|
|
ol.ChainMetaModelPowell,
|
|
ol.ChainMetaModelSQP,
|
|
ol.ChainNaiveTBPSACMAPowell,
|
|
ol.ChainNaiveTBPSAPowell,
|
|
ol.CmaFmin2,
|
|
ol.Cobyla,
|
|
ol.DE,
|
|
ol.DiagonalCMA,
|
|
ol.DiscreteBSOOnePlusOne,
|
|
ol.DiscreteDE,
|
|
ol.DiscreteDoerrOnePlusOne,
|
|
ol.DiscreteLenglerOnePlusOne,
|
|
ol.DiscreteLenglerOnePlusOneT,
|
|
ol.DiscreteOnePlusOne,
|
|
ol.DiscreteOnePlusOneT,
|
|
ol.DoubleFastGADiscreteOnePlusOne,
|
|
ol.ES,
|
|
ol.FCMA,
|
|
ol.GeneticDE,
|
|
ol.HaltonSearch,
|
|
ol.HaltonSearchPlusMiddlePoint,
|
|
ol.HammersleySearch,
|
|
ol.HammersleySearchPlusMiddlePoint,
|
|
ol.HullAvgMetaRecentering,
|
|
ol.HullAvgMetaTuneRecentering,
|
|
# ol.HyperOpt, # fairly slow with larger problems (NOTE: newly removed)
|
|
ol.LHSSearch,
|
|
ol.LargeHaltonSearch,
|
|
ol.LhsDE,
|
|
ol.MetaModel,
|
|
ol.MetaModelOnePlusOne,
|
|
ol.MetaRecentering,
|
|
ol.MetaTuneRecentering,
|
|
ol.MixES,
|
|
ol.MultiCMA,
|
|
ol.MultiScaleCMA,
|
|
ol.MutDE,
|
|
# ol.NEWUOA, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.
|
|
# ol.NLOPT, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.
|
|
ol.NaiveIsoEMNA,
|
|
ol.NaiveTBPSA,
|
|
ol.NelderMead,
|
|
ol.NoisyDE,
|
|
ol.NoisyDiscreteOnePlusOne,
|
|
ol.NoisyOnePlusOne,
|
|
ol.NonNSGAIIES,
|
|
ol.ORandomSearch,
|
|
ol.OScrHammersleySearch,
|
|
ol.OldCMA,
|
|
ol.OnePlusOne,
|
|
ol.OptimisticDiscreteOnePlusOne,
|
|
ol.OptimisticNoisyOnePlusOne,
|
|
# ol.PCABO, # ModuleNotFoundError: No module named 'bayes_optim'
|
|
ol.PSO,
|
|
ol.ParaPortfolio,
|
|
ol.PolyCMA,
|
|
ol.PortfolioDiscreteOnePlusOne,
|
|
ol.PortfolioDiscreteOnePlusOneT,
|
|
ol.Powell,
|
|
# ol.PymooBatchNSGA2, # NotImplementedError: This optimizer isn't supported by the way minimize works by default.
|
|
# ol.PymooNSGA2, # RuntimeError: Recast optimizer raised an error: The method `get_algorithm` has been deprecated since 0.6.0 Please use the object-oriented interface.
|
|
ol.QORandomSearch,
|
|
ol.QOScrHammersleySearch,
|
|
ol.QrDE,
|
|
ol.RCobyla,
|
|
ol.RLSOnePlusOne,
|
|
ol.RPowell,
|
|
# ol.RSLSQP,
|
|
ol.RSQP,
|
|
ol.RandomSearch,
|
|
ol.RandomSearchPlusMiddlePoint,
|
|
ol.RealSpacePSO,
|
|
ol.RecES,
|
|
ol.RecMixES,
|
|
ol.RecMutDE,
|
|
ol.RecombiningPortfolioDiscreteOnePlusOne,
|
|
ol.RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne,
|
|
ol.RescaledCMA,
|
|
ol.RotatedTwoPointsDE,
|
|
ol.RotationInvariantDE,
|
|
ol.SADiscreteLenglerOnePlusOneExp09,
|
|
ol.SADiscreteLenglerOnePlusOneExp099,
|
|
ol.SADiscreteLenglerOnePlusOneExp09Auto,
|
|
ol.SADiscreteLenglerOnePlusOneLin1,
|
|
ol.SADiscreteLenglerOnePlusOneLin100,
|
|
ol.SADiscreteLenglerOnePlusOneLinAuto,
|
|
ol.SADiscreteOnePlusOneExp09,
|
|
ol.SADiscreteOnePlusOneExp099,
|
|
ol.SADiscreteOnePlusOneLin100,
|
|
# ol.SLSQP,
|
|
ol.SQP,
|
|
ol.ScrHaltonSearch,
|
|
ol.ScrHaltonSearchPlusMiddlePoint,
|
|
ol.ScrHammersleySearch,
|
|
ol.ScrHammersleySearchPlusMiddlePoint,
|
|
ol.SparseDoubleFastGADiscreteOnePlusOne,
|
|
ol.TBPSA,
|
|
ol.TripleCMA,
|
|
ol.TwoPointsDE,
|
|
ol.discretememetic,
|
|
]
|
|
|
|
more_optimizers = [
|
|
ev.AlmostRotationInvariantDEAndBigPop,
|
|
ev.AnisoEMNA,
|
|
ev.AnisoEMNATBPSA,
|
|
ev.AvgHammersleySearch,
|
|
ev.AvgHammersleySearchPlusMiddlePoint,
|
|
ev.AvgRandomSearch,
|
|
# ev.BO, # slow
|
|
ev.BPRotationInvariantDE,
|
|
ev.CMA,
|
|
ev.CauchyRandomSearch,
|
|
# ev.ChainBOwithLHS, # slow
|
|
# ev.ChainBOwithLHS30, # slow
|
|
# ev.ChainBOwithLHSdim, # slow
|
|
# ev.ChainBOwithLHSsqrt, # slow
|
|
# ev.ChainBOwithMetaRecentering, # slow
|
|
# ev.ChainBOwithMetaRecentering30, # slow
|
|
# ev.ChainBOwithMetaRecenteringdim, # slow
|
|
# ev.ChainBOwithMetaRecenteringsqrt, # slow
|
|
# ev.ChainBOwithMetaTuneRecentering, # slow
|
|
# ev.ChainBOwithMetaTuneRecentering30, # slow
|
|
# ev.ChainBOwithMetaTuneRecenteringdim, # slow
|
|
# ev.ChainBOwithMetaTuneRecenteringsqrt, # slow
|
|
# ev.ChainBOwithR, # slow
|
|
# ev.ChainBOwithR30, # slow
|
|
# ev.ChainBOwithRdim, # slow
|
|
# ev.ChainBOwithRsqrt, # slow
|
|
ev.ChainCMASQP,
|
|
ev.ChainCMAwithLHS,
|
|
ev.ChainCMAwithLHS30,
|
|
ev.ChainCMAwithLHSdim,
|
|
ev.ChainCMAwithLHSsqrt,
|
|
ev.ChainCMAwithMetaRecentering,
|
|
ev.ChainCMAwithMetaRecentering30,
|
|
ev.ChainCMAwithMetaRecenteringdim,
|
|
ev.ChainCMAwithMetaRecenteringsqrt,
|
|
ev.ChainCMAwithR,
|
|
ev.ChainCMAwithR30,
|
|
ev.ChainCMAwithRdim,
|
|
ev.ChainCMAwithRsqrt,
|
|
ev.ChainDEwithLHS,
|
|
ev.ChainDEwithLHS30,
|
|
ev.ChainDEwithLHSdim,
|
|
ev.ChainDEwithLHSsqrt,
|
|
ev.ChainDEwithMetaRecentering,
|
|
ev.ChainDEwithMetaRecentering30,
|
|
ev.ChainDEwithMetaRecenteringdim,
|
|
ev.ChainDEwithMetaRecenteringsqrt,
|
|
ev.ChainDEwithMetaTuneRecentering,
|
|
ev.ChainDEwithMetaTuneRecentering30,
|
|
ev.ChainDEwithMetaTuneRecenteringdim,
|
|
ev.ChainDEwithMetaTuneRecenteringsqrt,
|
|
ev.ChainDEwithR,
|
|
ev.ChainDEwithR30,
|
|
ev.ChainDEwithRdim,
|
|
ev.ChainDEwithRsqrt,
|
|
ev.ChainPSOwithLHS,
|
|
ev.ChainPSOwithLHS30,
|
|
ev.ChainPSOwithLHSdim,
|
|
ev.ChainPSOwithLHSsqrt,
|
|
ev.ChainPSOwithMetaRecentering,
|
|
ev.ChainPSOwithMetaRecentering30,
|
|
ev.ChainPSOwithMetaRecenteringdim,
|
|
ev.ChainPSOwithMetaRecenteringsqrt,
|
|
ev.ChainPSOwithR,
|
|
ev.ChainPSOwithR30,
|
|
ev.ChainPSOwithRdim,
|
|
ev.ChainPSOwithRsqrt,
|
|
ev.CmaFmin2,
|
|
ev.DE,
|
|
ev.DiagonalCMA,
|
|
ev.DiscreteNoisy13Splits,
|
|
ev.DiscreteNoisyInfSplits,
|
|
ev.DoubleFastGAOptimisticNoisyDiscreteOnePlusOne,
|
|
ev.ECMA,
|
|
ev.FCMAp13,
|
|
ev.FCMAs03,
|
|
ev.FastGADiscreteOnePlusOne,
|
|
ev.FastGANoisyDiscreteOnePlusOne,
|
|
ev.FastGAOptimisticNoisyDiscreteOnePlusOne,
|
|
ev.GeneticDE,
|
|
ev.HSCMA,
|
|
ev.HSDE,
|
|
ev.HSMetaModel,
|
|
ev.HullCenterHullAvgCauchyLHSSearch,
|
|
ev.HullCenterHullAvgCauchyScrHammersleySearch,
|
|
ev.HullCenterHullAvgLHSSearch,
|
|
ev.HullCenterHullAvgLargeHammersleySearch,
|
|
ev.HullCenterHullAvgRandomSearch,
|
|
ev.HullCenterHullAvgScrHaltonSearch,
|
|
ev.HullCenterHullAvgScrHaltonSearchPlusMiddlePoint,
|
|
ev.HullCenterHullAvgScrHammersleySearch,
|
|
ev.HullCenterHullAvgScrHammersleySearchPlusMiddlePoint,
|
|
ev.IsoEMNA,
|
|
ev.IsoEMNATBPSA,
|
|
# ev.LBO, # slow
|
|
ev.LHSSearch,
|
|
ev.LhsHSDE,
|
|
ev.MetaCauchyRecentering,
|
|
ev.MetaModelDiagonalCMA,
|
|
ev.MetaModelFmin2,
|
|
# ev.MetaNGOpt10, # hangs: Optimizing the schwefel function with the ngx_metangopt10 optimizer
|
|
ev.MetaRecentering,
|
|
ev.MetaTuneRecentering,
|
|
ev.MicroCMA,
|
|
# ev.MidQRBO, # slow
|
|
ev.MilliCMA,
|
|
ev.MiniDE,
|
|
ev.MiniLhsDE,
|
|
ev.MiniQrDE,
|
|
ev.MixDeterministicRL, # InefficientSettingsWarning: DE algorithms are inefficient with budget < 60
|
|
# ev.NGOptSingle16, # slow
|
|
# ev.NGOptSingle25, # slow
|
|
# ev.NGOptSingle9, # slow
|
|
ev.NaiveAnisoEMNA,
|
|
ev.NaiveAnisoEMNATBPSA,
|
|
ev.NaiveIsoEMNATBPSA,
|
|
ev.Noisy13Splits,
|
|
ev.NoisyInfSplits,
|
|
ev.NoisyOnePlusOne,
|
|
ev.NoisyRL1, # note: inefficiency warnings
|
|
ev.NoisyRL2, # note: inefficiency warnings
|
|
ev.NoisyRL3, # note: inefficiency warnings
|
|
ev.OnePointDE,
|
|
ev.OptimisticNoisyOnePlusOne,
|
|
# ev.PCABO80, # ModuleNotFoundError: No module named 'bayes_optim'
|
|
# ev.PCABO95DoE20, # ModuleNotFoundError: No module named 'bayes_optim'
|
|
ev.PSO,
|
|
ev.ParametrizationDE,
|
|
ev.PortfolioNoisyDiscreteOnePlusOne,
|
|
ev.PortfolioOptimisticNoisyDiscreteOnePlusOne,
|
|
# ev.QRBO, # slow
|
|
# ev.RBO, # slow
|
|
ev.RandomScaleRandomSearch,
|
|
ev.RandomScaleRandomSearchPlusMiddlePoint,
|
|
ev.RandomSearch,
|
|
ev.RecombiningGA,
|
|
ev.RecombiningOptimisticNoisyDiscreteOnePlusOne,
|
|
ev.RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne,
|
|
ev.RescaleScrHammersleySearch,
|
|
ev.RotatedRecombiningGA,
|
|
ev.SQP,
|
|
ev.SmoothAdaptiveDiscreteOnePlusOne,
|
|
ev.SmoothDiscreteLenglerOnePlusOne,
|
|
ev.SmoothDiscreteOnePlusOne,
|
|
ev.SmoothPortfolioDiscreteOnePlusOne,
|
|
ev.SparseDiscreteOnePlusOne,
|
|
ev.SpecialRL, # note: inefficiency warnings
|
|
ev.StupidRandom,
|
|
ev.TBPSA,
|
|
ev.TEAvgCauchyLHSSearch,
|
|
ev.TEAvgCauchyScrHammersleySearch,
|
|
ev.TEAvgLHSSearch,
|
|
ev.TEAvgRandomSearch,
|
|
ev.TEAvgScrHammersleySearch,
|
|
ev.TEAvgScrHammersleySearchPlusMiddlePoint,
|
|
ev.Zero,
|
|
]
|
|
|
|
bayes_optimizers = [
|
|
ol.BO,
|
|
ev.RBO,
|
|
ev.QRBO,
|
|
ev.MidQRBO,
|
|
ev.LBO,
|
|
ol.ParametrizedBO(utility_kind="ei").set_name("BOEI"),
|
|
ol.ParametrizedBO(utility_kind="poi").set_name("BOPOI"),
|
|
]
|
|
|
|
assert ol.RSLSQP is ol.RSQP, "weirdness is gone, please adjust accordingly"
|
|
assert ol.SLSQP is ol.SQP, "weirdness is gone, please adjust accordingly"
|
|
|
|
|
|
def nevergrad_cube_factory(optimizer, objective, size, budget):
|
|
instrument = ng.p.Array(lower=0, upper=1, shape=(size,)) # better sigma
|
|
# ev.RBO, ev.QRBO, ev.MidQRBO, and ev.LBO still complain, though:
|
|
# /home/py/.local/lib/python3.10/site-packages/nevergrad/parametrization/_datalayers.py:107: NevergradRuntimeWarning: Bounds are 1.0 sigma away from each other at the closest, you should aim for at least 3 for better quality.
|
|
if optimizer in (ev.RBO, ev.QRBO, ev.MidQRBO, ev.LBO):
|
|
instrument.set_mutation(sigma=0.3)
|
|
|
|
opt = optimizer
|
|
assert opt is not None, optimizer
|
|
|
|
optimizer = opt(parametrization=instrument, budget=budget, num_workers=1)
|
|
|
|
feval_count = 0
|
|
|
|
def cube_objective(us):
|
|
nonlocal feval_count
|
|
feval_count += 1
|
|
return objective(us)
|
|
|
|
recommendation = optimizer.minimize(cube_objective)
|
|
best_x = recommendation.value
|
|
best_val = cube_objective(best_x) # don't trust recommendation.loss
|
|
|
|
return best_val, best_x, feval_count
|
|
|
|
|
|
def named_optimizer(optimizer, experimental=False):
|
|
def f(*args, **kwargs):
|
|
return nevergrad_cube_factory(optimizer, *args, **kwargs)
|
|
|
|
name = optimizer.name if hasattr(optimizer, "name") else optimizer.__name__
|
|
|
|
# TODO: make these names less awful.
|
|
new_name = ("ngx_" if experimental else "ng_") + name + "_cube"
|
|
new_name = new_name.replace("OnePlusOne", "_1p1")
|
|
new_name = new_name.replace("PlusMiddlePoint", "_pmp")
|
|
new_name = new_name.replace("Search", "")
|
|
while "__" in new_name:
|
|
new_name = new_name.replace("__", "_")
|
|
f.__name__ = new_name.lower()
|
|
return f
|
|
|
|
|
|
# some selections:
|
|
ng_1p1_cube = named_optimizer(ol.OnePlusOne)
|
|
ng_bo_cube = named_optimizer(ol.BO)
|
|
ng_cauchy_1p1_cube = named_optimizer(ol.CauchyOnePlusOne)
|
|
ng_chaincmapowell_cube = named_optimizer(ol.ChainCMAPowell)
|
|
ng_chaindiagonalcmapowell_cube = named_optimizer(ol.ChainDiagonalCMAPowell)
|
|
ng_chainmetamodelpowell_cube = named_optimizer(ol.ChainMetaModelPowell)
|
|
ng_chainmetamodelsqp_cube = named_optimizer(ol.ChainMetaModelSQP)
|
|
ng_chainnaivetbpsapowell_cube = named_optimizer(ol.ChainNaiveTBPSAPowell)
|
|
ng_cma_cube = named_optimizer(ol.CMA)
|
|
ng_cobyla_cube = named_optimizer(ol.Cobyla)
|
|
ng_diagonalcma_cube = named_optimizer(ol.DiagonalCMA)
|
|
ng_ecma_cube = named_optimizer(ev.ECMA)
|
|
ng_fcma_cube = named_optimizer(ol.FCMA)
|
|
ng_fcmap13_cube = named_optimizer(ev.FCMAp13)
|
|
ng_fcmas03_cube = named_optimizer(ev.FCMAs03)
|
|
ng_metamodel_1p1_cube = named_optimizer(ol.MetaModelOnePlusOne)
|
|
ng_metamodel_cube = named_optimizer(ol.MetaModel)
|
|
ng_neldermead_cube = named_optimizer(ol.NelderMead)
|
|
ng_parametrizationde_cube = named_optimizer(ev.ParametrizationDE)
|
|
ng_powell_cube = named_optimizer(ol.Powell)
|
|
ng_rescaledcma_cube = named_optimizer(ol.RescaledCMA)
|
|
ng_rpowell_cube = named_optimizer(ol.RPowell)
|
|
ng_rsqp_cube = named_optimizer(ol.RSQP)
|
|
ng_sqp_cube = named_optimizer(ol.SQP)
|
|
ngx_chaincmasqp_cube = named_optimizer(ev.ChainCMASQP)
|
|
ngx_chaincmawithmetarecenteringdim_cube = named_optimizer(ev.ChainCMAwithMetaRecenteringdim)
|
|
|
|
NEVERGRAD2_OPTIMIZERS = list(map(named_optimizer, optimizers)) + list(
|
|
map(lambda o: named_optimizer(o, experimental=True), more_optimizers)
|
|
)
|
|
|
|
BAYES_OPTIMIZERS = list(map(lambda o: named_optimizer(o, experimental=o != ol.BO), bayes_optimizers))
|