2023-05-04 15:52:08 -07:00
from nevergrad . common . errors import InefficientSettingsWarning
import logging
import nevergrad as ng
import nevergrad . optimization . experimentalvariants as ev
import nevergrad . optimization . optimizerlib as ol
import warnings
logging . getLogger ( " nevergrad " ) . setLevel ( logging . WARNING )
warnings . filterwarnings ( " ignore " , category = InefficientSettingsWarning )
optimizers = [
ol . ASCMADEthird ,
ol . AdaptiveDiscreteOnePlusOne ,
ol . AlmostRotationInvariantDE ,
ol . AnisotropicAdaptiveDiscreteOnePlusOne ,
ol . AvgMetaRecenteringNoHull ,
# ol.BO, # slow
# ol.BOSplit, # slow
# ol.BayesOptimBO, # ModuleNotFoundError: No module named 'bayes_optim'
ol . CMA ,
ol . CMAbounded ,
ol . CMApara ,
ol . CMAsmall ,
ol . CMAstd ,
ol . CMAtuning ,
ol . CauchyLHSSearch ,
ol . CauchyOnePlusOne ,
ol . CauchyScrHammersleySearch ,
ol . ChainCMAPowell ,
ol . ChainDiagonalCMAPowell ,
ol . ChainMetaModelPowell ,
ol . ChainMetaModelSQP ,
ol . ChainNaiveTBPSACMAPowell ,
ol . ChainNaiveTBPSAPowell ,
ol . CmaFmin2 ,
ol . Cobyla ,
ol . DE ,
ol . DiagonalCMA ,
ol . DiscreteBSOOnePlusOne ,
ol . DiscreteDE ,
ol . DiscreteDoerrOnePlusOne ,
ol . DiscreteLenglerOnePlusOne ,
ol . DiscreteLenglerOnePlusOneT ,
ol . DiscreteOnePlusOne ,
ol . DiscreteOnePlusOneT ,
ol . DoubleFastGADiscreteOnePlusOne ,
ol . ES ,
ol . FCMA ,
ol . GeneticDE ,
ol . HaltonSearch ,
ol . HaltonSearchPlusMiddlePoint ,
ol . HammersleySearch ,
ol . HammersleySearchPlusMiddlePoint ,
ol . HullAvgMetaRecentering ,
ol . HullAvgMetaTuneRecentering ,
# ol.HyperOpt, # fairly slow with larger problems (NOTE: newly removed)
ol . LHSSearch ,
ol . LargeHaltonSearch ,
ol . LhsDE ,
ol . MetaModel ,
ol . MetaModelOnePlusOne ,
ol . MetaRecentering ,
ol . MetaTuneRecentering ,
ol . MixES ,
ol . MultiCMA ,
ol . MultiScaleCMA ,
ol . MutDE ,
# ol.NEWUOA, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.
# ol.NLOPT, # RuntimeError: Recast optimizer raised an error: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.
ol . NaiveIsoEMNA ,
ol . NaiveTBPSA ,
ol . NelderMead ,
ol . NoisyDE ,
ol . NoisyDiscreteOnePlusOne ,
ol . NoisyOnePlusOne ,
ol . NonNSGAIIES ,
ol . ORandomSearch ,
ol . OScrHammersleySearch ,
ol . OldCMA ,
ol . OnePlusOne ,
ol . OptimisticDiscreteOnePlusOne ,
ol . OptimisticNoisyOnePlusOne ,
# ol.PCABO, # ModuleNotFoundError: No module named 'bayes_optim'
ol . PSO ,
ol . ParaPortfolio ,
ol . PolyCMA ,
ol . PortfolioDiscreteOnePlusOne ,
ol . PortfolioDiscreteOnePlusOneT ,
ol . Powell ,
# ol.PymooBatchNSGA2, # NotImplementedError: This optimizer isn't supported by the way minimize works by default.
# ol.PymooNSGA2, # RuntimeError: Recast optimizer raised an error: The method `get_algorithm` has been deprecated since 0.6.0 Please use the object-oriented interface.
ol . QORandomSearch ,
ol . QOScrHammersleySearch ,
ol . QrDE ,
ol . RCobyla ,
ol . RLSOnePlusOne ,
ol . RPowell ,
# ol.RSLSQP,
ol . RSQP ,
ol . RandomSearch ,
ol . RandomSearchPlusMiddlePoint ,
ol . RealSpacePSO ,
ol . RecES ,
ol . RecMixES ,
ol . RecMutDE ,
ol . RecombiningPortfolioDiscreteOnePlusOne ,
ol . RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne ,
ol . RescaledCMA ,
ol . RotatedTwoPointsDE ,
ol . RotationInvariantDE ,
ol . SADiscreteLenglerOnePlusOneExp09 ,
ol . SADiscreteLenglerOnePlusOneExp099 ,
ol . SADiscreteLenglerOnePlusOneExp09Auto ,
ol . SADiscreteLenglerOnePlusOneLin1 ,
ol . SADiscreteLenglerOnePlusOneLin100 ,
ol . SADiscreteLenglerOnePlusOneLinAuto ,
ol . SADiscreteOnePlusOneExp09 ,
ol . SADiscreteOnePlusOneExp099 ,
ol . SADiscreteOnePlusOneLin100 ,
# ol.SLSQP,
ol . SQP ,
ol . ScrHaltonSearch ,
ol . ScrHaltonSearchPlusMiddlePoint ,
ol . ScrHammersleySearch ,
ol . ScrHammersleySearchPlusMiddlePoint ,
ol . SparseDoubleFastGADiscreteOnePlusOne ,
ol . TBPSA ,
ol . TripleCMA ,
ol . TwoPointsDE ,
ol . discretememetic ,
]
more_optimizers = [
ev . AlmostRotationInvariantDEAndBigPop ,
ev . AnisoEMNA ,
ev . AnisoEMNATBPSA ,
ev . AvgHammersleySearch ,
ev . AvgHammersleySearchPlusMiddlePoint ,
ev . AvgRandomSearch ,
# ev.BO, # slow
ev . BPRotationInvariantDE ,
ev . CMA ,
ev . CauchyRandomSearch ,
# ev.ChainBOwithLHS, # slow
# ev.ChainBOwithLHS30, # slow
# ev.ChainBOwithLHSdim, # slow
# ev.ChainBOwithLHSsqrt, # slow
# ev.ChainBOwithMetaRecentering, # slow
# ev.ChainBOwithMetaRecentering30, # slow
# ev.ChainBOwithMetaRecenteringdim, # slow
# ev.ChainBOwithMetaRecenteringsqrt, # slow
# ev.ChainBOwithMetaTuneRecentering, # slow
# ev.ChainBOwithMetaTuneRecentering30, # slow
# ev.ChainBOwithMetaTuneRecenteringdim, # slow
# ev.ChainBOwithMetaTuneRecenteringsqrt, # slow
# ev.ChainBOwithR, # slow
# ev.ChainBOwithR30, # slow
# ev.ChainBOwithRdim, # slow
# ev.ChainBOwithRsqrt, # slow
ev . ChainCMASQP ,
ev . ChainCMAwithLHS ,
ev . ChainCMAwithLHS30 ,
ev . ChainCMAwithLHSdim ,
ev . ChainCMAwithLHSsqrt ,
ev . ChainCMAwithMetaRecentering ,
ev . ChainCMAwithMetaRecentering30 ,
ev . ChainCMAwithMetaRecenteringdim ,
ev . ChainCMAwithMetaRecenteringsqrt ,
ev . ChainCMAwithR ,
ev . ChainCMAwithR30 ,
ev . ChainCMAwithRdim ,
ev . ChainCMAwithRsqrt ,
ev . ChainDEwithLHS ,
ev . ChainDEwithLHS30 ,
ev . ChainDEwithLHSdim ,
ev . ChainDEwithLHSsqrt ,
ev . ChainDEwithMetaRecentering ,
ev . ChainDEwithMetaRecentering30 ,
ev . ChainDEwithMetaRecenteringdim ,
ev . ChainDEwithMetaRecenteringsqrt ,
ev . ChainDEwithMetaTuneRecentering ,
ev . ChainDEwithMetaTuneRecentering30 ,
ev . ChainDEwithMetaTuneRecenteringdim ,
ev . ChainDEwithMetaTuneRecenteringsqrt ,
ev . ChainDEwithR ,
ev . ChainDEwithR30 ,
ev . ChainDEwithRdim ,
ev . ChainDEwithRsqrt ,
ev . ChainPSOwithLHS ,
ev . ChainPSOwithLHS30 ,
ev . ChainPSOwithLHSdim ,
ev . ChainPSOwithLHSsqrt ,
ev . ChainPSOwithMetaRecentering ,
ev . ChainPSOwithMetaRecentering30 ,
ev . ChainPSOwithMetaRecenteringdim ,
ev . ChainPSOwithMetaRecenteringsqrt ,
ev . ChainPSOwithR ,
ev . ChainPSOwithR30 ,
ev . ChainPSOwithRdim ,
ev . ChainPSOwithRsqrt ,
ev . CmaFmin2 ,
ev . DE ,
ev . DiagonalCMA ,
ev . DiscreteNoisy13Splits ,
ev . DiscreteNoisyInfSplits ,
ev . DoubleFastGAOptimisticNoisyDiscreteOnePlusOne ,
ev . ECMA ,
ev . FCMAp13 ,
ev . FCMAs03 ,
ev . FastGADiscreteOnePlusOne ,
ev . FastGANoisyDiscreteOnePlusOne ,
ev . FastGAOptimisticNoisyDiscreteOnePlusOne ,
ev . GeneticDE ,
ev . HSCMA ,
ev . HSDE ,
ev . HSMetaModel ,
ev . HullCenterHullAvgCauchyLHSSearch ,
ev . HullCenterHullAvgCauchyScrHammersleySearch ,
ev . HullCenterHullAvgLHSSearch ,
ev . HullCenterHullAvgLargeHammersleySearch ,
ev . HullCenterHullAvgRandomSearch ,
ev . HullCenterHullAvgScrHaltonSearch ,
ev . HullCenterHullAvgScrHaltonSearchPlusMiddlePoint ,
ev . HullCenterHullAvgScrHammersleySearch ,
ev . HullCenterHullAvgScrHammersleySearchPlusMiddlePoint ,
ev . IsoEMNA ,
ev . IsoEMNATBPSA ,
# ev.LBO, # slow
ev . LHSSearch ,
ev . LhsHSDE ,
ev . MetaCauchyRecentering ,
ev . MetaModelDiagonalCMA ,
ev . MetaModelFmin2 ,
# ev.MetaNGOpt10, # hangs: Optimizing the schwefel function with the ngx_metangopt10 optimizer
ev . MetaRecentering ,
ev . MetaTuneRecentering ,
ev . MicroCMA ,
# ev.MidQRBO, # slow
ev . MilliCMA ,
ev . MiniDE ,
ev . MiniLhsDE ,
ev . MiniQrDE ,
ev . MixDeterministicRL , # InefficientSettingsWarning: DE algorithms are inefficient with budget < 60
# ev.NGOptSingle16, # slow
# ev.NGOptSingle25, # slow
# ev.NGOptSingle9, # slow
ev . NaiveAnisoEMNA ,
ev . NaiveAnisoEMNATBPSA ,
ev . NaiveIsoEMNATBPSA ,
ev . Noisy13Splits ,
ev . NoisyInfSplits ,
ev . NoisyOnePlusOne ,
ev . NoisyRL1 , # note: inefficiency warnings
ev . NoisyRL2 , # note: inefficiency warnings
ev . NoisyRL3 , # note: inefficiency warnings
ev . OnePointDE ,
ev . OptimisticNoisyOnePlusOne ,
# ev.PCABO80, # ModuleNotFoundError: No module named 'bayes_optim'
# ev.PCABO95DoE20, # ModuleNotFoundError: No module named 'bayes_optim'
ev . PSO ,
ev . ParametrizationDE ,
ev . PortfolioNoisyDiscreteOnePlusOne ,
ev . PortfolioOptimisticNoisyDiscreteOnePlusOne ,
# ev.QRBO, # slow
# ev.RBO, # slow
ev . RandomScaleRandomSearch ,
ev . RandomScaleRandomSearchPlusMiddlePoint ,
ev . RandomSearch ,
ev . RecombiningGA ,
ev . RecombiningOptimisticNoisyDiscreteOnePlusOne ,
ev . RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne ,
ev . RescaleScrHammersleySearch ,
ev . RotatedRecombiningGA ,
ev . SQP ,
ev . SmoothAdaptiveDiscreteOnePlusOne ,
ev . SmoothDiscreteLenglerOnePlusOne ,
ev . SmoothDiscreteOnePlusOne ,
ev . SmoothPortfolioDiscreteOnePlusOne ,
ev . SparseDiscreteOnePlusOne ,
ev . SpecialRL , # note: inefficiency warnings
ev . StupidRandom ,
ev . TBPSA ,
ev . TEAvgCauchyLHSSearch ,
ev . TEAvgCauchyScrHammersleySearch ,
ev . TEAvgLHSSearch ,
ev . TEAvgRandomSearch ,
ev . TEAvgScrHammersleySearch ,
ev . TEAvgScrHammersleySearchPlusMiddlePoint ,
ev . Zero ,
]
bayes_optimizers = [
ol . BO ,
ev . RBO ,
ev . QRBO ,
ev . MidQRBO ,
ev . LBO ,
ol . ParametrizedBO ( utility_kind = " ei " ) . set_name ( " BOEI " ) ,
ol . ParametrizedBO ( utility_kind = " poi " ) . set_name ( " BOPOI " ) ,
]
assert ol . RSLSQP is ol . RSQP , " weirdness is gone, please adjust accordingly "
assert ol . SLSQP is ol . SQP , " weirdness is gone, please adjust accordingly "
2023-05-07 07:30:55 -07:00
def nevergrad_cube_factory ( optimizer , objective , size , budget ) :
instrument = ng . p . Array ( lower = 0 , upper = 1 , shape = ( size , ) ) # better sigma
2023-05-04 15:52:08 -07:00
# ev.RBO, ev.QRBO, ev.MidQRBO, and ev.LBO still complain, though:
# /home/py/.local/lib/python3.10/site-packages/nevergrad/parametrization/_datalayers.py:107: NevergradRuntimeWarning: Bounds are 1.0 sigma away from each other at the closest, you should aim for at least 3 for better quality.
if optimizer in ( ev . RBO , ev . QRBO , ev . MidQRBO , ev . LBO ) :
instrument . set_mutation ( sigma = 0.3 )
opt = optimizer
assert opt is not None , optimizer
2023-05-07 07:30:55 -07:00
optimizer = opt ( parametrization = instrument , budget = budget , num_workers = 1 )
2023-05-04 15:52:08 -07:00
feval_count = 0
def cube_objective ( us ) :
nonlocal feval_count
feval_count + = 1
return objective ( us )
recommendation = optimizer . minimize ( cube_objective )
best_x = recommendation . value
best_val = cube_objective ( best_x ) # don't trust recommendation.loss
2023-05-07 07:21:13 -07:00
return best_val , best_x , feval_count
2023-05-04 15:52:08 -07:00
def named_optimizer ( optimizer , experimental = False ) :
def f ( * args , * * kwargs ) :
return nevergrad_cube_factory ( optimizer , * args , * * kwargs )
name = optimizer . name if hasattr ( optimizer , " name " ) else optimizer . __name__
# TODO: make these names less awful.
new_name = ( " ngx_ " if experimental else " ng_ " ) + name + " _cube "
new_name = new_name . replace ( " OnePlusOne " , " _1p1 " )
new_name = new_name . replace ( " PlusMiddlePoint " , " _pmp " )
new_name = new_name . replace ( " Search " , " " )
while " __ " in new_name :
new_name = new_name . replace ( " __ " , " _ " )
f . __name__ = new_name . lower ( )
return f
# some selections:
ng_1p1_cube = named_optimizer ( ol . OnePlusOne )
ng_bo_cube = named_optimizer ( ol . BO )
ng_cauchy_1p1_cube = named_optimizer ( ol . CauchyOnePlusOne )
ng_chaincmapowell_cube = named_optimizer ( ol . ChainCMAPowell )
ng_chaindiagonalcmapowell_cube = named_optimizer ( ol . ChainDiagonalCMAPowell )
ng_chainmetamodelpowell_cube = named_optimizer ( ol . ChainMetaModelPowell )
ng_chainmetamodelsqp_cube = named_optimizer ( ol . ChainMetaModelSQP )
ng_chainnaivetbpsapowell_cube = named_optimizer ( ol . ChainNaiveTBPSAPowell )
ng_cma_cube = named_optimizer ( ol . CMA )
ng_cobyla_cube = named_optimizer ( ol . Cobyla )
ng_diagonalcma_cube = named_optimizer ( ol . DiagonalCMA )
ng_ecma_cube = named_optimizer ( ev . ECMA )
ng_fcma_cube = named_optimizer ( ol . FCMA )
ng_fcmap13_cube = named_optimizer ( ev . FCMAp13 )
ng_fcmas03_cube = named_optimizer ( ev . FCMAs03 )
ng_metamodel_1p1_cube = named_optimizer ( ol . MetaModelOnePlusOne )
ng_metamodel_cube = named_optimizer ( ol . MetaModel )
ng_neldermead_cube = named_optimizer ( ol . NelderMead )
ng_parametrizationde_cube = named_optimizer ( ev . ParametrizationDE )
ng_powell_cube = named_optimizer ( ol . Powell )
ng_rescaledcma_cube = named_optimizer ( ol . RescaledCMA )
ng_rpowell_cube = named_optimizer ( ol . RPowell )
ng_rsqp_cube = named_optimizer ( ol . RSQP )
ng_sqp_cube = named_optimizer ( ol . SQP )
ngx_chaincmasqp_cube = named_optimizer ( ev . ChainCMASQP )
ngx_chaincmawithmetarecenteringdim_cube = named_optimizer ( ev . ChainCMAwithMetaRecenteringdim )
NEVERGRAD2_OPTIMIZERS = list ( map ( named_optimizer , optimizers ) ) + list (
map ( lambda o : named_optimizer ( o , experimental = True ) , more_optimizers )
)
BAYES_OPTIMIZERS = list ( map ( lambda o : named_optimizer ( o , experimental = o != ol . BO ) , bayes_optimizers ) )