From 05343dad244a1b51eb2b3d01bbbaf26b853da6d0 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Fri, 5 May 2023 14:11:44 -0700 Subject: [PATCH] add history of optimal values and incorporate into each ranking --- go_benchmark_it.py | 32 ++++++++++++++++++------------ utils.py | 29 ++++++++++++++++++--------- utils_np.py | 49 +++++++++++++++++++++++++++++++++++++++------- 3 files changed, 82 insertions(+), 28 deletions(-) diff --git a/go_benchmark_it.py b/go_benchmark_it.py index d7ebff9..51020e3 100644 --- a/go_benchmark_it.py +++ b/go_benchmark_it.py @@ -269,21 +269,28 @@ if __name__ == "__main__": new_results = {} for obj_name, obj_res in results.items(): new_res = {} - for fopt, opt_name in sorted(obj_res): - new_res.setdefault(opt_name, []).append(fopt) - for opt_name, fopts in new_res.items(): + for fopt, opt_name, extra in sorted(obj_res): + l = new_res.setdefault(opt_name, [[], []]) + l[0].append(fopt) + l[1].append(extra) + slices = {} + for opt_name, res in new_res.items(): # in the event that an odd number of results needs to be trimmed, # prefer trimming from the bottom (i.e. worse solutions get removed first). + fopts, extras = res down = (len(fopts) - multiple) // 2 up = len(fopts) - (len(fopts) - multiple + 1) // 2 - # print("asdf", len(fopts), down, up) - new_res[opt_name] = fopts[down:up] - for opt_name, fopts in new_res.items(): + slices[opt_name] = slice(down, up) + for opt_name, res in new_res.items(): + fopts, extras = res + s = slices[opt_name] + fopts, extras = fopts[s], extras[s] if not no_summary: assert len(fopts) == multiple, (len(fopts), multiple) if len(fopts) == multiple: - for fopt in fopts: - new_results.setdefault(obj_name, []).append((fopt, opt_name)) + for fopt, extra in zip(fopts, extras): + result = (fopt, opt_name, extra) + new_results.setdefault(obj_name, []).append(result) return results reset = "\033[m" @@ -354,8 +361,8 @@ if __name__ == "__main__": run = 1 while (cache := wrapped.cached(run)) is not None: run += 1 - fopt, xopt = cache - results.setdefault(obj_name, []).append((fopt, opt_name)) + fopt, xopt, history = cache + results.setdefault(obj_name, []).append((fopt, opt_name, history)) note = (lambda s: None) if quieter else m36 @@ -377,7 +384,8 @@ if __name__ == "__main__": ) _ = optimizer(wrapped, n_trials=n_trials, n_dim=n_dim, with_count=False) fopt, xopt = wrapped.finish() - results.setdefault(obj_name, []).append((fopt, opt_name)) + result = (fopt, opt_name, wrapped.history) + results.setdefault(obj_name, []).append(result) once = True run += 1 @@ -391,7 +399,7 @@ if __name__ == "__main__": print() m1(f"{obj_name}:") all_res = {} - for fopt, opt_name in obj_res: + for fopt, opt_name, extra in obj_res: all_res.setdefault(fopt, []).append(opt_name) all_opt_names.add(opt_name) scores.setdefault(opt_name, 0.0) diff --git a/utils.py b/utils.py index caff946..8c4be65 100644 --- a/utils.py +++ b/utils.py @@ -173,6 +173,16 @@ class AcquireForWriting: def perform_another_experimental_scoring_method(results): + if len(results) and len(something := next(iter(results.values()))[0]) == 3: + history_length = len(something[2]) + each = {} + for i in range(history_length): + # for k, v in results.items(): for vi in v: assert len(vi) == 3, vi + l = {k: [(res[2][i], res[1]) for res in v] for k, v in results.items()} + for k, v in perform_another_experimental_scoring_method(l).items(): + each.setdefault(k, []).append(v) + return {k: sum(v) / len(v) for k, v in each.items()} + new_results = {} all_opt_names = set() for obj_name, obj_res in results.items(): @@ -203,7 +213,7 @@ def perform_another_experimental_scoring_method(results): scores = {k: 0.0 for k in all_opt_names} for opt_name, (rank, count) in best_ranks_and_counts.items(): - points = 2**(1 - rank) + points = 2 ** (1 - rank) count = min(count, limited_by_floating_point_precision) scores[opt_name] = score = sum(points / 2**i for i in range(count)) @@ -211,17 +221,18 @@ def perform_another_experimental_scoring_method(results): def needs_rerun(key, value): + if value["duration"] < 0.0 or "history" not in value: + return True + + if value["timestamp"] < 1683295630.0: # bugged history field + return True + + if not value["history"]: # not sure what happened here + return True + n_dim = len(value["xopt"]) ng = [] - if value["duration"] > -1683000000.0: - if n_dim == 3: - ng += "go_ackley01 go_alpine01 go_deflectedcorrugatedspring go_exponential go_griewank go_infinity go_mishra11 go_multimodal go_pinter go_qing go_rastrigin go_salomon go_schwefel01 go_schwefel20 go_schwefel21 go_schwefel22 go_sineenvelope go_sodp go_trigonometric02 go_wavy go_weierstrass go_xinsheyang01 go_xinsheyang02 go_xinsheyang03 go_yaoliu04 go_yaoliu09 go_zerosum".split() - ng += "go_amgm go_brown go_cigar go_deb01 go_deb03 go_deceptive go_needleeye go_penalty01 go_schwefel04 go_schwefel26 go_sphere go_stretchedv go_styblinskitang go_zacharov".split() - if n_dim == 4: - ng += "go_ackley01 go_alpine01 go_cigar go_exponential go_griewank go_infinity go_pinter go_qing go_schwefel01 go_schwefel02 go_schwefel20 go_schwefel21 go_schwefel22 go_sineenvelope go_sphere go_step2 go_step go_stretchedv go_trigonometric02 go_weierstrass go_whitley go_xinsheyang01 go_xinsheyang02 go_yaoliu04 go_yaoliu09 go_zerosum".split() - ng += "go_amgm go_brown go_colville go_deb03 go_penalty01 go_penalty02 go_powell go_rosenbrock go_schwefel04 go_shekel05 go_shekel07 go_shekel10 go_shubert03 go_shubert04 go_styblinskitang go_vincent".split() - kd = decode_key(key) assert kd is not None, key if kd.obj in ng: diff --git a/utils_np.py b/utils_np.py index 3c3706a..7d2bef2 100644 --- a/utils_np.py +++ b/utils_np.py @@ -1,4 +1,5 @@ # i've separated numpy-dependent methods from the rest of the utils. +from time import time from utils import AcquireForWriting, merge_summaries, feps, m33, m34, m93 import numpy as np @@ -60,7 +61,14 @@ def do_bounding(x, method="clip"): class OWrap: - def __init__(self, objective, n_trials, frugal_percent=1.0, greedy_percent=2.0): + def __init__( + self, + objective, + n_trials, + frugal_percent=1.0, + greedy_percent=2.0, + history_frequency=10, + ): self.feval_count = 0 self.best_so_far = None self.warning = None @@ -69,6 +77,8 @@ class OWrap: self.__name__ = objective.__name__ # for evolopy self.frugal_percent = float(frugal_percent) self.greedy_percent = float(greedy_percent) + self.history_frequency = history_frequency + self.history = [] def __str__(self): return ( @@ -106,6 +116,10 @@ class OWrap: if self.best_so_far is None or fx < self.best_so_far[0]: self.best_so_far = (fx, x) + if self.history_frequency > 0: + if self.feval_count % self.history_frequency == 0: + self.history.append(self.best_so_far[0]) + return float(fx) def finish(self, optimizer_name): @@ -137,6 +151,7 @@ class COWrap: self.n_dim = n_dim self.kwargs = kwargs self._dirty = False + self._history = None from pathlib import Path @@ -157,6 +172,8 @@ class COWrap: def __call__(self, x, *args, **kwargs): assert not self._ran, "please run .finish() before continuing!" + if not self._dirty: + self.start_time = time() result = self.ow.__call__(x, *args, **kwargs) self._dirty = True return result @@ -176,6 +193,12 @@ class COWrap: opt_name = self.optimizer.__name__ return f"COWrap_d{self.n_dim:02}_n{self.n_trials:03}_{opt_name}" + @property + def history(self): + assert not self._dirty + assert self._history + return self._history + @property def cache_key(self): opt_name = self.optimizer.__name__ @@ -237,15 +260,16 @@ class COWrap: return None assert "fopt" in summary, summary assert "xopt" in summary, summary - assert "duration" in summary, summary + assert "timestamp" in summary, summary + assert "history" in summary, summary fopt = float(summary["fopt"]) xopt = np.array(summary["xopt"], np.float64) - duration = float(summary["duration"]) - return fopt, xopt + history = [fval for fval in summary["history"]] + assert history, "history cannot be empty" # this should get filtered now + return fopt, xopt, history def finish(self, opt_name=None): from json import dumps - from time import time assert self._dirty self._ran = True @@ -255,12 +279,23 @@ class COWrap: assert self.ow.best_so_far is not None # fopt, xopt = self.ow.best_so_far fopt, xopt = self.ow.finish(self.optimizer.__name__) + + expected_length = self.n_trials // self.ow.history_frequency + history = [float(fval) for fval in self.ow.history] + history += [fopt] * (expected_length - len(history)) + + finish_time = time() summary = dict( fopt=float(fopt), xopt=[float(x) for x in xopt], - # duration=float(-1), # old, bad for uniqueness - duration=float(-time()), + # timestamp=float(-1), # old, bad for uniqueness + timestamp=finish_time, + # optional: (for now?) + history=history, + duration=finish_time - self.start_time, ) + self._history = summary["history"] + assert self._history, "why" with AcquireForWriting(self.cache_file) as fp: self._cached_summaries = None # force reload