add history of optimal values and incorporate into each ranking

go_benchmark_it.py

@@ -269,21 +269,28 @@ if __name__ == "__main__":
         new_results = {}
         for obj_name, obj_res in results.items():
             new_res = {}
-            for fopt, opt_name in sorted(obj_res):
-                new_res.setdefault(opt_name, []).append(fopt)
-            for opt_name, fopts in new_res.items():
+            for fopt, opt_name, extra in sorted(obj_res):
+                l = new_res.setdefault(opt_name, [[], []])
+                l[0].append(fopt)
+                l[1].append(extra)
+            slices = {}
+            for opt_name, res in new_res.items():
                 # in the event that an odd number of results needs to be trimmed,
                 # prefer trimming from the bottom (i.e. worse solutions get removed first).
+                fopts, extras = res
                 down = (len(fopts) - multiple) // 2
                 up = len(fopts) - (len(fopts) - multiple + 1) // 2
-                # print("asdf", len(fopts), down, up)
-                new_res[opt_name] = fopts[down:up]
-            for opt_name, fopts in new_res.items():
+                slices[opt_name] = slice(down, up)
+            for opt_name, res in new_res.items():
+                fopts, extras = res
+                s = slices[opt_name]
+                fopts, extras = fopts[s], extras[s]
                 if not no_summary:
                     assert len(fopts) == multiple, (len(fopts), multiple)
                 if len(fopts) == multiple:
-                    for fopt in fopts:
-                        new_results.setdefault(obj_name, []).append((fopt, opt_name))
+                    for fopt, extra in zip(fopts, extras):
+                        result = (fopt, opt_name, extra)
+                        new_results.setdefault(obj_name, []).append(result)
         return results
 
     reset = "\033[m"
@@ -354,8 +361,8 @@ if __name__ == "__main__":
             run = 1
             while (cache := wrapped.cached(run)) is not None:
                 run += 1
-                fopt, xopt = cache
-                results.setdefault(obj_name, []).append((fopt, opt_name))
+                fopt, xopt, history = cache
+                results.setdefault(obj_name, []).append((fopt, opt_name, history))
 
             note = (lambda s: None) if quieter else m36
 
@@ -377,7 +384,8 @@ if __name__ == "__main__":
                 )
                 _ = optimizer(wrapped, n_trials=n_trials, n_dim=n_dim, with_count=False)
                 fopt, xopt = wrapped.finish()
-                results.setdefault(obj_name, []).append((fopt, opt_name))
+                result = (fopt, opt_name, wrapped.history)
+                results.setdefault(obj_name, []).append(result)
                 once = True
                 run += 1
 
@@ -391,7 +399,7 @@ if __name__ == "__main__":
             print()
             m1(f"{obj_name}:")
         all_res = {}
-        for fopt, opt_name in obj_res:
+        for fopt, opt_name, extra in obj_res:
             all_res.setdefault(fopt, []).append(opt_name)
             all_opt_names.add(opt_name)
             scores.setdefault(opt_name, 0.0)

utils.py

@@ -173,6 +173,16 @@ class AcquireForWriting:
 
 
 def perform_another_experimental_scoring_method(results):
+    if len(results) and len(something := next(iter(results.values()))[0]) == 3:
+        history_length = len(something[2])
+        each = {}
+        for i in range(history_length):
+            # for k, v in results.items(): for vi in v: assert len(vi) == 3, vi
+            l = {k: [(res[2][i], res[1]) for res in v] for k, v in results.items()}
+            for k, v in perform_another_experimental_scoring_method(l).items():
+                each.setdefault(k, []).append(v)
+        return {k: sum(v) / len(v) for k, v in each.items()}
+
     new_results = {}
     all_opt_names = set()
     for obj_name, obj_res in results.items():
@@ -203,7 +213,7 @@ def perform_another_experimental_scoring_method(results):
 
     scores = {k: 0.0 for k in all_opt_names}
     for opt_name, (rank, count) in best_ranks_and_counts.items():
-        points = 2**(1 - rank)
+        points = 2 ** (1 - rank)
         count = min(count, limited_by_floating_point_precision)
         scores[opt_name] = score = sum(points / 2**i for i in range(count))
 
@@ -211,17 +221,18 @@ def perform_another_experimental_scoring_method(results):
 
 
 def needs_rerun(key, value):
+    if value["duration"] < 0.0 or "history" not in value:
+        return True
+
+    if value["timestamp"] < 1683295630.0:  # bugged history field
+        return True
+
+    if not value["history"]:  # not sure what happened here
+        return True
+
     n_dim = len(value["xopt"])
     ng = []
 
-    if value["duration"] > -1683000000.0:
-        if n_dim == 3:
-            ng += "go_ackley01 go_alpine01 go_deflectedcorrugatedspring go_exponential go_griewank go_infinity go_mishra11 go_multimodal go_pinter go_qing go_rastrigin go_salomon go_schwefel01 go_schwefel20 go_schwefel21 go_schwefel22 go_sineenvelope go_sodp go_trigonometric02 go_wavy go_weierstrass go_xinsheyang01 go_xinsheyang02 go_xinsheyang03 go_yaoliu04 go_yaoliu09 go_zerosum".split()
-            ng += "go_amgm go_brown go_cigar go_deb01 go_deb03 go_deceptive go_needleeye go_penalty01 go_schwefel04 go_schwefel26 go_sphere go_stretchedv go_styblinskitang go_zacharov".split()
-        if n_dim == 4:
-            ng += "go_ackley01 go_alpine01 go_cigar go_exponential go_griewank go_infinity go_pinter go_qing go_schwefel01 go_schwefel02 go_schwefel20 go_schwefel21 go_schwefel22 go_sineenvelope go_sphere go_step2 go_step go_stretchedv go_trigonometric02 go_weierstrass go_whitley go_xinsheyang01 go_xinsheyang02 go_yaoliu04 go_yaoliu09 go_zerosum".split()
-            ng += "go_amgm go_brown go_colville go_deb03 go_penalty01 go_penalty02 go_powell go_rosenbrock go_schwefel04 go_shekel05 go_shekel07 go_shekel10 go_shubert03 go_shubert04 go_styblinskitang go_vincent".split()
-
     kd = decode_key(key)
     assert kd is not None, key
     if kd.obj in ng:

utils_np.py

@@ -1,4 +1,5 @@
 # i've separated numpy-dependent methods from the rest of the utils.
+from time import time
 from utils import AcquireForWriting, merge_summaries, feps, m33, m34, m93
 import numpy as np
 
@@ -60,7 +61,14 @@ def do_bounding(x, method="clip"):
 
 
 class OWrap:
-    def __init__(self, objective, n_trials, frugal_percent=1.0, greedy_percent=2.0):
+    def __init__(
+        self,
+        objective,
+        n_trials,
+        frugal_percent=1.0,
+        greedy_percent=2.0,
+        history_frequency=10,
+    ):
         self.feval_count = 0
         self.best_so_far = None
         self.warning = None
@@ -69,6 +77,8 @@ class OWrap:
         self.__name__ = objective.__name__  # for evolopy
         self.frugal_percent = float(frugal_percent)
         self.greedy_percent = float(greedy_percent)
+        self.history_frequency = history_frequency
+        self.history = []
 
     def __str__(self):
         return (
@@ -106,6 +116,10 @@ class OWrap:
             if self.best_so_far is None or fx < self.best_so_far[0]:
                 self.best_so_far = (fx, x)
 
+            if self.history_frequency > 0:
+                if self.feval_count % self.history_frequency == 0:
+                    self.history.append(self.best_so_far[0])
+
         return float(fx)
 
     def finish(self, optimizer_name):
@@ -137,6 +151,7 @@ class COWrap:
         self.n_dim = n_dim
         self.kwargs = kwargs
         self._dirty = False
+        self._history = None
 
         from pathlib import Path
 
@@ -157,6 +172,8 @@ class COWrap:
 
     def __call__(self, x, *args, **kwargs):
         assert not self._ran, "please run .finish() before continuing!"
+        if not self._dirty:
+            self.start_time = time()
         result = self.ow.__call__(x, *args, **kwargs)
         self._dirty = True
         return result
@@ -176,6 +193,12 @@ class COWrap:
         opt_name = self.optimizer.__name__
         return f"COWrap_d{self.n_dim:02}_n{self.n_trials:03}_{opt_name}"
 
+    @property
+    def history(self):
+        assert not self._dirty
+        assert self._history
+        return self._history
+
     @property
     def cache_key(self):
         opt_name = self.optimizer.__name__
@@ -237,15 +260,16 @@ class COWrap:
             return None
         assert "fopt" in summary, summary
         assert "xopt" in summary, summary
-        assert "duration" in summary, summary
+        assert "timestamp" in summary, summary
+        assert "history" in summary, summary
         fopt = float(summary["fopt"])
         xopt = np.array(summary["xopt"], np.float64)
-        duration = float(summary["duration"])
-        return fopt, xopt
+        history = [fval for fval in summary["history"]]
+        assert history, "history cannot be empty"  # this should get filtered now
+        return fopt, xopt, history
 
     def finish(self, opt_name=None):
         from json import dumps
-        from time import time
 
         assert self._dirty
         self._ran = True
@@ -255,12 +279,23 @@ class COWrap:
         assert self.ow.best_so_far is not None
         # fopt, xopt = self.ow.best_so_far
         fopt, xopt = self.ow.finish(self.optimizer.__name__)
+
+        expected_length = self.n_trials // self.ow.history_frequency
+        history = [float(fval) for fval in self.ow.history]
+        history += [fopt] * (expected_length - len(history))
+
+        finish_time = time()
         summary = dict(
             fopt=float(fopt),
             xopt=[float(x) for x in xopt],
-            # duration=float(-1),  # old, bad for uniqueness
-            duration=float(-time()),
+            # timestamp=float(-1),  # old, bad for uniqueness
+            timestamp=finish_time,
+            # optional: (for now?)
+            history=history,
+            duration=finish_time - self.start_time,
         )
+        self._history = summary["history"]
+        assert self._history, "why"
 
         with AcquireForWriting(self.cache_file) as fp:
             self._cached_summaries = None  # force reload