from dataclasses import dataclass @dataclass class KeyData: key: str d: int n: int opt: str obj: str run: int def decode_key(key, _filtering=False): # example key: # COWrap_d03_n130_freelunch_qpso_ps16_cube_go_amgm_on_cube k, _, run = key.partition("[") run, _, _ = run.partition("]") k = k.removeprefix("COWrap_") d, _, k = k.partition("_") n, _, k = k.partition("_") opt, _, k = k.partition("_cube_") obj, _, k = k.partition("_on_cube") if not obj: if opt.endswith("_on_cube"): return # fcmaes_biteopt was missing the _cube in its name for a while if _filtering and obj in ("go_stochastic", "go_xinsheyang01"): return # these are random assert not k, k d = int(d.removeprefix("d"), 10) n = int(n.removeprefix("n"), 10) run = int(run, 10) return KeyData(key=key, d=d, n=n, opt=opt, obj=obj, run=run) class AcquireForWriting: """ A context manager that allows for very crude file-locking-like functionality when the FileLock module is missing. """ def __init__(self, filepath, usingfilelock=None): from pathlib import Path self.filepath = Path(filepath) if usingfilelock is None: try: from filelock import FileLock except ModuleNotFoundError: self._locking = False self.lock = None else: self._locking = True self.lock = FileLock(self._altpath) elif usingfilelock: from filelock import FileLock self._locking = True self.lock = FileLock(self._altpath) else: self._locking = False self.lock = None @property def _altpath(self): suffix = ".lock" if self._locking else "~" return self.filepath.with_suffix(self.filepath.suffix + suffix) def __enter__(self): if self._locking: self.lock.__enter__() else: from time import sleep for _ in range(3): if self._altpath.exists(): sleep(1) assert not self._altpath.exists(), f"file is locked: {self.filepath}" if not self._locking: self._altpath.write_bytes(b"") return self.filepath if self._locking else self._altpath def __exit__(self, *exc): if self._locking: self.lock.__exit__(*exc) elif exc == (None, None, None): assert self._altpath.exists(), f"file went missing: {self.filepath}" try: data = self._altpath.read_bytes() if data: self.filepath.write_bytes(data) finally: self._altpath.unlink() # from shutil import move # move(self._altpath, self.filepath) # assumes os.rename overwrites files def needs_rerun(key, value): if value["duration"] < 0.0 or "history" not in value: return True if value["timestamp"] < 1683295630.0: # bugged history field return True if not value["history"]: # not sure what happened here return True size = len(value["xopt"]) ng = [] kd = decode_key(key) assert kd is not None, key if kd.obj in ng: # print("filtered", key, file=__import__("sys").stderr) return True return False def merge_summaries(all_summaries): # i only needed to write this because i effed up my filenames at one point. oh well. if len(all_summaries) == 0: return {} elif len(all_summaries) == 1: return {k: v for k, v in all_summaries[0].items() if not needs_rerun(k, v)} new_summaries = {} for s in all_summaries: for key, value in s.items(): if needs_rerun(key, value): continue k, _, run = key.partition("[") run, _, _ = run.partition("]") for i in range(1, 100): new_key = f"{k}[{i}]" if new_key in new_summaries: if new_summaries[new_key] == value: # this works 'cause it's POD break # already exists (probably; duration is fucked) continue new_summaries[new_key] = value break return new_summaries