142 lines
4.3 KiB
Python
142 lines
4.3 KiB
Python
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass
|
|
class KeyData:
|
|
key: str
|
|
d: int
|
|
n: int
|
|
opt: str
|
|
obj: str
|
|
run: int
|
|
|
|
|
|
def decode_key(key, _filtering=False):
|
|
# example key:
|
|
# COWrap_d03_n130_freelunch_qpso_ps16_cube_go_amgm_on_cube
|
|
k, _, run = key.partition("[")
|
|
run, _, _ = run.partition("]")
|
|
k = k.removeprefix("COWrap_")
|
|
d, _, k = k.partition("_")
|
|
n, _, k = k.partition("_")
|
|
opt, _, k = k.partition("_cube_")
|
|
obj, _, k = k.partition("_on_cube")
|
|
if not obj:
|
|
if opt.endswith("_on_cube"):
|
|
return # fcmaes_biteopt was missing the _cube in its name for a while
|
|
if _filtering and obj in ("go_stochastic", "go_xinsheyang01"):
|
|
return # these are random
|
|
assert not k, k
|
|
d = int(d.removeprefix("d"), 10)
|
|
n = int(n.removeprefix("n"), 10)
|
|
run = int(run, 10)
|
|
return KeyData(key=key, d=d, n=n, opt=opt, obj=obj, run=run)
|
|
|
|
|
|
class AcquireForWriting:
|
|
"""
|
|
A context manager that allows for very crude file-locking-like
|
|
functionality when the FileLock module is missing.
|
|
"""
|
|
|
|
def __init__(self, filepath, usingfilelock=None):
|
|
from pathlib import Path
|
|
|
|
self.filepath = Path(filepath)
|
|
if usingfilelock is None:
|
|
try:
|
|
from filelock import FileLock
|
|
except ModuleNotFoundError:
|
|
self._locking = False
|
|
self.lock = None
|
|
else:
|
|
self._locking = True
|
|
self.lock = FileLock(self._altpath)
|
|
elif usingfilelock:
|
|
from filelock import FileLock
|
|
|
|
self._locking = True
|
|
self.lock = FileLock(self._altpath)
|
|
else:
|
|
self._locking = False
|
|
self.lock = None
|
|
|
|
@property
|
|
def _altpath(self):
|
|
suffix = ".lock" if self._locking else "~"
|
|
return self.filepath.with_suffix(self.filepath.suffix + suffix)
|
|
|
|
def __enter__(self):
|
|
if self._locking:
|
|
self.lock.__enter__()
|
|
else:
|
|
from time import sleep
|
|
|
|
for _ in range(3):
|
|
if self._altpath.exists():
|
|
sleep(1)
|
|
assert not self._altpath.exists(), f"file is locked: {self.filepath}"
|
|
if not self._locking:
|
|
self._altpath.write_bytes(b"")
|
|
return self.filepath if self._locking else self._altpath
|
|
|
|
def __exit__(self, *exc):
|
|
if self._locking:
|
|
self.lock.__exit__(*exc)
|
|
elif exc == (None, None, None):
|
|
assert self._altpath.exists(), f"file went missing: {self.filepath}"
|
|
try:
|
|
data = self._altpath.read_bytes()
|
|
if data:
|
|
self.filepath.write_bytes(data)
|
|
finally:
|
|
self._altpath.unlink()
|
|
# from shutil import move
|
|
# move(self._altpath, self.filepath) # assumes os.rename overwrites files
|
|
|
|
|
|
def needs_rerun(key, value):
|
|
if value["duration"] < 0.0 or "history" not in value:
|
|
return True
|
|
|
|
if value["timestamp"] < 1683295630.0: # bugged history field
|
|
return True
|
|
|
|
if not value["history"]: # not sure what happened here
|
|
return True
|
|
|
|
size = len(value["xopt"])
|
|
ng = []
|
|
|
|
kd = decode_key(key)
|
|
assert kd is not None, key
|
|
if kd.obj in ng:
|
|
# print("filtered", key, file=__import__("sys").stderr)
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def merge_summaries(all_summaries):
|
|
# i only needed to write this because i effed up my filenames at one point. oh well.
|
|
if len(all_summaries) == 0:
|
|
return {}
|
|
elif len(all_summaries) == 1:
|
|
return {k: v for k, v in all_summaries[0].items() if not needs_rerun(k, v)}
|
|
new_summaries = {}
|
|
for s in all_summaries:
|
|
for key, value in s.items():
|
|
if needs_rerun(key, value):
|
|
continue
|
|
k, _, run = key.partition("[")
|
|
run, _, _ = run.partition("]")
|
|
for i in range(1, 100):
|
|
new_key = f"{k}[{i}]"
|
|
if new_key in new_summaries:
|
|
if new_summaries[new_key] == value: # this works 'cause it's POD
|
|
break # already exists (probably; duration is fucked)
|
|
continue
|
|
new_summaries[new_key] = value
|
|
break
|
|
return new_summaries
|