jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
5.1 kB
"""Atomic evaluation outputs and cumulative write-ahead counters. No model imports."""
from pathlib import Path
import hashlib, json, os, time
import torch
def sha(path):
h=hashlib.sha256()
with Path(path).open('rb') as f:
for b in iter(lambda:f.read(1048576),b''):h.update(b)
return h.hexdigest()
def tensor_hash(x):
h=hashlib.sha256();h.update(str(tuple(x.shape)).encode());h.update(str(x.dtype).encode())
h.update(x.detach().reshape(-1).contiguous().view(torch.uint8).cpu().numpy().tobytes())
return h.hexdigest()
def parameter_hash(state):
h=hashlib.sha256()
for n,v in state.items():
h.update(n.encode());h.update(v.detach().reshape(-1).contiguous().view(torch.uint8).cpu().numpy().tobytes())
return h.hexdigest()
def atomic_json(path,value,*,replace=False):
path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
if path.exists() and not replace:raise FileExistsError(path)
tmp=path.with_name(path.name+f'.partial.{time.time_ns()}')
with tmp.open('x') as f:json.dump(value,f,indent=2,allow_nan=False);f.write('\n');f.flush();os.fsync(f.fileno())
os.replace(tmp,path)
def atomic_tensor(path,value):
path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
if path.exists():raise FileExistsError(path)
tmp=path.with_name(path.name+f'.partial.{time.time_ns()}')
with tmp.open('xb') as f:torch.save(value,f);f.flush();os.fsync(f.fileno())
# Verify a complete CPU load before making it the committed output.
torch.load(tmp,map_location='cpu',weights_only=True)
os.replace(tmp,path)
return sha(path)
def save_prediction(*,path,key,prediction,metadata,noise_hash,incoming_hash,candidate,solver_records):
assert prediction.device.type=='cpu' and prediction.shape==(1,10,32)
assert prediction.dtype==torch.float32 and torch.isfinite(prediction).all()
assert not prediction.requires_grad and prediction.grad_fn is None
value={'key':key,'prediction':prediction.clone(),'metadata':metadata,'noise_hash':noise_hash,
'incoming_hash':incoming_hash,'candidate':candidate,'solver_records':solver_records,
'query_target_free':True,'space':'internal_normalized_action'}
return atomic_tensor(path,value)
def load_prediction(*,path,key,noise_hash,manifest_hash):
value=torch.load(path,map_location='cpu',weights_only=True)
assert value['key']==key and value['noise_hash']==noise_hash
assert value['metadata']['execution_manifest_sha256']==manifest_hash
assert value['prediction'].shape==(1,10,32) and torch.isfinite(value['prediction']).all()
return value
class Ledger:
def __init__(self,root,limits):
self.root=Path(root);self.root.mkdir(parents=True,exist_ok=True);self.limits=limits
self.events=self.root/'events.jsonl';self.attempted={k:0 for k in limits};self.completed={k:0 for k in limits}
self.outputs={}
if self.events.exists():
for line in self.events.read_text().splitlines():
r=json.loads(line)
if r['event']=='reserve':self.attempted[r['kind']]+=1
if r['event']=='complete':self.completed[r['kind']]+=1
if r['event']=='output_complete':self.outputs[r['key']]=r
assert all(self.completed[k]<=self.attempted[k]<=limits[k] for k in limits)
def event(self,event,**fields):
if {'event','epoch','pid'} & set(fields):raise ValueError('Reserved event fields')
serialized=json.dumps({'event':event,'epoch':time.time(),'pid':os.getpid(),**fields},allow_nan=False)
with self.events.open('a') as f:
f.write(serialized+'\n');f.flush()
readonly=event in ('reserve','complete') and fields.get('kind') in ('expert','inner','backward')
if not (getattr(self,'defer_readonly_fsync',False) and readonly):os.fsync(f.fileno())
def reserve(self,kind,scope):
if self.attempted[kind]>=self.limits[kind]:raise RuntimeError(f'Budget exhausted: {kind}')
self.event('reserve',kind=kind,scope=scope);self.attempted[kind]+=1
def complete(self,kind,scope):
assert self.completed[kind]<self.attempted[kind]
self.event('complete',kind=kind,scope=scope);self.completed[kind]+=1
def output_complete(self,*,key,path):
rec={'key':key,'path':str(path),'sha256':sha(path)}
self.event('output_complete',**rec);self.outputs[key]=rec;self.save(key)
def save(self,phase):
atomic_json(self.root/'progress.json',{'phase':phase,'pid':os.getpid(),'attempted':self.attempted,
'completed':self.completed,'limits':self.limits,'output_count':len(self.outputs),'epoch':time.time()},replace=True)
def state_hash(s):
return parameter_hash({**{str(i):w for i,w in enumerate(s['weights'])},'updates':s['updates']})
def completed_output(ledger,key,path):
if key not in ledger.outputs:return False
rec=ledger.outputs[key]
if str(path)!=rec['path'] or not Path(path).is_file() or sha(path)!=rec['sha256']:
raise ValueError('Completed artifact changed or missing')
return True