Download code/src/robottt_pi05/eval_io.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 5.1 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/eval_io.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/src/robottt_pi05/eval_io.py
-
curl -L -o eval_io.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/eval_io.py
5.1 kB
| """Atomic evaluation outputs and cumulative write-ahead counters. No model imports.""" | |
| from pathlib import Path | |
| import hashlib, json, os, time | |
| import torch | |
| def sha(path): | |
| h=hashlib.sha256() | |
| with Path(path).open('rb') as f: | |
| for b in iter(lambda:f.read(1048576),b''):h.update(b) | |
| return h.hexdigest() | |
| def tensor_hash(x): | |
| h=hashlib.sha256();h.update(str(tuple(x.shape)).encode());h.update(str(x.dtype).encode()) | |
| h.update(x.detach().reshape(-1).contiguous().view(torch.uint8).cpu().numpy().tobytes()) | |
| return h.hexdigest() | |
| def parameter_hash(state): | |
| h=hashlib.sha256() | |
| for n,v in state.items(): | |
| h.update(n.encode());h.update(v.detach().reshape(-1).contiguous().view(torch.uint8).cpu().numpy().tobytes()) | |
| return h.hexdigest() | |
| def atomic_json(path,value,*,replace=False): | |
| path=Path(path);path.parent.mkdir(parents=True,exist_ok=True) | |
| if path.exists() and not replace:raise FileExistsError(path) | |
| tmp=path.with_name(path.name+f'.partial.{time.time_ns()}') | |
| with tmp.open('x') as f:json.dump(value,f,indent=2,allow_nan=False);f.write('\n');f.flush();os.fsync(f.fileno()) | |
| os.replace(tmp,path) | |
| def atomic_tensor(path,value): | |
| path=Path(path);path.parent.mkdir(parents=True,exist_ok=True) | |
| if path.exists():raise FileExistsError(path) | |
| tmp=path.with_name(path.name+f'.partial.{time.time_ns()}') | |
| with tmp.open('xb') as f:torch.save(value,f);f.flush();os.fsync(f.fileno()) | |
| # Verify a complete CPU load before making it the committed output. | |
| torch.load(tmp,map_location='cpu',weights_only=True) | |
| os.replace(tmp,path) | |
| return sha(path) | |
| def save_prediction(*,path,key,prediction,metadata,noise_hash,incoming_hash,candidate,solver_records): | |
| assert prediction.device.type=='cpu' and prediction.shape==(1,10,32) | |
| assert prediction.dtype==torch.float32 and torch.isfinite(prediction).all() | |
| assert not prediction.requires_grad and prediction.grad_fn is None | |
| value={'key':key,'prediction':prediction.clone(),'metadata':metadata,'noise_hash':noise_hash, | |
| 'incoming_hash':incoming_hash,'candidate':candidate,'solver_records':solver_records, | |
| 'query_target_free':True,'space':'internal_normalized_action'} | |
| return atomic_tensor(path,value) | |
| def load_prediction(*,path,key,noise_hash,manifest_hash): | |
| value=torch.load(path,map_location='cpu',weights_only=True) | |
| assert value['key']==key and value['noise_hash']==noise_hash | |
| assert value['metadata']['execution_manifest_sha256']==manifest_hash | |
| assert value['prediction'].shape==(1,10,32) and torch.isfinite(value['prediction']).all() | |
| return value | |
| class Ledger: | |
| def __init__(self,root,limits): | |
| self.root=Path(root);self.root.mkdir(parents=True,exist_ok=True);self.limits=limits | |
| self.events=self.root/'events.jsonl';self.attempted={k:0 for k in limits};self.completed={k:0 for k in limits} | |
| self.outputs={} | |
| if self.events.exists(): | |
| for line in self.events.read_text().splitlines(): | |
| r=json.loads(line) | |
| if r['event']=='reserve':self.attempted[r['kind']]+=1 | |
| if r['event']=='complete':self.completed[r['kind']]+=1 | |
| if r['event']=='output_complete':self.outputs[r['key']]=r | |
| assert all(self.completed[k]<=self.attempted[k]<=limits[k] for k in limits) | |
| def event(self,event,**fields): | |
| if {'event','epoch','pid'} & set(fields):raise ValueError('Reserved event fields') | |
| serialized=json.dumps({'event':event,'epoch':time.time(),'pid':os.getpid(),**fields},allow_nan=False) | |
| with self.events.open('a') as f: | |
| f.write(serialized+'\n');f.flush() | |
| readonly=event in ('reserve','complete') and fields.get('kind') in ('expert','inner','backward') | |
| if not (getattr(self,'defer_readonly_fsync',False) and readonly):os.fsync(f.fileno()) | |
| def reserve(self,kind,scope): | |
| if self.attempted[kind]>=self.limits[kind]:raise RuntimeError(f'Budget exhausted: {kind}') | |
| self.event('reserve',kind=kind,scope=scope);self.attempted[kind]+=1 | |
| def complete(self,kind,scope): | |
| assert self.completed[kind]<self.attempted[kind] | |
| self.event('complete',kind=kind,scope=scope);self.completed[kind]+=1 | |
| def output_complete(self,*,key,path): | |
| rec={'key':key,'path':str(path),'sha256':sha(path)} | |
| self.event('output_complete',**rec);self.outputs[key]=rec;self.save(key) | |
| def save(self,phase): | |
| atomic_json(self.root/'progress.json',{'phase':phase,'pid':os.getpid(),'attempted':self.attempted, | |
| 'completed':self.completed,'limits':self.limits,'output_count':len(self.outputs),'epoch':time.time()},replace=True) | |
| def state_hash(s): | |
| return parameter_hash({**{str(i):w for i,w in enumerate(s['weights'])},'updates':s['updates']}) | |
| def completed_output(ledger,key,path): | |
| if key not in ledger.outputs:return False | |
| rec=ledger.outputs[key] | |
| if str(path)!=rec['path'] or not Path(path).is_file() or sha(path)!=rec['sha256']: | |
| raise ValueError('Completed artifact changed or missing') | |
| return True | |