jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
5.18 kB
"""Optional four-chunk fixed/learned carry/reset audit using supplied saved artifacts."""
from pathlib import Path
import fcntl,json
import torch
from .config import ExperimentConfig
from .modeling import create_adapter,configure_trainables
from .checkpoint import load_weights
from .openpi_backend import load_model
from .engine import SafetyGuard,generate_chunk
from .assets import prepared_prefix,prefix_signature,prefix_unchanged
from .layer_state import packed,unpacked
from .eval_io import sha,state_hash,parameter_hash,tensor_hash,atomic_tensor,atomic_json
from .training_io import RunLedger
from .runner import _gpu_preflight
def run(manifest_path):
manifest_path=Path(manifest_path).resolve();m=json.loads(manifest_path.read_text());base=manifest_path.parent
def asset(record):
p=base/record['path']
if sha(p)!=record['sha256']:raise ValueError('Regression artifact hash mismatch')
return p
out=base/m['output'];out.mkdir(parents=True,exist_ok=True)
with (out/'regression.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (out/'completed.json').exists():
z=json.loads((out/'completed.json').read_text())
if z['manifest_sha256']!=sha(manifest_path):raise ValueError('Completed regression manifest mismatch')
return z
if [x['key'] for x in m['cases']]!=['FIX_carry','FIX_reset','LEARN_carry','LEARN_reset']:raise ValueError('Exactly four fixed regression cases required')
prefix_path=asset(m['prefix']);bank_path=asset(m['bank'])
for c in m['cases']:
asset(c['checkpoint']);asset(c['expected'])
if c['mode']=='carry':asset(c['incoming'])
ledger=RunLedger(out/'ledger',dict(model_load=1,rollout=4,expert=40,inner=160,optimizer_update=0,backward=0,prefix=0))
if any(ledger.attempted.values()):raise RuntimeError('Interrupted regression: inspect artifacts/budget before recovery; do not auto replay')
_gpu_preflight(m['gpu_uuid']);ledger.reserve('model_load','regression')
model=load_model(source=base/m['openpi_source'],checkpoint=asset(m['model_checkpoint']));ledger.complete('model_load','regression')
guard=SafetyGuard();scope=['reference']
def observer(kind,phase):getattr(ledger,'reserve' if phase=='attempt' else 'complete')(kind,scope[0])
ad,slow=create_adapter(model,ExperimentConfig(),observer=observer,guard=guard,inference=True)
z=torch.load(prefix_path,map_location='cpu',weights_only=True)
if z['provenance']!=m['prefix']['provenance']:raise ValueError('Reference prefix provenance')
prepared=prepared_prefix(z,'cuda:0');signature=prefix_signature([prepared])
noise=torch.load(bank_path,map_location='cpu',weights_only=True)['evaluation_noise'][0,:,15].cuda()
if tensor_hash(noise)!=m['noise_chunk_sha256']:raise ValueError('Reference noise indexing')
results={};frozen=[(p,p._version) for p in model.parameters()]
for c in m['cases']:
scope[0]=c['key'];cfg=ExperimentConfig(learn_inner_lr=c['key'].startswith('LEARN'))
configure_trainables(slow,cfg,inference=True);load_weights(asset(c['checkpoint']),slow,cfg,expected_sha256=c['checkpoint']['sha256'])
expected=torch.load(asset(c['expected']),map_location='cpu',weights_only=True)
if expected['checkpoint_hash']!=c['checkpoint']['sha256']:raise ValueError('Reference checkpoint association')
state=ad.initial_state() if c['mode']=='reset' else unpacked(torch.load(asset(c['incoming']),map_location='cpu',weights_only=True)['candidate'],device='cuda')
incoming={k:state_hash(v) for k,v in packed(state).items()};slow_hash=parameter_hash(slow.state_dict())
if incoming!=expected['incoming_hash']:raise ValueError('Reference incoming mismatch')
ledger.reserve('rollout',scope[0]);prediction,candidate,solver=generate_chunk(ad,prepared,noise,state)
checks={'prediction_exact':torch.equal(prediction.cpu(),expected['prediction']),
'candidate_exact':all(state_hash(v)==state_hash(expected['candidate'][k]) for k,v in packed(candidate).items()),
'solver_exact':[x['candidate_hash'] for x in solver['calls']]==[x['candidate_hash'] for x in expected['solver']['calls']],
'prefix_unchanged':prefix_unchanged(signature),'slow_unchanged':parameter_hash(slow.state_dict())==slow_hash,
'base_unchanged':all(p._version==v and p.grad is None for p,v in frozen)}
atomic_tensor(out/(c['key']+'.pt'),{'checks':checks,'prediction':prediction.cpu(),'candidate':packed(candidate)})
ledger.complete('rollout',scope[0])
if not all(checks.values()):raise RuntimeError('Exact regression failed; no tolerance changes permitted')
results[c['key']]=checks;ad.history.clear();ad.call_contracts.clear()
result={'manifest_sha256':sha(manifest_path),'results':results,'attempted':ledger.attempted,'completed':ledger.completed,'peak_allocated':torch.cuda.max_memory_allocated()}
atomic_json(out/'completed.json',result);return result