File size: 6,949 Bytes
07a447a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Audited rollback to a saved boundary after interruption; never use on a live run.

Only GPU identity, model-load budget, and checkpoint provenance metadata change.
The numerical state and seeded inputs are verified and preserved exactly.
"""
import argparse,fcntl,json,os,pathlib,shutil,subprocess,time
import torch
from robottt_pi05.config import ExperimentConfig
from robottt_pi05.checkpoint import digest,fsync_dir
from robottt_pi05.eval_io import sha,atomic_json,atomic_tensor
from robottt_pi05.profile_common import runtime_signature


def trim_events(lines,saved):
    attempted={k:0 for k in saved['attempted']};completed={k:0 for k in saved['completed']};kept=[]
    for index,line in enumerate(lines):
        try:record=json.loads(line)
        except json.JSONDecodeError:
            if index==len(lines)-1:break
            raise ValueError('Corrupt journal before its final line')
        scope=record.get('scope');step=record.get('step')
        if (isinstance(scope,str) and scope.isdigit() and int(scope)>saved['step']) or (isinstance(step,int) and step>saved['step']):break
        if record['event'] in ('reserve','complete'):
            counters=attempted if record['event']=='reserve' else completed
            target=saved['attempted'] if record['event']=='reserve' else saved['completed']
            kind=record['kind']
            if counters[kind]+1>target[kind]:break
            counters[kind]+=1
        kept.append(line)
    if attempted!=saved['attempted']:raise ValueError('Saved boundary and journal attempts differ')
    for kind,count in completed.items():
        missing=saved['completed'][kind]-count
        if missing and not (kind=='optimizer_update' and missing==1):raise ValueError('Saved boundary and journal completions differ')
    return ''.join(line if line.endswith('\n') else line+'\n' for line in kept)


def numerical_digest(saved):
    return digest({key:saved[key] for key in ('step','next_schedule_index','slow','optimizer','rng','config','parameter_names')})


def restore_transaction(transaction):
    record=json.loads(transaction.read_text())
    if record['state']=='committed':return
    archive=pathlib.Path(record['archive'])
    for name,target in record['originals'].items():
        target=pathlib.Path(target);temporary=target.with_name(target.name+'.rollback')
        if temporary.exists():temporary.unlink()
        if name=='checkpoint.pt':os.link(archive/name,temporary)
        else:shutil.copyfile(archive/name,temporary)
        os.replace(temporary,target);fsync_dir(target.parent)
    record['state']='rolled_back';atomic_json(transaction,record,replace=True)


def recover(run_config):
    rp=pathlib.Path(run_config).resolve();r=json.loads(rp.read_text())
    def path(key):return (rp.parent/r['paths'][key]).resolve()
    out=path('output');out.mkdir(parents=True,exist_ok=True)
    with (out/'run.lock').open('a') as lock:
        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
        transaction=out/'recovery-transaction.json'
        if transaction.exists():restore_transaction(transaction)
        # A rolled-back interrupted transaction may have restored the old config.
        r=json.loads(rp.read_text())
        if (out/'completed.json').exists():return dict(state='already_completed',run=str(rp))
        if list(out.glob('failure*.json')):raise RuntimeError('Recorded training failure requires diagnosis before rollback')
        checkpoint=out/'checkpoints/latest.pt';journal=out/'ledger/events.jsonl'
        if not checkpoint.exists():raise FileNotFoundError('No saved boundary; restore a verified HF checkpoint first')
        saved=torch.load(checkpoint,map_location='cpu',weights_only=True)
        old_provenance=dict(experiment_config=sha(path('experiment_config')),asset_index=sha(path('asset_index')),run_config=sha(rp),implementation_sha256=runtime_signature())
        if saved['provenance']!=old_provenance:raise ValueError('Original checkpoint/config/code provenance differs')
        if saved['config']!=ExperimentConfig.load(path('experiment_config')).to_dict():raise ValueError('Numerical experiment configuration differs')
        for key,expected in r['hashes'].items():
            if sha(path(key))!=expected:raise ValueError('Immutable input hash differs: '+key)
        trimmed=trim_events(journal.read_text().splitlines(keepends=True),saved)
        gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name,memory.total,driver_version','--format=csv,noheader'],text=True).strip()
        parts=[item.strip() for item in gpu.split(',')]
        if parts[1]!='NVIDIA GeForce RTX 3090':raise ValueError('Unvalidated replacement GPU model')
        archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True)
        originals={'config.json':str(rp),'checkpoint.pt':str(checkpoint),'events.jsonl':str(journal)}
        shutil.copyfile(rp,archive/'config.json');os.link(checkpoint,archive/'checkpoint.pt');shutil.copyfile(journal,archive/'events.jsonl')
        old_hash=sha(checkpoint);before=numerical_digest(saved)
        replacement=json.loads(json.dumps(r));replacement['gpu_uuid']=parts[0]
        replacement['limits']['model_load']=max(1024,replacement['limits']['model_load'])
        new_config=archive/'config-next.json';new_config.write_text(json.dumps(replacement,indent=2)+'\n')
        updated=dict(saved);updated['provenance']=dict(saved['provenance'],run_config=sha(new_config))
        new_checkpoint=archive/'checkpoint-next.pt';atomic_tensor(new_checkpoint,updated)
        if numerical_digest(torch.load(new_checkpoint,map_location='cpu',weights_only=True))!=before:raise ValueError('Recovery altered numerical state')
        new_journal=archive/'events-next.jsonl';new_journal.write_text(trimmed)
        record=dict(state='prepared',archive=str(archive),originals=originals,source_checkpoint_sha256=old_hash,
            migrated_checkpoint_sha256=sha(new_checkpoint),preserved_numerical_state_sha256=before,step=saved['step'],gpu=gpu,
            discarded_unsaved_journal_bytes=journal.stat().st_size-len(trimmed.encode()),reason='Workspace interruption: resume the saved optimizer/RNG boundary; administrative GPU/budget metadata only')
        atomic_json(transaction,record,replace=transaction.exists())
        for source,target in ((new_config,rp),(new_checkpoint,checkpoint),(new_journal,journal)):
            temporary=target.with_name(target.name+'.recovery')
            if temporary.exists():temporary.unlink()
            os.link(source,temporary);os.replace(temporary,target);fsync_dir(target.parent)
        record['state']='committed';atomic_json(transaction,record,replace=True)
        atomic_json(archive/'recovery.json',record)
        return record


if __name__=='__main__':
    parser=argparse.ArgumentParser();parser.add_argument('--run-config',required=True);args=parser.parse_args()
    print(json.dumps(recover(args.run_config),indent=2))