File size: 8,784 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""Create an explicit changed-topology/data continuation; never edit its parent.

Preserves model, Adam moments/steps and the global/flow draw count. The schedule
is reparameterized to start at the saved LR and end at the same absolute floor,
on the extended budget. This is NOT bit-exact replay with the old data/world.
"""
import argparse
import copy
import json
import math
from pathlib import Path
import shutil
import sys

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'src'))


def extended_schedule(saved, *, step, old_world, new_world, warmup, total, floor):
    if not (0 < new_world <= old_world and warmup < step < total):
        raise ValueError('Expected a post-warmup, extended, non-increasing-world continuation')
    if saved['last_epoch'] != step * old_world:
        raise ValueError('Scheduler clock does not match checkpoint step/world')
    old_base = saved['base_lrs']
    old_lr = saved['_last_lr']
    phase = (step-warmup)/(total-warmup)
    cosine = .5 * (1 + math.cos(math.pi * phase))
    new_base = [b*floor + (lr-b*floor)/cosine for b,lr in zip(old_base,old_lr)]
    ratios = [b*floor/n for b,n in zip(old_base,new_base)]
    if not new_base or any(not math.isfinite(n) or n <= 0 for n in new_base):
        raise ValueError('Invalid LR state')
    if any(not math.isclose(r,ratios[0],rel_tol=1e-8) for r in ratios):
        raise ValueError('Parameter groups do not share a common cosine schedule')
    if any(lr < b*floor for b,lr in zip(old_base,old_lr)):
        raise ValueError('Saved LR is below the requested old floor')
    result = copy.deepcopy(saved)
    result.update(base_lrs=new_base,last_epoch=step*new_world,_step_count=step*new_world+1)
    return result, ratios[0]


def prepare(source, destination, *, world, batch, total, timestamp, output):
    import torch
    import tyro
    from psi.dlc_storage import digest, ready, copy_verified, atomic_json
    from psi.hfd_training_launcher import saved_options, to_args, CONFIG_MODULE
    from psi.config.hfd_current_recipe import FIELDS
    from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig
    source=Path(source).resolve(); destination=Path(destination).resolve()
    parent=source.parent.parent
    cfg=json.loads((parent/'run_config.json').read_text())
    marker=json.loads((source/'full_duplex_source_boundary.json').read_text())
    old_world=len(list(source.glob('random_states_*.pkl')))
    step=int(source.name.removeprefix('ckpt_'))
    if destination.exists() or source.is_relative_to(destination) or destination.is_relative_to(parent):
        raise ValueError('Destination must be new and outside the original run')
    if cfg['train']['stage'] != 0 or marker.get('stage2_data_cursor') is not None:
        raise ValueError('Only cache-boundary joint stage=0 is supported')
    if not ready(source,old_world):
        raise ValueError('Incomplete source checkpoint')
    backup=json.loads((source/'backup_complete.json').read_text())
    for name,proof in backup['files'].items():
        if Path(name).name != name:
            raise ValueError('Invalid backup filename')
        p=source/name
        if p.stat().st_size != proof['bytes'] or digest(p) != proof['sha256']:
            raise ValueError(f'Corrupt source checkpoint: {name}')
    if batch < 1 or cfg['train']['lr_scheduler_type'] != 'cosine_with_min_lr':
        raise ValueError('Expected positive batch and cosine_with_min_lr')
    schedule=torch.load(source/'scheduler.bin',map_location='cpu',weights_only=False)
    schedule,floor=extended_schedule(schedule,step=step,old_world=old_world,new_world=world,
        warmup=cfg['train']['warmup_steps'],total=total,floor=cfg['train']['cosine_min_lr_rate'])
    optimizer=torch.load(source/'optimizer.bin',map_location='cpu',weights_only=False,mmap=True)
    if len(optimizer['param_groups']) != len(schedule['base_lrs']):
        raise ValueError('Optimizer/scheduler group count mismatch')
    groups={}
    for g,base,lr in zip(optimizer['param_groups'],schedule['base_lrs'],schedule['_last_lr']):
        if not math.isclose(g['lr'],lr,rel_tol=1e-7):
            raise ValueError('Optimizer LR disagrees with scheduler')
        g['initial_lr']=base
        groups[g['group_name']]=base
        # qa_base_lr/qa_updates and every Adam tensor are intentionally unchanged.
    flow=torch.load(source/'custom_checkpoint_0.pkl',map_location='cpu',weights_only=False)
    if flow['config']['world_size'] != old_world or flow['draw_index'] != step:
        raise ValueError('Flow clock/world mismatch')
    flow['config']['world_size']=world
    options=saved_options(source)
    options.update({'train.train-batch-size':batch,'train.target-effective-sources-per-gpu':batch,
        'train.max-training-steps':total,'train.cosine-min-lr-rate':floor,
        'train.learning-rate':groups['action_header'],'model.lang-backbone-lr':groups['vlm_model'],
        'timestamp':timestamp,'train.output-dir':str(output),'train.resume-load-full-state':True,
        'train.num-workers':4,'train.val-num-workers':2})
    for key in ('data.third-party-root','data.third-party-train-selection','data.third-party-val-selection'):
        options[key]=FIELDS[key][0]
    target=destination/'checkpoints'/source.name
    options['train.resume-from-checkpoint']=str(target)
    options['wandb.id']=cfg['wandb']['id']; options['wandb.resume']='must'
    config=tyro.cli(DynamicLaunchConfig,config=(tyro.conf.ConsolidateSubcommandArgs,),args=to_args(options)[1:])
    target.mkdir(parents=True)
    copy_verified(source/'model.safetensors',target/'model.safetensors')
    torch.save(optimizer,target/'optimizer.bin');torch.save(schedule,target/'scheduler.bin')
    torch.save(flow,target/'custom_checkpoint_0.pkl')
    for rank in range(world):
        state=torch.load(source/f'random_states_{rank}.pkl',map_location='cpu',weights_only=False)
        if len(state['torch_cuda_manual_seed']) != old_world:
            raise ValueError('Unexpected saved CUDA RNG topology')
        state['torch_cuda_manual_seed']=state['torch_cuda_manual_seed'][:world]
        torch.save(state,target/f'random_states_{rank}.pkl')
    for name,contract_key in [('hfd_fused_delta_manifest.json','contract'),
                              ('full_duplex_source_boundary.json','train_infer_contract')]:
        payload=json.loads((source/name).read_text())
        for key in ('train_microbatch_per_gpu','effective_sources_per_gpu'):
            if payload[contract_key][key] != cfg['train']['train_batch_size']:
                raise ValueError('Saved batch contract mismatch')
            payload[contract_key][key]=batch
        atomic_json(target/name,payload)
    for name in ('control_tokenizer','control_token_contract.json','architecture_manifest.json'):
        p=parent/name
        if p.is_dir():shutil.copytree(p,destination/name)
        elif p.is_file():shutil.copy2(p,destination/name)
    atomic_json(destination/'run_config.json',config.model_dump(mode='json'))
    lines=['scripts/train.py']
    for arg in to_args(options):
        if arg.startswith('--') or arg==CONFIG_MODULE:lines.append(arg)
        else:lines[-1]+=' '+arg
    (destination/'argv.txt').write_text('\n'.join(lines)+'\n')
    provenance=dict(format='hfd_changed_data_continuation_v1',parent_checkpoint=str(source),
        parent_backup=backup,global_step=step,old_world=old_world,new_world=world,
        old_batch=cfg['train']['train_batch_size'],new_batch=batch,new_total_steps=total,
        lr_at_start=schedule['_last_lr'],new_base_lrs=schedule['base_lrs'],new_min_lr_rate=floor,
        original_checkpoint_modified=False,optimizer_moments_preserved=True,
        exact_replay=False,data_cursor='restart sampler on expanded data; no active KV at boundary',
        qa_independent_lr_and_update_clock='unchanged',random_state_mapping='old ranks 0..new_world-1',
        checkpoint=str(target))
    atomic_json(destination/'continuation_provenance.json',provenance)
    if not ready(target,world):raise RuntimeError('Prepared checkpoint failed readiness check')
    print(json.dumps(provenance,ensure_ascii=False,indent=2),flush=True)
    return target


if __name__=='__main__':
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument('--checkpoint',type=Path,required=True);p.add_argument('--destination',type=Path,required=True)
    p.add_argument('--world',type=int,required=True);p.add_argument('--batch',type=int,required=True)
    p.add_argument('--total',type=int,required=True);p.add_argument('--timestamp',required=True)
    p.add_argument('--output',type=Path,required=True)
    a=p.parse_args();prepare(a.checkpoint,a.destination,world=a.world,batch=a.batch,total=a.total,timestamp=a.timestamp,output=a.output)