File size: 8,784 Bytes
9fc9873 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | """Create an explicit changed-topology/data continuation; never edit its parent.
Preserves model, Adam moments/steps and the global/flow draw count. The schedule
is reparameterized to start at the saved LR and end at the same absolute floor,
on the extended budget. This is NOT bit-exact replay with the old data/world.
"""
import argparse
import copy
import json
import math
from pathlib import Path
import shutil
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'src'))
def extended_schedule(saved, *, step, old_world, new_world, warmup, total, floor):
if not (0 < new_world <= old_world and warmup < step < total):
raise ValueError('Expected a post-warmup, extended, non-increasing-world continuation')
if saved['last_epoch'] != step * old_world:
raise ValueError('Scheduler clock does not match checkpoint step/world')
old_base = saved['base_lrs']
old_lr = saved['_last_lr']
phase = (step-warmup)/(total-warmup)
cosine = .5 * (1 + math.cos(math.pi * phase))
new_base = [b*floor + (lr-b*floor)/cosine for b,lr in zip(old_base,old_lr)]
ratios = [b*floor/n for b,n in zip(old_base,new_base)]
if not new_base or any(not math.isfinite(n) or n <= 0 for n in new_base):
raise ValueError('Invalid LR state')
if any(not math.isclose(r,ratios[0],rel_tol=1e-8) for r in ratios):
raise ValueError('Parameter groups do not share a common cosine schedule')
if any(lr < b*floor for b,lr in zip(old_base,old_lr)):
raise ValueError('Saved LR is below the requested old floor')
result = copy.deepcopy(saved)
result.update(base_lrs=new_base,last_epoch=step*new_world,_step_count=step*new_world+1)
return result, ratios[0]
def prepare(source, destination, *, world, batch, total, timestamp, output):
import torch
import tyro
from psi.dlc_storage import digest, ready, copy_verified, atomic_json
from psi.hfd_training_launcher import saved_options, to_args, CONFIG_MODULE
from psi.config.hfd_current_recipe import FIELDS
from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig
source=Path(source).resolve(); destination=Path(destination).resolve()
parent=source.parent.parent
cfg=json.loads((parent/'run_config.json').read_text())
marker=json.loads((source/'full_duplex_source_boundary.json').read_text())
old_world=len(list(source.glob('random_states_*.pkl')))
step=int(source.name.removeprefix('ckpt_'))
if destination.exists() or source.is_relative_to(destination) or destination.is_relative_to(parent):
raise ValueError('Destination must be new and outside the original run')
if cfg['train']['stage'] != 0 or marker.get('stage2_data_cursor') is not None:
raise ValueError('Only cache-boundary joint stage=0 is supported')
if not ready(source,old_world):
raise ValueError('Incomplete source checkpoint')
backup=json.loads((source/'backup_complete.json').read_text())
for name,proof in backup['files'].items():
if Path(name).name != name:
raise ValueError('Invalid backup filename')
p=source/name
if p.stat().st_size != proof['bytes'] or digest(p) != proof['sha256']:
raise ValueError(f'Corrupt source checkpoint: {name}')
if batch < 1 or cfg['train']['lr_scheduler_type'] != 'cosine_with_min_lr':
raise ValueError('Expected positive batch and cosine_with_min_lr')
schedule=torch.load(source/'scheduler.bin',map_location='cpu',weights_only=False)
schedule,floor=extended_schedule(schedule,step=step,old_world=old_world,new_world=world,
warmup=cfg['train']['warmup_steps'],total=total,floor=cfg['train']['cosine_min_lr_rate'])
optimizer=torch.load(source/'optimizer.bin',map_location='cpu',weights_only=False,mmap=True)
if len(optimizer['param_groups']) != len(schedule['base_lrs']):
raise ValueError('Optimizer/scheduler group count mismatch')
groups={}
for g,base,lr in zip(optimizer['param_groups'],schedule['base_lrs'],schedule['_last_lr']):
if not math.isclose(g['lr'],lr,rel_tol=1e-7):
raise ValueError('Optimizer LR disagrees with scheduler')
g['initial_lr']=base
groups[g['group_name']]=base
# qa_base_lr/qa_updates and every Adam tensor are intentionally unchanged.
flow=torch.load(source/'custom_checkpoint_0.pkl',map_location='cpu',weights_only=False)
if flow['config']['world_size'] != old_world or flow['draw_index'] != step:
raise ValueError('Flow clock/world mismatch')
flow['config']['world_size']=world
options=saved_options(source)
options.update({'train.train-batch-size':batch,'train.target-effective-sources-per-gpu':batch,
'train.max-training-steps':total,'train.cosine-min-lr-rate':floor,
'train.learning-rate':groups['action_header'],'model.lang-backbone-lr':groups['vlm_model'],
'timestamp':timestamp,'train.output-dir':str(output),'train.resume-load-full-state':True,
'train.num-workers':4,'train.val-num-workers':2})
for key in ('data.third-party-root','data.third-party-train-selection','data.third-party-val-selection'):
options[key]=FIELDS[key][0]
target=destination/'checkpoints'/source.name
options['train.resume-from-checkpoint']=str(target)
options['wandb.id']=cfg['wandb']['id']; options['wandb.resume']='must'
config=tyro.cli(DynamicLaunchConfig,config=(tyro.conf.ConsolidateSubcommandArgs,),args=to_args(options)[1:])
target.mkdir(parents=True)
copy_verified(source/'model.safetensors',target/'model.safetensors')
torch.save(optimizer,target/'optimizer.bin');torch.save(schedule,target/'scheduler.bin')
torch.save(flow,target/'custom_checkpoint_0.pkl')
for rank in range(world):
state=torch.load(source/f'random_states_{rank}.pkl',map_location='cpu',weights_only=False)
if len(state['torch_cuda_manual_seed']) != old_world:
raise ValueError('Unexpected saved CUDA RNG topology')
state['torch_cuda_manual_seed']=state['torch_cuda_manual_seed'][:world]
torch.save(state,target/f'random_states_{rank}.pkl')
for name,contract_key in [('hfd_fused_delta_manifest.json','contract'),
('full_duplex_source_boundary.json','train_infer_contract')]:
payload=json.loads((source/name).read_text())
for key in ('train_microbatch_per_gpu','effective_sources_per_gpu'):
if payload[contract_key][key] != cfg['train']['train_batch_size']:
raise ValueError('Saved batch contract mismatch')
payload[contract_key][key]=batch
atomic_json(target/name,payload)
for name in ('control_tokenizer','control_token_contract.json','architecture_manifest.json'):
p=parent/name
if p.is_dir():shutil.copytree(p,destination/name)
elif p.is_file():shutil.copy2(p,destination/name)
atomic_json(destination/'run_config.json',config.model_dump(mode='json'))
lines=['scripts/train.py']
for arg in to_args(options):
if arg.startswith('--') or arg==CONFIG_MODULE:lines.append(arg)
else:lines[-1]+=' '+arg
(destination/'argv.txt').write_text('\n'.join(lines)+'\n')
provenance=dict(format='hfd_changed_data_continuation_v1',parent_checkpoint=str(source),
parent_backup=backup,global_step=step,old_world=old_world,new_world=world,
old_batch=cfg['train']['train_batch_size'],new_batch=batch,new_total_steps=total,
lr_at_start=schedule['_last_lr'],new_base_lrs=schedule['base_lrs'],new_min_lr_rate=floor,
original_checkpoint_modified=False,optimizer_moments_preserved=True,
exact_replay=False,data_cursor='restart sampler on expanded data; no active KV at boundary',
qa_independent_lr_and_update_clock='unchanged',random_state_mapping='old ranks 0..new_world-1',
checkpoint=str(target))
atomic_json(destination/'continuation_provenance.json',provenance)
if not ready(target,world):raise RuntimeError('Prepared checkpoint failed readiness check')
print(json.dumps(provenance,ensure_ascii=False,indent=2),flush=True)
return target
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--checkpoint',type=Path,required=True);p.add_argument('--destination',type=Path,required=True)
p.add_argument('--world',type=int,required=True);p.add_argument('--batch',type=int,required=True)
p.add_argument('--total',type=int,required=True);p.add_argument('--timestamp',required=True)
p.add_argument('--output',type=Path,required=True)
a=p.parse_args();prepare(a.checkpoint,a.destination,world=a.world,batch=a.batch,total=a.total,timestamp=a.timestamp,output=a.output)
|