code-HFD / scripts /prepare_hfd_continuation.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
9fc9873 verified
Raw History Blame Contribute Delete
8.78 kB
"""Create an explicit changed-topology/data continuation; never edit its parent.
Preserves model, Adam moments/steps and the global/flow draw count. The schedule
is reparameterized to start at the saved LR and end at the same absolute floor,
on the extended budget. This is NOT bit-exact replay with the old data/world.
"""
import argparse
import copy
import json
import math
from pathlib import Path
import shutil
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'src'))
def extended_schedule(saved, *, step, old_world, new_world, warmup, total, floor):
if not (0 < new_world <= old_world and warmup < step < total):
raise ValueError('Expected a post-warmup, extended, non-increasing-world continuation')
if saved['last_epoch'] != step * old_world:
raise ValueError('Scheduler clock does not match checkpoint step/world')
old_base = saved['base_lrs']
old_lr = saved['_last_lr']
phase = (step-warmup)/(total-warmup)
cosine = .5 * (1 + math.cos(math.pi * phase))
new_base = [b*floor + (lr-b*floor)/cosine for b,lr in zip(old_base,old_lr)]
ratios = [b*floor/n for b,n in zip(old_base,new_base)]
if not new_base or any(not math.isfinite(n) or n <= 0 for n in new_base):
raise ValueError('Invalid LR state')
if any(not math.isclose(r,ratios[0],rel_tol=1e-8) for r in ratios):
raise ValueError('Parameter groups do not share a common cosine schedule')
if any(lr < b*floor for b,lr in zip(old_base,old_lr)):
raise ValueError('Saved LR is below the requested old floor')
result = copy.deepcopy(saved)
result.update(base_lrs=new_base,last_epoch=step*new_world,_step_count=step*new_world+1)
return result, ratios[0]
def prepare(source, destination, *, world, batch, total, timestamp, output):
import torch
import tyro
from psi.dlc_storage import digest, ready, copy_verified, atomic_json
from psi.hfd_training_launcher import saved_options, to_args, CONFIG_MODULE
from psi.config.hfd_current_recipe import FIELDS
from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig
source=Path(source).resolve(); destination=Path(destination).resolve()
parent=source.parent.parent
cfg=json.loads((parent/'run_config.json').read_text())
marker=json.loads((source/'full_duplex_source_boundary.json').read_text())
old_world=len(list(source.glob('random_states_*.pkl')))
step=int(source.name.removeprefix('ckpt_'))
if destination.exists() or source.is_relative_to(destination) or destination.is_relative_to(parent):
raise ValueError('Destination must be new and outside the original run')
if cfg['train']['stage'] != 0 or marker.get('stage2_data_cursor') is not None:
raise ValueError('Only cache-boundary joint stage=0 is supported')
if not ready(source,old_world):
raise ValueError('Incomplete source checkpoint')
backup=json.loads((source/'backup_complete.json').read_text())
for name,proof in backup['files'].items():
if Path(name).name != name:
raise ValueError('Invalid backup filename')
p=source/name
if p.stat().st_size != proof['bytes'] or digest(p) != proof['sha256']:
raise ValueError(f'Corrupt source checkpoint: {name}')
if batch < 1 or cfg['train']['lr_scheduler_type'] != 'cosine_with_min_lr':
raise ValueError('Expected positive batch and cosine_with_min_lr')
schedule=torch.load(source/'scheduler.bin',map_location='cpu',weights_only=False)
schedule,floor=extended_schedule(schedule,step=step,old_world=old_world,new_world=world,
warmup=cfg['train']['warmup_steps'],total=total,floor=cfg['train']['cosine_min_lr_rate'])
optimizer=torch.load(source/'optimizer.bin',map_location='cpu',weights_only=False,mmap=True)
if len(optimizer['param_groups']) != len(schedule['base_lrs']):
raise ValueError('Optimizer/scheduler group count mismatch')
groups={}
for g,base,lr in zip(optimizer['param_groups'],schedule['base_lrs'],schedule['_last_lr']):
if not math.isclose(g['lr'],lr,rel_tol=1e-7):
raise ValueError('Optimizer LR disagrees with scheduler')
g['initial_lr']=base
groups[g['group_name']]=base
# qa_base_lr/qa_updates and every Adam tensor are intentionally unchanged.
flow=torch.load(source/'custom_checkpoint_0.pkl',map_location='cpu',weights_only=False)
if flow['config']['world_size'] != old_world or flow['draw_index'] != step:
raise ValueError('Flow clock/world mismatch')
flow['config']['world_size']=world
options=saved_options(source)
options.update({'train.train-batch-size':batch,'train.target-effective-sources-per-gpu':batch,
'train.max-training-steps':total,'train.cosine-min-lr-rate':floor,
'train.learning-rate':groups['action_header'],'model.lang-backbone-lr':groups['vlm_model'],
'timestamp':timestamp,'train.output-dir':str(output),'train.resume-load-full-state':True,
'train.num-workers':4,'train.val-num-workers':2})
for key in ('data.third-party-root','data.third-party-train-selection','data.third-party-val-selection'):
options[key]=FIELDS[key][0]
target=destination/'checkpoints'/source.name
options['train.resume-from-checkpoint']=str(target)
options['wandb.id']=cfg['wandb']['id']; options['wandb.resume']='must'
config=tyro.cli(DynamicLaunchConfig,config=(tyro.conf.ConsolidateSubcommandArgs,),args=to_args(options)[1:])
target.mkdir(parents=True)
copy_verified(source/'model.safetensors',target/'model.safetensors')
torch.save(optimizer,target/'optimizer.bin');torch.save(schedule,target/'scheduler.bin')
torch.save(flow,target/'custom_checkpoint_0.pkl')
for rank in range(world):
state=torch.load(source/f'random_states_{rank}.pkl',map_location='cpu',weights_only=False)
if len(state['torch_cuda_manual_seed']) != old_world:
raise ValueError('Unexpected saved CUDA RNG topology')
state['torch_cuda_manual_seed']=state['torch_cuda_manual_seed'][:world]
torch.save(state,target/f'random_states_{rank}.pkl')
for name,contract_key in [('hfd_fused_delta_manifest.json','contract'),
('full_duplex_source_boundary.json','train_infer_contract')]:
payload=json.loads((source/name).read_text())
for key in ('train_microbatch_per_gpu','effective_sources_per_gpu'):
if payload[contract_key][key] != cfg['train']['train_batch_size']:
raise ValueError('Saved batch contract mismatch')
payload[contract_key][key]=batch
atomic_json(target/name,payload)
for name in ('control_tokenizer','control_token_contract.json','architecture_manifest.json'):
p=parent/name
if p.is_dir():shutil.copytree(p,destination/name)
elif p.is_file():shutil.copy2(p,destination/name)
atomic_json(destination/'run_config.json',config.model_dump(mode='json'))
lines=['scripts/train.py']
for arg in to_args(options):
if arg.startswith('--') or arg==CONFIG_MODULE:lines.append(arg)
else:lines[-1]+=' '+arg
(destination/'argv.txt').write_text('\n'.join(lines)+'\n')
provenance=dict(format='hfd_changed_data_continuation_v1',parent_checkpoint=str(source),
parent_backup=backup,global_step=step,old_world=old_world,new_world=world,
old_batch=cfg['train']['train_batch_size'],new_batch=batch,new_total_steps=total,
lr_at_start=schedule['_last_lr'],new_base_lrs=schedule['base_lrs'],new_min_lr_rate=floor,
original_checkpoint_modified=False,optimizer_moments_preserved=True,
exact_replay=False,data_cursor='restart sampler on expanded data; no active KV at boundary',
qa_independent_lr_and_update_clock='unchanged',random_state_mapping='old ranks 0..new_world-1',
checkpoint=str(target))
atomic_json(destination/'continuation_provenance.json',provenance)
if not ready(target,world):raise RuntimeError('Prepared checkpoint failed readiness check')
print(json.dumps(provenance,ensure_ascii=False,indent=2),flush=True)
return target
if __name__=='__main__':
p=argparse.ArgumentParser(description=__doc__)
p.add_argument('--checkpoint',type=Path,required=True);p.add_argument('--destination',type=Path,required=True)
p.add_argument('--world',type=int,required=True);p.add_argument('--batch',type=int,required=True)
p.add_argument('--total',type=int,required=True);p.add_argument('--timestamp',required=True)
p.add_argument('--output',type=Path,required=True)
a=p.parse_args();prepare(a.checkpoint,a.destination,world=a.world,batch=a.batch,total=a.total,timestamp=a.timestamp,output=a.output)