"""Create an explicit changed-topology/data continuation; never edit its parent. Preserves model, Adam moments/steps and the global/flow draw count. The schedule is reparameterized to start at the saved LR and end at the same absolute floor, on the extended budget. This is NOT bit-exact replay with the old data/world. """ import argparse import copy import json import math from pathlib import Path import shutil import sys ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT/'src')) def extended_schedule(saved, *, step, old_world, new_world, warmup, total, floor): if not (0 < new_world <= old_world and warmup < step < total): raise ValueError('Expected a post-warmup, extended, non-increasing-world continuation') if saved['last_epoch'] != step * old_world: raise ValueError('Scheduler clock does not match checkpoint step/world') old_base = saved['base_lrs'] old_lr = saved['_last_lr'] phase = (step-warmup)/(total-warmup) cosine = .5 * (1 + math.cos(math.pi * phase)) new_base = [b*floor + (lr-b*floor)/cosine for b,lr in zip(old_base,old_lr)] ratios = [b*floor/n for b,n in zip(old_base,new_base)] if not new_base or any(not math.isfinite(n) or n <= 0 for n in new_base): raise ValueError('Invalid LR state') if any(not math.isclose(r,ratios[0],rel_tol=1e-8) for r in ratios): raise ValueError('Parameter groups do not share a common cosine schedule') if any(lr < b*floor for b,lr in zip(old_base,old_lr)): raise ValueError('Saved LR is below the requested old floor') result = copy.deepcopy(saved) result.update(base_lrs=new_base,last_epoch=step*new_world,_step_count=step*new_world+1) return result, ratios[0] def prepare(source, destination, *, world, batch, total, timestamp, output): import torch import tyro from psi.dlc_storage import digest, ready, copy_verified, atomic_json from psi.hfd_training_launcher import saved_options, to_args, CONFIG_MODULE from psi.config.hfd_current_recipe import FIELDS from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig source=Path(source).resolve(); destination=Path(destination).resolve() parent=source.parent.parent cfg=json.loads((parent/'run_config.json').read_text()) marker=json.loads((source/'full_duplex_source_boundary.json').read_text()) old_world=len(list(source.glob('random_states_*.pkl'))) step=int(source.name.removeprefix('ckpt_')) if destination.exists() or source.is_relative_to(destination) or destination.is_relative_to(parent): raise ValueError('Destination must be new and outside the original run') if cfg['train']['stage'] != 0 or marker.get('stage2_data_cursor') is not None: raise ValueError('Only cache-boundary joint stage=0 is supported') if not ready(source,old_world): raise ValueError('Incomplete source checkpoint') backup=json.loads((source/'backup_complete.json').read_text()) for name,proof in backup['files'].items(): if Path(name).name != name: raise ValueError('Invalid backup filename') p=source/name if p.stat().st_size != proof['bytes'] or digest(p) != proof['sha256']: raise ValueError(f'Corrupt source checkpoint: {name}') if batch < 1 or cfg['train']['lr_scheduler_type'] != 'cosine_with_min_lr': raise ValueError('Expected positive batch and cosine_with_min_lr') schedule=torch.load(source/'scheduler.bin',map_location='cpu',weights_only=False) schedule,floor=extended_schedule(schedule,step=step,old_world=old_world,new_world=world, warmup=cfg['train']['warmup_steps'],total=total,floor=cfg['train']['cosine_min_lr_rate']) optimizer=torch.load(source/'optimizer.bin',map_location='cpu',weights_only=False,mmap=True) if len(optimizer['param_groups']) != len(schedule['base_lrs']): raise ValueError('Optimizer/scheduler group count mismatch') groups={} for g,base,lr in zip(optimizer['param_groups'],schedule['base_lrs'],schedule['_last_lr']): if not math.isclose(g['lr'],lr,rel_tol=1e-7): raise ValueError('Optimizer LR disagrees with scheduler') g['initial_lr']=base groups[g['group_name']]=base # qa_base_lr/qa_updates and every Adam tensor are intentionally unchanged. flow=torch.load(source/'custom_checkpoint_0.pkl',map_location='cpu',weights_only=False) if flow['config']['world_size'] != old_world or flow['draw_index'] != step: raise ValueError('Flow clock/world mismatch') flow['config']['world_size']=world options=saved_options(source) options.update({'train.train-batch-size':batch,'train.target-effective-sources-per-gpu':batch, 'train.max-training-steps':total,'train.cosine-min-lr-rate':floor, 'train.learning-rate':groups['action_header'],'model.lang-backbone-lr':groups['vlm_model'], 'timestamp':timestamp,'train.output-dir':str(output),'train.resume-load-full-state':True, 'train.num-workers':4,'train.val-num-workers':2}) for key in ('data.third-party-root','data.third-party-train-selection','data.third-party-val-selection'): options[key]=FIELDS[key][0] target=destination/'checkpoints'/source.name options['train.resume-from-checkpoint']=str(target) options['wandb.id']=cfg['wandb']['id']; options['wandb.resume']='must' config=tyro.cli(DynamicLaunchConfig,config=(tyro.conf.ConsolidateSubcommandArgs,),args=to_args(options)[1:]) target.mkdir(parents=True) copy_verified(source/'model.safetensors',target/'model.safetensors') torch.save(optimizer,target/'optimizer.bin');torch.save(schedule,target/'scheduler.bin') torch.save(flow,target/'custom_checkpoint_0.pkl') for rank in range(world): state=torch.load(source/f'random_states_{rank}.pkl',map_location='cpu',weights_only=False) if len(state['torch_cuda_manual_seed']) != old_world: raise ValueError('Unexpected saved CUDA RNG topology') state['torch_cuda_manual_seed']=state['torch_cuda_manual_seed'][:world] torch.save(state,target/f'random_states_{rank}.pkl') for name,contract_key in [('hfd_fused_delta_manifest.json','contract'), ('full_duplex_source_boundary.json','train_infer_contract')]: payload=json.loads((source/name).read_text()) for key in ('train_microbatch_per_gpu','effective_sources_per_gpu'): if payload[contract_key][key] != cfg['train']['train_batch_size']: raise ValueError('Saved batch contract mismatch') payload[contract_key][key]=batch atomic_json(target/name,payload) for name in ('control_tokenizer','control_token_contract.json','architecture_manifest.json'): p=parent/name if p.is_dir():shutil.copytree(p,destination/name) elif p.is_file():shutil.copy2(p,destination/name) atomic_json(destination/'run_config.json',config.model_dump(mode='json')) lines=['scripts/train.py'] for arg in to_args(options): if arg.startswith('--') or arg==CONFIG_MODULE:lines.append(arg) else:lines[-1]+=' '+arg (destination/'argv.txt').write_text('\n'.join(lines)+'\n') provenance=dict(format='hfd_changed_data_continuation_v1',parent_checkpoint=str(source), parent_backup=backup,global_step=step,old_world=old_world,new_world=world, old_batch=cfg['train']['train_batch_size'],new_batch=batch,new_total_steps=total, lr_at_start=schedule['_last_lr'],new_base_lrs=schedule['base_lrs'],new_min_lr_rate=floor, original_checkpoint_modified=False,optimizer_moments_preserved=True, exact_replay=False,data_cursor='restart sampler on expanded data; no active KV at boundary', qa_independent_lr_and_update_clock='unchanged',random_state_mapping='old ranks 0..new_world-1', checkpoint=str(target)) atomic_json(destination/'continuation_provenance.json',provenance) if not ready(target,world):raise RuntimeError('Prepared checkpoint failed readiness check') print(json.dumps(provenance,ensure_ascii=False,indent=2),flush=True) return target if __name__=='__main__': p=argparse.ArgumentParser(description=__doc__) p.add_argument('--checkpoint',type=Path,required=True);p.add_argument('--destination',type=Path,required=True) p.add_argument('--world',type=int,required=True);p.add_argument('--batch',type=int,required=True) p.add_argument('--total',type=int,required=True);p.add_argument('--timestamp',required=True) p.add_argument('--output',type=Path,required=True) a=p.parse_args();prepare(a.checkpoint,a.destination,world=a.world,batch=a.batch,total=a.total,timestamp=a.timestamp,output=a.output)