Download scripts/prepare_hfd_continuation.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 8.78 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/prepare_hfd_continuation.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/scripts/prepare_hfd_continuation.py
-
curl -L -o prepare_hfd_continuation.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/scripts/prepare_hfd_continuation.py
8.78 kB
| """Create an explicit changed-topology/data continuation; never edit its parent. | |
| Preserves model, Adam moments/steps and the global/flow draw count. The schedule | |
| is reparameterized to start at the saved LR and end at the same absolute floor, | |
| on the extended budget. This is NOT bit-exact replay with the old data/world. | |
| """ | |
| import argparse | |
| import copy | |
| import json | |
| import math | |
| from pathlib import Path | |
| import shutil | |
| import sys | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT/'src')) | |
| def extended_schedule(saved, *, step, old_world, new_world, warmup, total, floor): | |
| if not (0 < new_world <= old_world and warmup < step < total): | |
| raise ValueError('Expected a post-warmup, extended, non-increasing-world continuation') | |
| if saved['last_epoch'] != step * old_world: | |
| raise ValueError('Scheduler clock does not match checkpoint step/world') | |
| old_base = saved['base_lrs'] | |
| old_lr = saved['_last_lr'] | |
| phase = (step-warmup)/(total-warmup) | |
| cosine = .5 * (1 + math.cos(math.pi * phase)) | |
| new_base = [b*floor + (lr-b*floor)/cosine for b,lr in zip(old_base,old_lr)] | |
| ratios = [b*floor/n for b,n in zip(old_base,new_base)] | |
| if not new_base or any(not math.isfinite(n) or n <= 0 for n in new_base): | |
| raise ValueError('Invalid LR state') | |
| if any(not math.isclose(r,ratios[0],rel_tol=1e-8) for r in ratios): | |
| raise ValueError('Parameter groups do not share a common cosine schedule') | |
| if any(lr < b*floor for b,lr in zip(old_base,old_lr)): | |
| raise ValueError('Saved LR is below the requested old floor') | |
| result = copy.deepcopy(saved) | |
| result.update(base_lrs=new_base,last_epoch=step*new_world,_step_count=step*new_world+1) | |
| return result, ratios[0] | |
| def prepare(source, destination, *, world, batch, total, timestamp, output): | |
| import torch | |
| import tyro | |
| from psi.dlc_storage import digest, ready, copy_verified, atomic_json | |
| from psi.hfd_training_launcher import saved_options, to_args, CONFIG_MODULE | |
| from psi.config.hfd_current_recipe import FIELDS | |
| from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig | |
| source=Path(source).resolve(); destination=Path(destination).resolve() | |
| parent=source.parent.parent | |
| cfg=json.loads((parent/'run_config.json').read_text()) | |
| marker=json.loads((source/'full_duplex_source_boundary.json').read_text()) | |
| old_world=len(list(source.glob('random_states_*.pkl'))) | |
| step=int(source.name.removeprefix('ckpt_')) | |
| if destination.exists() or source.is_relative_to(destination) or destination.is_relative_to(parent): | |
| raise ValueError('Destination must be new and outside the original run') | |
| if cfg['train']['stage'] != 0 or marker.get('stage2_data_cursor') is not None: | |
| raise ValueError('Only cache-boundary joint stage=0 is supported') | |
| if not ready(source,old_world): | |
| raise ValueError('Incomplete source checkpoint') | |
| backup=json.loads((source/'backup_complete.json').read_text()) | |
| for name,proof in backup['files'].items(): | |
| if Path(name).name != name: | |
| raise ValueError('Invalid backup filename') | |
| p=source/name | |
| if p.stat().st_size != proof['bytes'] or digest(p) != proof['sha256']: | |
| raise ValueError(f'Corrupt source checkpoint: {name}') | |
| if batch < 1 or cfg['train']['lr_scheduler_type'] != 'cosine_with_min_lr': | |
| raise ValueError('Expected positive batch and cosine_with_min_lr') | |
| schedule=torch.load(source/'scheduler.bin',map_location='cpu',weights_only=False) | |
| schedule,floor=extended_schedule(schedule,step=step,old_world=old_world,new_world=world, | |
| warmup=cfg['train']['warmup_steps'],total=total,floor=cfg['train']['cosine_min_lr_rate']) | |
| optimizer=torch.load(source/'optimizer.bin',map_location='cpu',weights_only=False,mmap=True) | |
| if len(optimizer['param_groups']) != len(schedule['base_lrs']): | |
| raise ValueError('Optimizer/scheduler group count mismatch') | |
| groups={} | |
| for g,base,lr in zip(optimizer['param_groups'],schedule['base_lrs'],schedule['_last_lr']): | |
| if not math.isclose(g['lr'],lr,rel_tol=1e-7): | |
| raise ValueError('Optimizer LR disagrees with scheduler') | |
| g['initial_lr']=base | |
| groups[g['group_name']]=base | |
| # qa_base_lr/qa_updates and every Adam tensor are intentionally unchanged. | |
| flow=torch.load(source/'custom_checkpoint_0.pkl',map_location='cpu',weights_only=False) | |
| if flow['config']['world_size'] != old_world or flow['draw_index'] != step: | |
| raise ValueError('Flow clock/world mismatch') | |
| flow['config']['world_size']=world | |
| options=saved_options(source) | |
| options.update({'train.train-batch-size':batch,'train.target-effective-sources-per-gpu':batch, | |
| 'train.max-training-steps':total,'train.cosine-min-lr-rate':floor, | |
| 'train.learning-rate':groups['action_header'],'model.lang-backbone-lr':groups['vlm_model'], | |
| 'timestamp':timestamp,'train.output-dir':str(output),'train.resume-load-full-state':True, | |
| 'train.num-workers':4,'train.val-num-workers':2}) | |
| for key in ('data.third-party-root','data.third-party-train-selection','data.third-party-val-selection'): | |
| options[key]=FIELDS[key][0] | |
| target=destination/'checkpoints'/source.name | |
| options['train.resume-from-checkpoint']=str(target) | |
| options['wandb.id']=cfg['wandb']['id']; options['wandb.resume']='must' | |
| config=tyro.cli(DynamicLaunchConfig,config=(tyro.conf.ConsolidateSubcommandArgs,),args=to_args(options)[1:]) | |
| target.mkdir(parents=True) | |
| copy_verified(source/'model.safetensors',target/'model.safetensors') | |
| torch.save(optimizer,target/'optimizer.bin');torch.save(schedule,target/'scheduler.bin') | |
| torch.save(flow,target/'custom_checkpoint_0.pkl') | |
| for rank in range(world): | |
| state=torch.load(source/f'random_states_{rank}.pkl',map_location='cpu',weights_only=False) | |
| if len(state['torch_cuda_manual_seed']) != old_world: | |
| raise ValueError('Unexpected saved CUDA RNG topology') | |
| state['torch_cuda_manual_seed']=state['torch_cuda_manual_seed'][:world] | |
| torch.save(state,target/f'random_states_{rank}.pkl') | |
| for name,contract_key in [('hfd_fused_delta_manifest.json','contract'), | |
| ('full_duplex_source_boundary.json','train_infer_contract')]: | |
| payload=json.loads((source/name).read_text()) | |
| for key in ('train_microbatch_per_gpu','effective_sources_per_gpu'): | |
| if payload[contract_key][key] != cfg['train']['train_batch_size']: | |
| raise ValueError('Saved batch contract mismatch') | |
| payload[contract_key][key]=batch | |
| atomic_json(target/name,payload) | |
| for name in ('control_tokenizer','control_token_contract.json','architecture_manifest.json'): | |
| p=parent/name | |
| if p.is_dir():shutil.copytree(p,destination/name) | |
| elif p.is_file():shutil.copy2(p,destination/name) | |
| atomic_json(destination/'run_config.json',config.model_dump(mode='json')) | |
| lines=['scripts/train.py'] | |
| for arg in to_args(options): | |
| if arg.startswith('--') or arg==CONFIG_MODULE:lines.append(arg) | |
| else:lines[-1]+=' '+arg | |
| (destination/'argv.txt').write_text('\n'.join(lines)+'\n') | |
| provenance=dict(format='hfd_changed_data_continuation_v1',parent_checkpoint=str(source), | |
| parent_backup=backup,global_step=step,old_world=old_world,new_world=world, | |
| old_batch=cfg['train']['train_batch_size'],new_batch=batch,new_total_steps=total, | |
| lr_at_start=schedule['_last_lr'],new_base_lrs=schedule['base_lrs'],new_min_lr_rate=floor, | |
| original_checkpoint_modified=False,optimizer_moments_preserved=True, | |
| exact_replay=False,data_cursor='restart sampler on expanded data; no active KV at boundary', | |
| qa_independent_lr_and_update_clock='unchanged',random_state_mapping='old ranks 0..new_world-1', | |
| checkpoint=str(target)) | |
| atomic_json(destination/'continuation_provenance.json',provenance) | |
| if not ready(target,world):raise RuntimeError('Prepared checkpoint failed readiness check') | |
| print(json.dumps(provenance,ensure_ascii=False,indent=2),flush=True) | |
| return target | |
| if __name__=='__main__': | |
| p=argparse.ArgumentParser(description=__doc__) | |
| p.add_argument('--checkpoint',type=Path,required=True);p.add_argument('--destination',type=Path,required=True) | |
| p.add_argument('--world',type=int,required=True);p.add_argument('--batch',type=int,required=True) | |
| p.add_argument('--total',type=int,required=True);p.add_argument('--timestamp',required=True) | |
| p.add_argument('--output',type=Path,required=True) | |
| a=p.parse_args();prepare(a.checkpoint,a.destination,world=a.world,batch=a.batch,total=a.total,timestamp=a.timestamp,output=a.output) | |