Download scale/script/train_reference_baseline_fresh.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/train_reference_baseline_fresh.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/train_reference_baseline_fresh.py
-
curl -L -o train_reference_baseline_fresh.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/train_reference_baseline_fresh.py
11.2 kB
| """Official Omni + random normal-size expert; exact brother training worker. | |
| Storage/cache paths and initialization provenance are isolated. No training | |
| implementation is patched. Effective LRs, batch/accumulation and loss follow | |
| the brother's saved 89k configuration. This is NOT replay of his warmstart. | |
| """ | |
| import argparse | |
| import copy | |
| import datetime | |
| import fcntl | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shutil | |
| import signal | |
| import subprocess | |
| import sys | |
| import threading | |
| import time | |
| ROOT = Path(__file__).resolve().parents[2] | |
| WS = ROOT.parents[2] | |
| LOCAL = Path('/mnt/workspace/Movens-Zero_train/scale/baseline_reference_20261008') | |
| SOURCE = LOCAL/'source' | |
| BRO = WS/'buwendong/user/code/OmniHM-Zero_train_40d' | |
| REF = WS/'buwendong/user/tmp/train_dashboard_20260930/case_gallery/ckpt_89000__root40_v5/snapshot' | |
| ARCHIVE = ROOT/'scale/baseline_reference_20261008' | |
| PYTHON = ROOT.parent/'Psi0-yth-v26/.venv-psi/bin/python' | |
| GPU = 3 | |
| PORT = 29639 | |
| sys.path.insert(0,str(ROOT/'scale/script')) | |
| from dlc_local_storage import copy_verified, save | |
| import preview_gpu_lease as gpu | |
| def digest(path): | |
| return hashlib.sha256(path.read_bytes()).hexdigest() | |
| def prepare(): | |
| config_path=LOCAL/'fresh_config.json' | |
| if config_path.exists(): | |
| return config_path | |
| if not (SOURCE/'train_vlm.py').exists(): | |
| raise RuntimeError('First run scale/audits/training_baseline_20261008/audit.py') | |
| source_hashes=json.loads((ROOT/'scale/audits/training_baseline_20261008/source_hashes.json').read_text()) | |
| for rel,expected in source_hashes.items(): | |
| if digest(SOURCE/rel)!=expected: | |
| raise RuntimeError(f'Frozen reference code was changed: {rel}') | |
| original=json.loads((REF/'run_config.json').read_text()) | |
| cfg=copy.deepcopy(original) | |
| cfg.update(exp='baseline_reference_official_omni_random_action', | |
| timestamp=datetime.datetime.now().strftime('%y%m%d%H%M%S'),auto_tag_run=False, | |
| initial_action_checkpoint=None,warmstart_checkpoint=None,warmstart_provenance={}, | |
| resume_provenance={},resume_transition_provenance={},world_size_expected=1) | |
| cfg['absolute_height_stage']=dict(start_step=0,end_step=100000,lr_scale=0.1, | |
| scheduler_policy='constant_at_reference_effective_lr', | |
| source_checkpoint='NONE:official_omni_random_action_fresh_initialization') | |
| cfg['train'].update(resume_from_checkpoint=None,resume_load_full_state=False,skip_resumed_steps=False, | |
| output_dir=str(LOCAL/'runs'),max_checkpoints_to_keep=2) | |
| cfg['model']['pretrained_action_header_path']=None | |
| cfg['log']['report_to']='none' | |
| cfg['wandb'].update(id=None,name=None,resume='never',project='Movens-reference-baseline-fresh') | |
| # Keep all data sources/metadata and their logical fingerprint identical. | |
| # Only copy and relocate writable caches; do not write to shared originals. | |
| old_cache=BRO/'training_set/cache/records' | |
| new_cache=LOCAL/'cache/records' | |
| shutil.copytree(old_cache,new_cache,dirs_exist_ok=True) | |
| relocations={} | |
| for field in ['text','audio','audio_command','video_clip']: | |
| data=cfg['data'][field] | |
| sources=json.loads(data['sources_json']) | |
| for item in [data]+sources: | |
| old=item.get('records_cache_dir') | |
| if old: | |
| new=str(new_cache/Path(old).relative_to(old_cache)) | |
| item['records_cache_dir']=new;relocations[old]=new | |
| data['sources_json']=json.dumps(sources) | |
| # Never retain the brother's machine-local cache/preparation output paths. | |
| cfg['train']['jerk_kinematics_path']=str(SOURCE/'assets/robots/g1/g1_kinematics.json') | |
| if cfg['train'].get('absolute_height_startup_seed_path'): | |
| seed=LOCAL/'startup_pose_motion36.json' | |
| origin=Path(original['train']['absolute_height_startup_seed_path']) | |
| if origin.exists(): | |
| shutil.copy2(origin,seed) | |
| else: | |
| # Auxiliary loss is disabled, so this path is deliberately unused. | |
| seed=LOCAL/'UNUSED_startup_pose_motion36.json' | |
| cfg['train']['absolute_height_startup_seed_path']=str(seed) | |
| shutil.copy2(REF/'dataset_statistics.json',LOCAL/'dataset_statistics.json') | |
| cfg['data']['transform']['field']['stat_path']=str(LOCAL/'dataset_statistics.json') | |
| assert (cfg['model']['hidden_dim'],cfg['model']['num_blocks'],cfg['model']['nhead'])==(1536,6,24) | |
| t=cfg['train'] | |
| assert (t['train_batch_size'],t['gradient_accumulation_steps'],t['max_training_steps'])==(32,12,100000) | |
| assert t['absolute_height_loss_weight']==t['jerk_loss_weight']==0 | |
| assert t['cfg_condition_dropout_prob']==0.1 and t['warmup_steps']==0 | |
| assert abs(t['learning_rate']*.1-1e-6)<1e-15 | |
| assert abs(cfg['model']['lang_backbone_lr']*.1-1e-7)<1e-15 | |
| save(config_path,cfg) | |
| save(LOCAL/'reference_config.json',original) | |
| save(LOCAL/'provenance.json',dict(reference_config=str(REF/'run_config.json'), | |
| reference_config_sha256=digest(REF/'run_config.json'),source_hashes=source_hashes, | |
| initialization='official Omni + random normal-size action; optimizer/scheduler/RNG/data cursor fresh', | |
| batch=32,accumulation=12,global_batch=384,effective_action_lr=1e-6,effective_omni_lr=1e-7, | |
| cache_relocations=relocations,storage_changes='independent outputs; latest two checkpoints', | |
| code_changes='NONE: original byte-identical train_vlm.py and src execute --worker-config', | |
| optimizer_history_reproduced=False,reference_training_history_reproduced=False)) | |
| for name in ['fresh_config.json','reference_config.json','provenance.json','dataset_statistics.json']: | |
| copy_verified(LOCAL/name,ARCHIVE/name) | |
| return config_path | |
| def backup_once(): | |
| for run in (LOCAL/'runs/posttrain_v1_vlm').glob('*'): | |
| if not run.is_dir():continue | |
| destination=ARCHIVE/'runs'/run.name | |
| for name in ['run_config.json','dataset_statistics.json','metrics.jsonl']: | |
| if (run/name).is_file(): | |
| if name == 'metrics.jsonl': | |
| # copy_verified deliberately never replaces existing data; | |
| # live metrics need a verified new snapshot, then atomic publish. | |
| destination.mkdir(parents=True,exist_ok=True) | |
| temporary=destination/f'.metrics.snapshot.{os.getpid()}' | |
| if temporary.exists(): | |
| temporary.unlink() # only our own unfinished log snapshot | |
| copy_verified(run/name,temporary,append_only=True) | |
| os.replace(temporary,destination/name) | |
| else: | |
| copy_verified(run/name,destination/name) | |
| markers=sorted((run/'checkpoints').glob('ckpt_*/checkpoint_complete.json'),key=lambda p:int(p.parent.name[5:])) | |
| for marker in markers: | |
| ckpt=marker.parent;dest=destination/'checkpoints'/ckpt.name | |
| if (dest/'BACKUP_COMPLETE.json').exists():continue | |
| files={} | |
| for src in sorted(ckpt.iterdir()): | |
| if src.is_file():files[src.name]=copy_verified(src,dest/src.name) | |
| save(dest/'BACKUP_COMPLETE.json',dict(owner='reference-baseline-fresh-v1',step=int(ckpt.name[5:]),files=files)) | |
| save(ARCHIVE/'latest.json',dict(run=str(destination),step=int(ckpt.name[5:]),checkpoint=str(dest))) | |
| completed=sorted((destination/'checkpoints').glob('ckpt_*/BACKUP_COMPLETE.json'),key=lambda p:int(p.parent.name[5:])) | |
| for marker in completed[:-2]: | |
| if json.loads(marker.read_text()).get('owner')!='reference-baseline-fresh-v1': | |
| raise RuntimeError('Refusing to prune unowned backup') | |
| target=marker.parent.resolve() | |
| if not target.is_relative_to(ARCHIVE.resolve()) or not target.name.startswith('ckpt_'): | |
| raise RuntimeError('Unsafe backup pruning target') | |
| shutil.rmtree(target) | |
| def main(): | |
| parser=argparse.ArgumentParser();parser.add_argument('--prepare-only',action='store_true') | |
| args=parser.parse_args();LOCAL.mkdir(parents=True,exist_ok=True) | |
| with (LOCAL/'supervisor.lock').open('a') as lock: | |
| fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| config=prepare() | |
| if args.prepare_only: | |
| print(config,flush=True);return | |
| if (LOCAL/'launch_state.json').exists(): | |
| raise RuntimeError('A launch was already attempted. Inspect it; never silently restart from zero.') | |
| # Physical GPU 3 has a separate existing keeper; do not touch GPU 2's previews. | |
| gpu.PREVIEW_GPU=GPU;gpu.KEEPER_PREFIX=f'/tmp/movens_scale_preview_gpu{GPU}_keeper' | |
| gpu.PID_FILES=(Path(gpu.KEEPER_PREFIX+'_supervisor.pid'),Path(gpu.KEEPER_PREFIX+'.lock')) | |
| gpu.gpu_processes.__defaults__=(GPU,) | |
| def interrupt(sig,frame):raise KeyboardInterrupt(sig) | |
| signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt) | |
| stop=threading.Event() | |
| def backups(): | |
| while not stop.is_set(): | |
| try:backup_once() | |
| except Exception as e:save(LOCAL/'backup_status.json',dict(error=repr(e),time=time.time())) | |
| else:save(LOCAL/'backup_status.json',dict(status='ok',time=time.time())) | |
| stop.wait(60) | |
| thread=threading.Thread(target=backups,daemon=True) | |
| with gpu.lease_gpu2(): | |
| env=os.environ.copy() | |
| env.update(CUDA_VISIBLE_DEVICES=str(GPU),PSI_DATALOADER_PIN_MEMORY='1',PSI_DATALOADER_PREFETCH_FACTOR='2', | |
| PSI_WORKER_TORCH_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1', | |
| PSI_TRAIN_TORCH_THREADS='8',PSI_VIDEO_FAST_SEEK='1', | |
| PSI_VIDEO_LOCAL_CACHE_ROOT=str(LOCAL/'cache/video_clips'), | |
| PSI_VIDEO_CACHE_MIN_FREE_GIB='256',PSI_VIDEO_CACHE_MAX_SOURCE_GIB='2', | |
| HF_HUB_OFFLINE='1',PYTHONDONTWRITEBYTECODE='1',WANDB_MODE='disabled') | |
| command=[str(PYTHON),'-B','-u','-m','torch.distributed.run','--nproc_per_node=1', | |
| f'--master_port={PORT}',str(SOURCE/'train_vlm_shared_gpu.py'),'--worker-config',str(config)] | |
| with (LOCAL/'training.log').open('ab') as log: | |
| child=subprocess.Popen(command,cwd=SOURCE,env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) | |
| save(LOCAL/'launch_state.json',dict(pid=child.pid,supervisor_pid=os.getpid(),time=time.time(), | |
| command=command,gpu=GPU,status='started',initialization='official_omni_random_action')) | |
| thread.start() | |
| try: | |
| while child.poll() is None:time.sleep(5) | |
| save(LOCAL/'exit_status.json',dict(returncode=child.returncode,time=time.time())) | |
| if child.returncode:raise RuntimeError(f'Training failed with {child.returncode}; inspect training.log') | |
| finally: | |
| if child.poll() is None: | |
| os.killpg(child.pid,signal.SIGTERM) | |
| try:child.wait(timeout=45) | |
| except subprocess.TimeoutExpired:os.killpg(child.pid,signal.SIGKILL);child.wait() | |
| stop.set();thread.join(timeout=5) | |
| if __name__=='__main__':main() | |