Download scale/script/scale_benchmarks.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 19.6 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/scale_benchmarks.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/script/scale_benchmarks.py
-
curl -L -o scale_benchmarks.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/script/scale_benchmarks.py
19.6 kB
| # <modify>+在当前三档看板中接入完整 OMG Text1024 / Movens-Intent2048;所有输出独立保存。 | |
| import argparse | |
| import fcntl | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import re | |
| import shutil | |
| import subprocess | |
| import sys | |
| import time | |
| ROOT = Path(__file__).resolve().parents[2] | |
| GROUP = ROOT / 'scale/experiments/fresh_mxl_20261009' | |
| OUT = GROUP / 'benchmarks' | |
| OLD = ROOT.parents[2] / 'buwendong/user/tmp/train_dashboard_20260930' | |
| USER = OLD.parents[1] | |
| OMG = USER / 'code/OMG' | |
| BENCH = USER / 'code/OmniHM_bench' | |
| ENCODER = USER / 'code/omnihm_eval_encoder' | |
| PSI_PY = ROOT.parent / 'Psi0-yth-v26/.venv-psi/bin/python' | |
| OMG_PY = USER / 'envs/OMG/bin/python' | |
| POSE = USER / 'code/OmniHM-Zero_train_40d/assets/initial_pose/standard_standing.json' | |
| V13 = ENCODER / 'runs/finetune_v13_20260920/best.pt' | |
| COLUMNS = ['FID', 'Matching', 'R@1', 'R@2', 'R@3', 'Diversity', 'C-Slide', 'Jerk', | |
| 'MPJPE', 'g-MPJPE', 'E_vel', 'E_acc', 'Fall', 'J-Limit'] | |
| def read(path): | |
| return json.loads(path.read_text()) if path.is_file() else {} | |
| def save(path, value): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(path.suffix + '.tmp') | |
| temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n') | |
| temporary.replace(path) | |
| def digest(path): | |
| result = hashlib.sha256() | |
| with path.open('rb') as handle: | |
| for block in iter(lambda: handle.read(8 * 1024 * 1024), b''): | |
| result.update(block) | |
| return result.hexdigest() | |
| def dashboard_tables(): | |
| tables = {bench: dict(label=label, columns=COLUMNS, rows=[]) for bench, label in | |
| [('omg', 'OMG · Text 1024'), ('intent', 'Movens-Intent · Text 2048 · v13')]} | |
| for path in sorted((OUT / 'jobs').glob('*/job.json')): | |
| try: | |
| job = read(path) | |
| for bench, table in tables.items(): | |
| work = path.parent / bench | |
| status, report, protocol = read(work / 'status.json'), read(work / 'report.json'), read(work / 'protocol.json') | |
| complete = (report.get('status') == 'complete' and report.get('size') == job['size'] | |
| and report.get('checkpoint_sha256') == job['checkpoint_sha256'] | |
| and report.get('checkpoint_step') == job['step'] and report.get('columns') == COLUMNS | |
| and report.get('protocol_sha256') == digest(work / 'protocol.json') | |
| and len(report.get('values', [])) == len(COLUMNS) | |
| and all(isinstance(v, (int, float)) and math.isfinite(v) for v in report['values'])) | |
| table['rows'].append(dict(size=job['size'], step=job['step'], | |
| status='complete' if complete else status.get('status', 'queued'), phase=status.get('phase', 'waiting'), | |
| generated=status.get('samples_complete', 0), total=protocol.get('num_samples', 0), | |
| tracked=status.get('tracker_samples_complete', 0), updated_at=status.get('updated_at'), | |
| error=status.get('error'), values=report['values'] if complete else None, | |
| init_policy=protocol.get('init_policy'), evaluator=protocol.get('evaluator'), | |
| cfg_scale=protocol.get('cfg_scale'), cfg_interval=protocol.get('cfg_interval'))) | |
| except (OSError, ValueError, KeyError, TypeError) as error: | |
| for table in tables.values(): | |
| table.setdefault('errors', []).append(str(error)) | |
| return dict(tables=list(tables.values()), worker=read(OUT / 'worker.json')) | |
| def prepare(size, pointer): | |
| source = Path(pointer['checkpoint']) | |
| marker = read(source / 'BACKUP_COMPLETE.json') | |
| assert (source / 'checkpoint_complete.json').is_file() and marker['step'] == pointer['step'] | |
| info = marker['files']['model.safetensors'] | |
| assert (source / 'model.safetensors').stat().st_size == info['bytes'] | |
| jobdir = OUT / 'jobs' / f'{size}_ckpt_{pointer["step"]}_{info["sha256"][:12]}' | |
| job = read(jobdir / 'job.json') | |
| if job: | |
| return jobdir | |
| snapshot = jobdir / 'snapshot' | |
| destination = snapshot / 'checkpoints' / source.name | |
| destination.mkdir(parents=True, exist_ok=True) | |
| # <modify>+只硬链接 model 权重;源目录轮换删除时本评测仍可读取同一 inode,不复制 optimizer。 | |
| if not (destination / 'model.safetensors').is_file(): | |
| os.link(source / 'model.safetensors', destination / 'model.safetensors') | |
| assert (destination / 'model.safetensors').stat().st_size == info['bytes'] | |
| for filename in ('run_config.json', 'dataset_statistics.json'): | |
| shutil.copy2(Path(pointer['run']) / filename, snapshot / filename) | |
| sys.path.insert(0, str(OLD)) | |
| from omg_gt_common import checked_inputs, INPUTS, REFERENCE, MANIFEST, EVALUATOR | |
| from intent_standing_common import checked_cohort | |
| _, inputs = checked_inputs() | |
| conditions, reference, _, _, _ = checked_cohort() | |
| script_paths = [Path(__file__), ROOT / 'scale/script/scale_benchmark_generate.py', | |
| ROOT / 'src/psi/models/psi0.py', ROOT / 'src/psi/data/seed/motion_repr40.py', | |
| ROOT / 'src/psi/data/seed/progress.py', ROOT / 'src/psi/data/seed/text_prompt.py', | |
| BENCH / 'scripts/_shared/zero_runtime.py'] | |
| common = dict(size=size, checkpoint_step=pointer['step'], checkpoint=str(destination / 'model.safetensors'), | |
| checkpoint_sha256=info['sha256'], code_root=str(ROOT), seed=0, cfg_scale=1.5, cfg_interval='0.2,0.8', | |
| denoising_steps=10, native_fps=50, artifact_fps=30, smoothing='none', gt_each_chunk=False, | |
| ground_alignment='sequence_constant_min_robot_collision_distance', | |
| scripts_sha256={str(p): digest(p) for p in script_paths}, run_config_sha256=digest(snapshot / 'run_config.json'), | |
| statistics_sha256=digest(snapshot / 'dataset_statistics.json')) | |
| for bench, settings in [('omg', dict(num_samples=1024, init_policy='gt_history_last', init_inputs=str(INPUTS), | |
| init_inputs_sha256=digest(INPUTS), evaluator=str(EVALUATOR), evaluator_sha256=digest(EVALUATOR), | |
| reference=str(REFERENCE), reference_sha256=digest(REFERENCE), manifest=str(MANIFEST), | |
| manifest_sha256=digest(MANIFEST), temporal_protocol='61 frames including initial; score future frames1..60')), | |
| ('intent', dict(num_samples=2048, init_policy='standard_standing', init_inputs=str(POSE), | |
| init_inputs_sha256=digest(POSE), evaluator=str(V13), evaluator_sha256=digest(V13), | |
| v13_evaluator_sha256=digest(V13), condition=str(conditions), condition_sha256=digest(conditions), | |
| reference=str(reference), reference_sha256=digest(reference), retrieval_seed=0, retrieval_batch_size=32, | |
| temporal_protocol='variable valid lengths; pad450 with final frame, preserve reference valid mask'))]: | |
| work = jobdir / bench | |
| scripts = [OLD / ('omg_gt_generate.py' if bench == 'omg' else 'intent_standing_generate.py'), | |
| OLD / ('omg_gt_pack.py' if bench == 'omg' else 'intent_standing_pack.py')] | |
| protocol = dict(common, **settings, benchmark=bench, | |
| generation_scripts_sha256={str(p): digest(p) for p in scripts}) | |
| save(work / 'protocol.json', protocol) | |
| save(work / 'status.json', dict(status='queued', phase='waiting', samples_complete=0, | |
| samples_total=settings['num_samples'], updated_at=time.time())) | |
| save(jobdir / 'job.json', dict(size=size, step=pointer['step'], checkpoint_sha256=info['sha256'], | |
| source_checkpoint=str(source), snapshot=str(snapshot), created_at=time.time())) | |
| # The optional manual OMG smoke entry points at the first newly queued protocol. | |
| current = OUT / 'current_omg_work' | |
| if not current.exists(): | |
| current.symlink_to(jobdir / 'omg', target_is_directory=True) | |
| return jobdir | |
| def execute(command, cwd, env, work, phase, role=None): | |
| status = read(work / 'status.json') | |
| status.update(status='running', phase=phase, updated_at=time.time(), error=None) | |
| save(work / 'status.json', status) | |
| if role: | |
| command = [str(PSI_PY), '-B', '-u', str(OLD / 'gpu_eval_slots.py'), '--role', role, | |
| '--capacity', '3' if role == 'generation' else '1', '--'] + command | |
| log_path = work / f'{phase}.log' | |
| with log_path.open('ab') as log: | |
| process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT) | |
| try: | |
| while True: | |
| status.update(pid=process.pid, updated_at=time.time()) | |
| sampledir = work / read(work / 'protocol.json')['benchmark'] / 'samples' | |
| status['samples_complete'] = len(list(sampledir.glob('*.npz'))) | |
| with log_path.open('rb') as reader: | |
| reader.seek(max(0, log_path.stat().st_size - 32768)) | |
| tail = reader.read().decode(errors='replace') | |
| tracked = re.findall(r'Tracker execute:[^\r\n]*?\s(\d+)/(\d+)', tail) | |
| if tracked: | |
| status['tracker_samples_complete'] = int(tracked[-1][0]) | |
| save(work / 'status.json', status) | |
| try: | |
| code = process.wait(timeout=10) | |
| if code: | |
| raise RuntimeError(f'{phase} exited {code}; see {log_path}') | |
| break | |
| except subprocess.TimeoutExpired: | |
| pass | |
| finally: | |
| if process.poll() is None: | |
| process.terminate() | |
| process.wait(timeout=30) | |
| def evaluate(jobdir, gpu): | |
| job = read(jobdir / 'job.json') | |
| weights = Path(job['snapshot']) / 'checkpoints' / f'ckpt_{job["step"]}' / 'model.safetensors' | |
| if not read(jobdir / 'verified.json'): | |
| save(OUT / 'worker.json', dict(status='running', phase='verify_checkpoint', size=job['size'], | |
| step=job['step'], gpu=gpu, updated_at=time.time())) | |
| assert digest(weights) == job['checkpoint_sha256'], 'Pinned checkpoint checksum mismatch' | |
| save(jobdir / 'verified.json', dict(checkpoint_sha256=job['checkpoint_sha256'])) | |
| for bench in ('omg', 'intent'): | |
| work = jobdir / bench | |
| if read(work / 'report.json').get('status') == 'complete': | |
| continue | |
| protocol = read(work / 'protocol.json') | |
| save(OUT / 'worker.json', dict(status='running', phase='evaluate', size=job['size'], step=job['step'], | |
| bench=bench, gpu=gpu, updated_at=time.time())) | |
| env = os.environ.copy() | |
| env.update(CUDA_VISIBLE_DEVICES=gpu, PYTHONNOUSERSITE='1', PYTHONDONTWRITEBYTECODE='1', | |
| HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', TOKENIZERS_PARALLELISM='false', | |
| OMP_NUM_THREADS='2', MKL_NUM_THREADS='2', OPENBLAS_NUM_THREADS='2', | |
| LD_LIBRARY_PATH=str(PSI_PY.parent.parent / 'lib') + ':/usr/local/cuda/lib64', | |
| OMNIHM_TRACKER_WORKERS='2') | |
| for key in ('PYTHONHOME', 'LD_PRELOAD', 'OMG_LOCAL_SRC', 'PSI_CODE_ROOT', 'PSI_PRETRAIN_BACKBONE_PATH'): | |
| env.pop(key, None) | |
| generated = work / bench / 'generated_qpos.npz' | |
| if not generated.is_file(): | |
| execute([str(PSI_PY), '-B', '-u', str(ROOT / 'scale/script/scale_benchmark_generate.py'), | |
| '--bench', bench, '--work', str(work)], ROOT, env, work, 'generate', 'generation') | |
| execute([str(OMG_PY), '-B', '-u', str(OLD / ('omg_gt_pack.py' if bench == 'omg' else 'intent_standing_pack.py')), | |
| '--work', str(work)], OLD, env, work, 'pack') | |
| cache = work / 'metrics' | |
| cache.mkdir(exist_ok=True) | |
| metric_env = dict(env, PYTHONPATH=str(BENCH) + ':' + str(OMG / 'src'), | |
| LD_LIBRARY_PATH=str(OMG_PY.parent.parent / 'lib') + ':/usr/local/cuda/lib64', | |
| OMG_DATA_ROOT=str(USER.parent / 'videogen/OMG/data/OMG-Data'), OMG_MODELS_ROOT=str(OMG / 'models'), | |
| OMG_T5_3B_MODEL=str(OMG / 'models/t5-3b-local'), HF_HOME=str(USER.parent / 'videogen/OMG/hf_cache')) | |
| sys.path.insert(0, str(BENCH / 'scripts')) | |
| from _shared.tracker_config import TRACKER_OPTIONS | |
| if bench == 'omg': | |
| paper_path = cache / 'paper_row.json' | |
| if not (cache / 'benchmark.json').is_file(): | |
| command = [str(OMG_PY), '-B', '-u', str(OLD / 'omg_text_paper_metrics_parallel.py'), | |
| '--mode', 'text', '--generated_qpos', str(generated), '--reference_qpos', protocol['reference'], | |
| '--samples_path', protocol['manifest'], '--exp', '100m_omnimodal', '--data', 'omg_data_lerobot_omnimodal', | |
| '--output_dir', str(cache), '--num_frames', '60', '--batch-size', '64', '--seed', '0', | |
| '--label', f'Movens-{job["size"]} ckpt{job["step"]}', '--motion_key', 'body_pos_local', | |
| '--evaluator_checkpoint', protocol['evaluator'], '--enable_text_metrics', | |
| '--t5_3b_model', str(OMG / 'models/t5-3b-local'), '--text_cache_dir', str(ENCODER / 'omg_bench/text_cache'), | |
| '--no-transition-metrics', '--device', 'cuda', '--tracker-executed', '--tracker-num-samples', '1024', | |
| '--tracker-sample-seed', '0'] | |
| for key, value in TRACKER_OPTIONS.items(): | |
| if key != 'tracker-steps': | |
| command += ['--' + key, str(value)] | |
| execute(command, OMG, metric_env, work, 'metrics_tracking', 'metrics') | |
| if not paper_path.is_file(): | |
| execute([str(OMG_PY), '-B', str(OMG / 'scripts/extract_omg_paper_row.py'), '--mode', 'text', | |
| '--evaluation-dir', str(cache), '--xml-path', str(TRACKER_OPTIONS['tracker-robot-xml']), | |
| '--output-path', str(paper_path)], OMG, metric_env, work, 'paper_row') | |
| else: | |
| paper_path = cache / 'paper_metrics.json' | |
| if not paper_path.is_file(): | |
| options = dict(task='text', generated_path=generated, reference_path=protocol['reference'], | |
| condition_path=protocol['condition'], output_dir=cache, omg_root=OMG, expected_frames=450, | |
| expected_fps=30, device='cuda', batch_size=32, validate_only=0, | |
| kinematics_path=OMG / 'assets/robots/g1/g1_kinematics.json', contact_height_threshold=.12, | |
| contact_penetration_tolerance=.02, evaluator_checkpoint=V13, retrieval_evaluator_checkpoint=V13, | |
| raw_text_cache=BENCH / 'data/text_eval_2048/v13_t5_raw.npz', motion_key='body_pos_local', | |
| t5_model=OMG / 'models/t5-3b-local', text_cache_dir=cache / 'text_cache', text_batch_size=16, | |
| seed=0, beat_direction='music_to_motion', beat_threshold=.5, sigma_frames=3., | |
| min_motion_beat_distance_seconds=.25, tracker_mode='run', tracker_artifact=cache / 'unused.npz') | |
| options.update({k.replace('-', '_'): v for k, v in TRACKER_OPTIONS.items()}) | |
| command = [str(OMG_PY), '-B', '-u', str(OLD / 'evaluate_parallel_tracking.py')] | |
| for key, value in options.items(): | |
| command += ['--' + key.replace('_', '-'), str(value)] | |
| execute(command, BENCH, metric_env, work, 'metrics_tracking', 'metrics') | |
| paper = read(paper_path) | |
| assert paper['columns'] == COLUMNS | |
| assert all(math.isfinite(v) for v in paper['values']) and len(paper['values']) == 14 | |
| if bench == 'intent': | |
| assert paper['complete'] is True | |
| full = read(cache / 'metrics.json') | |
| assert full['tracker_executed']['num_samples'] == 2048 and full['tracker_executed']['paper_eligible'] is True | |
| text = full['generated_target']['text'] | |
| assert text['evaluator_checkpoint_sha256'] == text['retrieval_evaluator_checkpoint_sha256'] == protocol['evaluator_sha256'] | |
| assert text['retrieval_seed'] == 0 and text['retrieval_seeds'] == [0] | |
| else: | |
| assert paper['num_samples'] == 1024 | |
| full = read(cache / 'benchmark.json') | |
| assert full['tracker_executed']['num_samples'] == 1024 | |
| assert full['tracker_executed']['selected_indices'] == list(range(1024)) | |
| report = dict(status='complete', size=job['size'], checkpoint_step=job['step'], | |
| checkpoint_sha256=job['checkpoint_sha256'], protocol_sha256=digest(work / 'protocol.json'), | |
| columns=COLUMNS, values=paper['values'], paper_path=str(paper_path), paper_sha256=digest(paper_path), | |
| generated_sha256=digest(generated), num_samples=protocol['num_samples'], completed_at=time.time()) | |
| save(work / 'report.json', report) | |
| save(work / 'status.json', dict(status='complete', phase='complete', samples_complete=protocol['num_samples'], | |
| tracker_samples_complete=protocol['num_samples'], updated_at=time.time())) | |
| print(job['size'], job['step'], bench, paper['values'], flush=True) | |
| # <modify>+两个 bench 都完成后只删除本队列创建的权重硬链接,保留协议、生成数据和全部指标。 | |
| weights.unlink(missing_ok=True) | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--gpu', default='1') | |
| parser.add_argument('--once', action='store_true') | |
| parser.add_argument('--prepare-only', action='store_true') | |
| args = parser.parse_args() | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| lock = (OUT / 'queue.lock').open('a') | |
| fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | |
| while True: | |
| # Finish the first queued checkpoint of every size before pinning its next latest backup. | |
| for size, part in [('S', 'split_sm'), ('M', 'split_sm'), ('XL', 'split_xl')]: | |
| unfinished = [p.parent for p in sorted((OUT / 'jobs').glob(f'{size}_*/job.json')) | |
| if any(read(p.parent / b / 'report.json').get('status') != 'complete' for b in ('omg', 'intent'))] | |
| if not unfinished: | |
| try: | |
| prepare(size, read(GROUP / part / 'backups' / size / 'latest.json')) | |
| except Exception as error: | |
| print('Prepare failed:', size, repr(error), flush=True) | |
| save(OUT / f'prepare_error_{size}.json', dict(error=repr(error), updated_at=time.time())) | |
| if args.prepare_only: | |
| print(json.dumps(dashboard_tables(), ensure_ascii=False, indent=2)) | |
| break | |
| pending = [p.parent for p in sorted((OUT / 'jobs').glob('*/job.json')) | |
| if any(read(p.parent / b / 'report.json').get('status') != 'complete' for b in ('omg', 'intent'))] | |
| pending.sort(key=lambda p: read(p / 'job.json')['created_at']) | |
| for jobdir in pending: | |
| try: | |
| evaluate(jobdir, args.gpu) | |
| except Exception as error: | |
| print('Evaluation failed:', jobdir.name, repr(error), flush=True) | |
| for bench in ('omg', 'intent'): | |
| status = read(jobdir / bench / 'status.json') | |
| if status.get('status') == 'running': | |
| save(jobdir / bench / 'status.json', dict(status, status='error', error=repr(error), updated_at=time.time())) | |
| if args.once: | |
| break | |
| save(OUT / 'worker.json', dict(status='idle', gpu=args.gpu, updated_at=time.time())) | |
| time.sleep(60) | |