# +在当前三档看板中接入完整 OMG Text1024 / Movens-Intent2048;所有输出独立保存。 import argparse import fcntl import hashlib import json import math import os from pathlib import Path import re import shutil import subprocess import sys import time ROOT = Path(__file__).resolve().parents[2] GROUP = ROOT / 'scale/experiments/fresh_mxl_20261009' OUT = GROUP / 'benchmarks' OLD = ROOT.parents[2] / 'buwendong/user/tmp/train_dashboard_20260930' USER = OLD.parents[1] OMG = USER / 'code/OMG' BENCH = USER / 'code/OmniHM_bench' ENCODER = USER / 'code/omnihm_eval_encoder' PSI_PY = ROOT.parent / 'Psi0-yth-v26/.venv-psi/bin/python' OMG_PY = USER / 'envs/OMG/bin/python' POSE = USER / 'code/OmniHM-Zero_train_40d/assets/initial_pose/standard_standing.json' V13 = ENCODER / 'runs/finetune_v13_20260920/best.pt' COLUMNS = ['FID', 'Matching', 'R@1', 'R@2', 'R@3', 'Diversity', 'C-Slide', 'Jerk', 'MPJPE', 'g-MPJPE', 'E_vel', 'E_acc', 'Fall', 'J-Limit'] def read(path): return json.loads(path.read_text()) if path.is_file() else {} def save(path, value): path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + '.tmp') temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n') temporary.replace(path) def digest(path): result = hashlib.sha256() with path.open('rb') as handle: for block in iter(lambda: handle.read(8 * 1024 * 1024), b''): result.update(block) return result.hexdigest() def dashboard_tables(): tables = {bench: dict(label=label, columns=COLUMNS, rows=[]) for bench, label in [('omg', 'OMG · Text 1024'), ('intent', 'Movens-Intent · Text 2048 · v13')]} for path in sorted((OUT / 'jobs').glob('*/job.json')): try: job = read(path) for bench, table in tables.items(): work = path.parent / bench status, report, protocol = read(work / 'status.json'), read(work / 'report.json'), read(work / 'protocol.json') complete = (report.get('status') == 'complete' and report.get('size') == job['size'] and report.get('checkpoint_sha256') == job['checkpoint_sha256'] and report.get('checkpoint_step') == job['step'] and report.get('columns') == COLUMNS and report.get('protocol_sha256') == digest(work / 'protocol.json') and len(report.get('values', [])) == len(COLUMNS) and all(isinstance(v, (int, float)) and math.isfinite(v) for v in report['values'])) table['rows'].append(dict(size=job['size'], step=job['step'], status='complete' if complete else status.get('status', 'queued'), phase=status.get('phase', 'waiting'), generated=status.get('samples_complete', 0), total=protocol.get('num_samples', 0), tracked=status.get('tracker_samples_complete', 0), updated_at=status.get('updated_at'), error=status.get('error'), values=report['values'] if complete else None, init_policy=protocol.get('init_policy'), evaluator=protocol.get('evaluator'), cfg_scale=protocol.get('cfg_scale'), cfg_interval=protocol.get('cfg_interval'))) except (OSError, ValueError, KeyError, TypeError) as error: for table in tables.values(): table.setdefault('errors', []).append(str(error)) return dict(tables=list(tables.values()), worker=read(OUT / 'worker.json')) def prepare(size, pointer): source = Path(pointer['checkpoint']) marker = read(source / 'BACKUP_COMPLETE.json') assert (source / 'checkpoint_complete.json').is_file() and marker['step'] == pointer['step'] info = marker['files']['model.safetensors'] assert (source / 'model.safetensors').stat().st_size == info['bytes'] jobdir = OUT / 'jobs' / f'{size}_ckpt_{pointer["step"]}_{info["sha256"][:12]}' job = read(jobdir / 'job.json') if job: return jobdir snapshot = jobdir / 'snapshot' destination = snapshot / 'checkpoints' / source.name destination.mkdir(parents=True, exist_ok=True) # +只硬链接 model 权重;源目录轮换删除时本评测仍可读取同一 inode,不复制 optimizer。 if not (destination / 'model.safetensors').is_file(): os.link(source / 'model.safetensors', destination / 'model.safetensors') assert (destination / 'model.safetensors').stat().st_size == info['bytes'] for filename in ('run_config.json', 'dataset_statistics.json'): shutil.copy2(Path(pointer['run']) / filename, snapshot / filename) sys.path.insert(0, str(OLD)) from omg_gt_common import checked_inputs, INPUTS, REFERENCE, MANIFEST, EVALUATOR from intent_standing_common import checked_cohort _, inputs = checked_inputs() conditions, reference, _, _, _ = checked_cohort() script_paths = [Path(__file__), ROOT / 'scale/script/scale_benchmark_generate.py', ROOT / 'src/psi/models/psi0.py', ROOT / 'src/psi/data/seed/motion_repr40.py', ROOT / 'src/psi/data/seed/progress.py', ROOT / 'src/psi/data/seed/text_prompt.py', BENCH / 'scripts/_shared/zero_runtime.py'] common = dict(size=size, checkpoint_step=pointer['step'], checkpoint=str(destination / 'model.safetensors'), checkpoint_sha256=info['sha256'], code_root=str(ROOT), seed=0, cfg_scale=1.5, cfg_interval='0.2,0.8', denoising_steps=10, native_fps=50, artifact_fps=30, smoothing='none', gt_each_chunk=False, ground_alignment='sequence_constant_min_robot_collision_distance', scripts_sha256={str(p): digest(p) for p in script_paths}, run_config_sha256=digest(snapshot / 'run_config.json'), statistics_sha256=digest(snapshot / 'dataset_statistics.json')) for bench, settings in [('omg', dict(num_samples=1024, init_policy='gt_history_last', init_inputs=str(INPUTS), init_inputs_sha256=digest(INPUTS), evaluator=str(EVALUATOR), evaluator_sha256=digest(EVALUATOR), reference=str(REFERENCE), reference_sha256=digest(REFERENCE), manifest=str(MANIFEST), manifest_sha256=digest(MANIFEST), temporal_protocol='61 frames including initial; score future frames1..60')), ('intent', dict(num_samples=2048, init_policy='standard_standing', init_inputs=str(POSE), init_inputs_sha256=digest(POSE), evaluator=str(V13), evaluator_sha256=digest(V13), v13_evaluator_sha256=digest(V13), condition=str(conditions), condition_sha256=digest(conditions), reference=str(reference), reference_sha256=digest(reference), retrieval_seed=0, retrieval_batch_size=32, temporal_protocol='variable valid lengths; pad450 with final frame, preserve reference valid mask'))]: work = jobdir / bench scripts = [OLD / ('omg_gt_generate.py' if bench == 'omg' else 'intent_standing_generate.py'), OLD / ('omg_gt_pack.py' if bench == 'omg' else 'intent_standing_pack.py')] protocol = dict(common, **settings, benchmark=bench, generation_scripts_sha256={str(p): digest(p) for p in scripts}) save(work / 'protocol.json', protocol) save(work / 'status.json', dict(status='queued', phase='waiting', samples_complete=0, samples_total=settings['num_samples'], updated_at=time.time())) save(jobdir / 'job.json', dict(size=size, step=pointer['step'], checkpoint_sha256=info['sha256'], source_checkpoint=str(source), snapshot=str(snapshot), created_at=time.time())) # The optional manual OMG smoke entry points at the first newly queued protocol. current = OUT / 'current_omg_work' if not current.exists(): current.symlink_to(jobdir / 'omg', target_is_directory=True) return jobdir def execute(command, cwd, env, work, phase, role=None): status = read(work / 'status.json') status.update(status='running', phase=phase, updated_at=time.time(), error=None) save(work / 'status.json', status) if role: command = [str(PSI_PY), '-B', '-u', str(OLD / 'gpu_eval_slots.py'), '--role', role, '--capacity', '3' if role == 'generation' else '1', '--'] + command log_path = work / f'{phase}.log' with log_path.open('ab') as log: process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT) try: while True: status.update(pid=process.pid, updated_at=time.time()) sampledir = work / read(work / 'protocol.json')['benchmark'] / 'samples' status['samples_complete'] = len(list(sampledir.glob('*.npz'))) with log_path.open('rb') as reader: reader.seek(max(0, log_path.stat().st_size - 32768)) tail = reader.read().decode(errors='replace') tracked = re.findall(r'Tracker execute:[^\r\n]*?\s(\d+)/(\d+)', tail) if tracked: status['tracker_samples_complete'] = int(tracked[-1][0]) save(work / 'status.json', status) try: code = process.wait(timeout=10) if code: raise RuntimeError(f'{phase} exited {code}; see {log_path}') break except subprocess.TimeoutExpired: pass finally: if process.poll() is None: process.terminate() process.wait(timeout=30) def evaluate(jobdir, gpu): job = read(jobdir / 'job.json') weights = Path(job['snapshot']) / 'checkpoints' / f'ckpt_{job["step"]}' / 'model.safetensors' if not read(jobdir / 'verified.json'): save(OUT / 'worker.json', dict(status='running', phase='verify_checkpoint', size=job['size'], step=job['step'], gpu=gpu, updated_at=time.time())) assert digest(weights) == job['checkpoint_sha256'], 'Pinned checkpoint checksum mismatch' save(jobdir / 'verified.json', dict(checkpoint_sha256=job['checkpoint_sha256'])) for bench in ('omg', 'intent'): work = jobdir / bench if read(work / 'report.json').get('status') == 'complete': continue protocol = read(work / 'protocol.json') save(OUT / 'worker.json', dict(status='running', phase='evaluate', size=job['size'], step=job['step'], bench=bench, gpu=gpu, updated_at=time.time())) env = os.environ.copy() env.update(CUDA_VISIBLE_DEVICES=gpu, PYTHONNOUSERSITE='1', PYTHONDONTWRITEBYTECODE='1', HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', TOKENIZERS_PARALLELISM='false', OMP_NUM_THREADS='2', MKL_NUM_THREADS='2', OPENBLAS_NUM_THREADS='2', LD_LIBRARY_PATH=str(PSI_PY.parent.parent / 'lib') + ':/usr/local/cuda/lib64', OMNIHM_TRACKER_WORKERS='2') for key in ('PYTHONHOME', 'LD_PRELOAD', 'OMG_LOCAL_SRC', 'PSI_CODE_ROOT', 'PSI_PRETRAIN_BACKBONE_PATH'): env.pop(key, None) generated = work / bench / 'generated_qpos.npz' if not generated.is_file(): execute([str(PSI_PY), '-B', '-u', str(ROOT / 'scale/script/scale_benchmark_generate.py'), '--bench', bench, '--work', str(work)], ROOT, env, work, 'generate', 'generation') execute([str(OMG_PY), '-B', '-u', str(OLD / ('omg_gt_pack.py' if bench == 'omg' else 'intent_standing_pack.py')), '--work', str(work)], OLD, env, work, 'pack') cache = work / 'metrics' cache.mkdir(exist_ok=True) metric_env = dict(env, PYTHONPATH=str(BENCH) + ':' + str(OMG / 'src'), LD_LIBRARY_PATH=str(OMG_PY.parent.parent / 'lib') + ':/usr/local/cuda/lib64', OMG_DATA_ROOT=str(USER.parent / 'videogen/OMG/data/OMG-Data'), OMG_MODELS_ROOT=str(OMG / 'models'), OMG_T5_3B_MODEL=str(OMG / 'models/t5-3b-local'), HF_HOME=str(USER.parent / 'videogen/OMG/hf_cache')) sys.path.insert(0, str(BENCH / 'scripts')) from _shared.tracker_config import TRACKER_OPTIONS if bench == 'omg': paper_path = cache / 'paper_row.json' if not (cache / 'benchmark.json').is_file(): command = [str(OMG_PY), '-B', '-u', str(OLD / 'omg_text_paper_metrics_parallel.py'), '--mode', 'text', '--generated_qpos', str(generated), '--reference_qpos', protocol['reference'], '--samples_path', protocol['manifest'], '--exp', '100m_omnimodal', '--data', 'omg_data_lerobot_omnimodal', '--output_dir', str(cache), '--num_frames', '60', '--batch-size', '64', '--seed', '0', '--label', f'Movens-{job["size"]} ckpt{job["step"]}', '--motion_key', 'body_pos_local', '--evaluator_checkpoint', protocol['evaluator'], '--enable_text_metrics', '--t5_3b_model', str(OMG / 'models/t5-3b-local'), '--text_cache_dir', str(ENCODER / 'omg_bench/text_cache'), '--no-transition-metrics', '--device', 'cuda', '--tracker-executed', '--tracker-num-samples', '1024', '--tracker-sample-seed', '0'] for key, value in TRACKER_OPTIONS.items(): if key != 'tracker-steps': command += ['--' + key, str(value)] execute(command, OMG, metric_env, work, 'metrics_tracking', 'metrics') if not paper_path.is_file(): execute([str(OMG_PY), '-B', str(OMG / 'scripts/extract_omg_paper_row.py'), '--mode', 'text', '--evaluation-dir', str(cache), '--xml-path', str(TRACKER_OPTIONS['tracker-robot-xml']), '--output-path', str(paper_path)], OMG, metric_env, work, 'paper_row') else: paper_path = cache / 'paper_metrics.json' if not paper_path.is_file(): options = dict(task='text', generated_path=generated, reference_path=protocol['reference'], condition_path=protocol['condition'], output_dir=cache, omg_root=OMG, expected_frames=450, expected_fps=30, device='cuda', batch_size=32, validate_only=0, kinematics_path=OMG / 'assets/robots/g1/g1_kinematics.json', contact_height_threshold=.12, contact_penetration_tolerance=.02, evaluator_checkpoint=V13, retrieval_evaluator_checkpoint=V13, raw_text_cache=BENCH / 'data/text_eval_2048/v13_t5_raw.npz', motion_key='body_pos_local', t5_model=OMG / 'models/t5-3b-local', text_cache_dir=cache / 'text_cache', text_batch_size=16, seed=0, beat_direction='music_to_motion', beat_threshold=.5, sigma_frames=3., min_motion_beat_distance_seconds=.25, tracker_mode='run', tracker_artifact=cache / 'unused.npz') options.update({k.replace('-', '_'): v for k, v in TRACKER_OPTIONS.items()}) command = [str(OMG_PY), '-B', '-u', str(OLD / 'evaluate_parallel_tracking.py')] for key, value in options.items(): command += ['--' + key.replace('_', '-'), str(value)] execute(command, BENCH, metric_env, work, 'metrics_tracking', 'metrics') paper = read(paper_path) assert paper['columns'] == COLUMNS assert all(math.isfinite(v) for v in paper['values']) and len(paper['values']) == 14 if bench == 'intent': assert paper['complete'] is True full = read(cache / 'metrics.json') assert full['tracker_executed']['num_samples'] == 2048 and full['tracker_executed']['paper_eligible'] is True text = full['generated_target']['text'] assert text['evaluator_checkpoint_sha256'] == text['retrieval_evaluator_checkpoint_sha256'] == protocol['evaluator_sha256'] assert text['retrieval_seed'] == 0 and text['retrieval_seeds'] == [0] else: assert paper['num_samples'] == 1024 full = read(cache / 'benchmark.json') assert full['tracker_executed']['num_samples'] == 1024 assert full['tracker_executed']['selected_indices'] == list(range(1024)) report = dict(status='complete', size=job['size'], checkpoint_step=job['step'], checkpoint_sha256=job['checkpoint_sha256'], protocol_sha256=digest(work / 'protocol.json'), columns=COLUMNS, values=paper['values'], paper_path=str(paper_path), paper_sha256=digest(paper_path), generated_sha256=digest(generated), num_samples=protocol['num_samples'], completed_at=time.time()) save(work / 'report.json', report) save(work / 'status.json', dict(status='complete', phase='complete', samples_complete=protocol['num_samples'], tracker_samples_complete=protocol['num_samples'], updated_at=time.time())) print(job['size'], job['step'], bench, paper['values'], flush=True) # +两个 bench 都完成后只删除本队列创建的权重硬链接,保留协议、生成数据和全部指标。 weights.unlink(missing_ok=True) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--gpu', default='1') parser.add_argument('--once', action='store_true') parser.add_argument('--prepare-only', action='store_true') args = parser.parse_args() OUT.mkdir(parents=True, exist_ok=True) lock = (OUT / 'queue.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) while True: # Finish the first queued checkpoint of every size before pinning its next latest backup. for size, part in [('S', 'split_sm'), ('M', 'split_sm'), ('XL', 'split_xl')]: unfinished = [p.parent for p in sorted((OUT / 'jobs').glob(f'{size}_*/job.json')) if any(read(p.parent / b / 'report.json').get('status') != 'complete' for b in ('omg', 'intent'))] if not unfinished: try: prepare(size, read(GROUP / part / 'backups' / size / 'latest.json')) except Exception as error: print('Prepare failed:', size, repr(error), flush=True) save(OUT / f'prepare_error_{size}.json', dict(error=repr(error), updated_at=time.time())) if args.prepare_only: print(json.dumps(dashboard_tables(), ensure_ascii=False, indent=2)) break pending = [p.parent for p in sorted((OUT / 'jobs').glob('*/job.json')) if any(read(p.parent / b / 'report.json').get('status') != 'complete' for b in ('omg', 'intent'))] pending.sort(key=lambda p: read(p / 'job.json')['created_at']) for jobdir in pending: try: evaluate(jobdir, args.gpu) except Exception as error: print('Evaluation failed:', jobdir.name, repr(error), flush=True) for bench in ('omg', 'intent'): status = read(jobdir / bench / 'status.json') if status.get('status') == 'running': save(jobdir / bench / 'status.json', dict(status, status='error', error=repr(error), updated_at=time.time())) if args.once: break save(OUT / 'worker.json', dict(status='idle', gpu=args.gpu, updated_at=time.time())) time.sleep(60)