code-Movens-Zero_train / scale /script /scale_benchmarks.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
13cc860 verified
Raw History Blame Contribute Delete
19.6 kB
# <modify>+在当前三档看板中接入完整 OMG Text1024 / Movens-Intent2048;所有输出独立保存。
import argparse
import fcntl
import hashlib
import json
import math
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
GROUP = ROOT / 'scale/experiments/fresh_mxl_20261009'
OUT = GROUP / 'benchmarks'
OLD = ROOT.parents[2] / 'buwendong/user/tmp/train_dashboard_20260930'
USER = OLD.parents[1]
OMG = USER / 'code/OMG'
BENCH = USER / 'code/OmniHM_bench'
ENCODER = USER / 'code/omnihm_eval_encoder'
PSI_PY = ROOT.parent / 'Psi0-yth-v26/.venv-psi/bin/python'
OMG_PY = USER / 'envs/OMG/bin/python'
POSE = USER / 'code/OmniHM-Zero_train_40d/assets/initial_pose/standard_standing.json'
V13 = ENCODER / 'runs/finetune_v13_20260920/best.pt'
COLUMNS = ['FID', 'Matching', 'R@1', 'R@2', 'R@3', 'Diversity', 'C-Slide', 'Jerk',
'MPJPE', 'g-MPJPE', 'E_vel', 'E_acc', 'Fall', 'J-Limit']
def read(path):
return json.loads(path.read_text()) if path.is_file() else {}
def save(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + '.tmp')
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + '\n')
temporary.replace(path)
def digest(path):
result = hashlib.sha256()
with path.open('rb') as handle:
for block in iter(lambda: handle.read(8 * 1024 * 1024), b''):
result.update(block)
return result.hexdigest()
def dashboard_tables():
tables = {bench: dict(label=label, columns=COLUMNS, rows=[]) for bench, label in
[('omg', 'OMG · Text 1024'), ('intent', 'Movens-Intent · Text 2048 · v13')]}
for path in sorted((OUT / 'jobs').glob('*/job.json')):
try:
job = read(path)
for bench, table in tables.items():
work = path.parent / bench
status, report, protocol = read(work / 'status.json'), read(work / 'report.json'), read(work / 'protocol.json')
complete = (report.get('status') == 'complete' and report.get('size') == job['size']
and report.get('checkpoint_sha256') == job['checkpoint_sha256']
and report.get('checkpoint_step') == job['step'] and report.get('columns') == COLUMNS
and report.get('protocol_sha256') == digest(work / 'protocol.json')
and len(report.get('values', [])) == len(COLUMNS)
and all(isinstance(v, (int, float)) and math.isfinite(v) for v in report['values']))
table['rows'].append(dict(size=job['size'], step=job['step'],
status='complete' if complete else status.get('status', 'queued'), phase=status.get('phase', 'waiting'),
generated=status.get('samples_complete', 0), total=protocol.get('num_samples', 0),
tracked=status.get('tracker_samples_complete', 0), updated_at=status.get('updated_at'),
error=status.get('error'), values=report['values'] if complete else None,
init_policy=protocol.get('init_policy'), evaluator=protocol.get('evaluator'),
cfg_scale=protocol.get('cfg_scale'), cfg_interval=protocol.get('cfg_interval')))
except (OSError, ValueError, KeyError, TypeError) as error:
for table in tables.values():
table.setdefault('errors', []).append(str(error))
return dict(tables=list(tables.values()), worker=read(OUT / 'worker.json'))
def prepare(size, pointer):
source = Path(pointer['checkpoint'])
marker = read(source / 'BACKUP_COMPLETE.json')
assert (source / 'checkpoint_complete.json').is_file() and marker['step'] == pointer['step']
info = marker['files']['model.safetensors']
assert (source / 'model.safetensors').stat().st_size == info['bytes']
jobdir = OUT / 'jobs' / f'{size}_ckpt_{pointer["step"]}_{info["sha256"][:12]}'
job = read(jobdir / 'job.json')
if job:
return jobdir
snapshot = jobdir / 'snapshot'
destination = snapshot / 'checkpoints' / source.name
destination.mkdir(parents=True, exist_ok=True)
# <modify>+只硬链接 model 权重;源目录轮换删除时本评测仍可读取同一 inode,不复制 optimizer。
if not (destination / 'model.safetensors').is_file():
os.link(source / 'model.safetensors', destination / 'model.safetensors')
assert (destination / 'model.safetensors').stat().st_size == info['bytes']
for filename in ('run_config.json', 'dataset_statistics.json'):
shutil.copy2(Path(pointer['run']) / filename, snapshot / filename)
sys.path.insert(0, str(OLD))
from omg_gt_common import checked_inputs, INPUTS, REFERENCE, MANIFEST, EVALUATOR
from intent_standing_common import checked_cohort
_, inputs = checked_inputs()
conditions, reference, _, _, _ = checked_cohort()
script_paths = [Path(__file__), ROOT / 'scale/script/scale_benchmark_generate.py',
ROOT / 'src/psi/models/psi0.py', ROOT / 'src/psi/data/seed/motion_repr40.py',
ROOT / 'src/psi/data/seed/progress.py', ROOT / 'src/psi/data/seed/text_prompt.py',
BENCH / 'scripts/_shared/zero_runtime.py']
common = dict(size=size, checkpoint_step=pointer['step'], checkpoint=str(destination / 'model.safetensors'),
checkpoint_sha256=info['sha256'], code_root=str(ROOT), seed=0, cfg_scale=1.5, cfg_interval='0.2,0.8',
denoising_steps=10, native_fps=50, artifact_fps=30, smoothing='none', gt_each_chunk=False,
ground_alignment='sequence_constant_min_robot_collision_distance',
scripts_sha256={str(p): digest(p) for p in script_paths}, run_config_sha256=digest(snapshot / 'run_config.json'),
statistics_sha256=digest(snapshot / 'dataset_statistics.json'))
for bench, settings in [('omg', dict(num_samples=1024, init_policy='gt_history_last', init_inputs=str(INPUTS),
init_inputs_sha256=digest(INPUTS), evaluator=str(EVALUATOR), evaluator_sha256=digest(EVALUATOR),
reference=str(REFERENCE), reference_sha256=digest(REFERENCE), manifest=str(MANIFEST),
manifest_sha256=digest(MANIFEST), temporal_protocol='61 frames including initial; score future frames1..60')),
('intent', dict(num_samples=2048, init_policy='standard_standing', init_inputs=str(POSE),
init_inputs_sha256=digest(POSE), evaluator=str(V13), evaluator_sha256=digest(V13),
v13_evaluator_sha256=digest(V13), condition=str(conditions), condition_sha256=digest(conditions),
reference=str(reference), reference_sha256=digest(reference), retrieval_seed=0, retrieval_batch_size=32,
temporal_protocol='variable valid lengths; pad450 with final frame, preserve reference valid mask'))]:
work = jobdir / bench
scripts = [OLD / ('omg_gt_generate.py' if bench == 'omg' else 'intent_standing_generate.py'),
OLD / ('omg_gt_pack.py' if bench == 'omg' else 'intent_standing_pack.py')]
protocol = dict(common, **settings, benchmark=bench,
generation_scripts_sha256={str(p): digest(p) for p in scripts})
save(work / 'protocol.json', protocol)
save(work / 'status.json', dict(status='queued', phase='waiting', samples_complete=0,
samples_total=settings['num_samples'], updated_at=time.time()))
save(jobdir / 'job.json', dict(size=size, step=pointer['step'], checkpoint_sha256=info['sha256'],
source_checkpoint=str(source), snapshot=str(snapshot), created_at=time.time()))
# The optional manual OMG smoke entry points at the first newly queued protocol.
current = OUT / 'current_omg_work'
if not current.exists():
current.symlink_to(jobdir / 'omg', target_is_directory=True)
return jobdir
def execute(command, cwd, env, work, phase, role=None):
status = read(work / 'status.json')
status.update(status='running', phase=phase, updated_at=time.time(), error=None)
save(work / 'status.json', status)
if role:
command = [str(PSI_PY), '-B', '-u', str(OLD / 'gpu_eval_slots.py'), '--role', role,
'--capacity', '3' if role == 'generation' else '1', '--'] + command
log_path = work / f'{phase}.log'
with log_path.open('ab') as log:
process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT)
try:
while True:
status.update(pid=process.pid, updated_at=time.time())
sampledir = work / read(work / 'protocol.json')['benchmark'] / 'samples'
status['samples_complete'] = len(list(sampledir.glob('*.npz')))
with log_path.open('rb') as reader:
reader.seek(max(0, log_path.stat().st_size - 32768))
tail = reader.read().decode(errors='replace')
tracked = re.findall(r'Tracker execute:[^\r\n]*?\s(\d+)/(\d+)', tail)
if tracked:
status['tracker_samples_complete'] = int(tracked[-1][0])
save(work / 'status.json', status)
try:
code = process.wait(timeout=10)
if code:
raise RuntimeError(f'{phase} exited {code}; see {log_path}')
break
except subprocess.TimeoutExpired:
pass
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=30)
def evaluate(jobdir, gpu):
job = read(jobdir / 'job.json')
weights = Path(job['snapshot']) / 'checkpoints' / f'ckpt_{job["step"]}' / 'model.safetensors'
if not read(jobdir / 'verified.json'):
save(OUT / 'worker.json', dict(status='running', phase='verify_checkpoint', size=job['size'],
step=job['step'], gpu=gpu, updated_at=time.time()))
assert digest(weights) == job['checkpoint_sha256'], 'Pinned checkpoint checksum mismatch'
save(jobdir / 'verified.json', dict(checkpoint_sha256=job['checkpoint_sha256']))
for bench in ('omg', 'intent'):
work = jobdir / bench
if read(work / 'report.json').get('status') == 'complete':
continue
protocol = read(work / 'protocol.json')
save(OUT / 'worker.json', dict(status='running', phase='evaluate', size=job['size'], step=job['step'],
bench=bench, gpu=gpu, updated_at=time.time()))
env = os.environ.copy()
env.update(CUDA_VISIBLE_DEVICES=gpu, PYTHONNOUSERSITE='1', PYTHONDONTWRITEBYTECODE='1',
HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1', TOKENIZERS_PARALLELISM='false',
OMP_NUM_THREADS='2', MKL_NUM_THREADS='2', OPENBLAS_NUM_THREADS='2',
LD_LIBRARY_PATH=str(PSI_PY.parent.parent / 'lib') + ':/usr/local/cuda/lib64',
OMNIHM_TRACKER_WORKERS='2')
for key in ('PYTHONHOME', 'LD_PRELOAD', 'OMG_LOCAL_SRC', 'PSI_CODE_ROOT', 'PSI_PRETRAIN_BACKBONE_PATH'):
env.pop(key, None)
generated = work / bench / 'generated_qpos.npz'
if not generated.is_file():
execute([str(PSI_PY), '-B', '-u', str(ROOT / 'scale/script/scale_benchmark_generate.py'),
'--bench', bench, '--work', str(work)], ROOT, env, work, 'generate', 'generation')
execute([str(OMG_PY), '-B', '-u', str(OLD / ('omg_gt_pack.py' if bench == 'omg' else 'intent_standing_pack.py')),
'--work', str(work)], OLD, env, work, 'pack')
cache = work / 'metrics'
cache.mkdir(exist_ok=True)
metric_env = dict(env, PYTHONPATH=str(BENCH) + ':' + str(OMG / 'src'),
LD_LIBRARY_PATH=str(OMG_PY.parent.parent / 'lib') + ':/usr/local/cuda/lib64',
OMG_DATA_ROOT=str(USER.parent / 'videogen/OMG/data/OMG-Data'), OMG_MODELS_ROOT=str(OMG / 'models'),
OMG_T5_3B_MODEL=str(OMG / 'models/t5-3b-local'), HF_HOME=str(USER.parent / 'videogen/OMG/hf_cache'))
sys.path.insert(0, str(BENCH / 'scripts'))
from _shared.tracker_config import TRACKER_OPTIONS
if bench == 'omg':
paper_path = cache / 'paper_row.json'
if not (cache / 'benchmark.json').is_file():
command = [str(OMG_PY), '-B', '-u', str(OLD / 'omg_text_paper_metrics_parallel.py'),
'--mode', 'text', '--generated_qpos', str(generated), '--reference_qpos', protocol['reference'],
'--samples_path', protocol['manifest'], '--exp', '100m_omnimodal', '--data', 'omg_data_lerobot_omnimodal',
'--output_dir', str(cache), '--num_frames', '60', '--batch-size', '64', '--seed', '0',
'--label', f'Movens-{job["size"]} ckpt{job["step"]}', '--motion_key', 'body_pos_local',
'--evaluator_checkpoint', protocol['evaluator'], '--enable_text_metrics',
'--t5_3b_model', str(OMG / 'models/t5-3b-local'), '--text_cache_dir', str(ENCODER / 'omg_bench/text_cache'),
'--no-transition-metrics', '--device', 'cuda', '--tracker-executed', '--tracker-num-samples', '1024',
'--tracker-sample-seed', '0']
for key, value in TRACKER_OPTIONS.items():
if key != 'tracker-steps':
command += ['--' + key, str(value)]
execute(command, OMG, metric_env, work, 'metrics_tracking', 'metrics')
if not paper_path.is_file():
execute([str(OMG_PY), '-B', str(OMG / 'scripts/extract_omg_paper_row.py'), '--mode', 'text',
'--evaluation-dir', str(cache), '--xml-path', str(TRACKER_OPTIONS['tracker-robot-xml']),
'--output-path', str(paper_path)], OMG, metric_env, work, 'paper_row')
else:
paper_path = cache / 'paper_metrics.json'
if not paper_path.is_file():
options = dict(task='text', generated_path=generated, reference_path=protocol['reference'],
condition_path=protocol['condition'], output_dir=cache, omg_root=OMG, expected_frames=450,
expected_fps=30, device='cuda', batch_size=32, validate_only=0,
kinematics_path=OMG / 'assets/robots/g1/g1_kinematics.json', contact_height_threshold=.12,
contact_penetration_tolerance=.02, evaluator_checkpoint=V13, retrieval_evaluator_checkpoint=V13,
raw_text_cache=BENCH / 'data/text_eval_2048/v13_t5_raw.npz', motion_key='body_pos_local',
t5_model=OMG / 'models/t5-3b-local', text_cache_dir=cache / 'text_cache', text_batch_size=16,
seed=0, beat_direction='music_to_motion', beat_threshold=.5, sigma_frames=3.,
min_motion_beat_distance_seconds=.25, tracker_mode='run', tracker_artifact=cache / 'unused.npz')
options.update({k.replace('-', '_'): v for k, v in TRACKER_OPTIONS.items()})
command = [str(OMG_PY), '-B', '-u', str(OLD / 'evaluate_parallel_tracking.py')]
for key, value in options.items():
command += ['--' + key.replace('_', '-'), str(value)]
execute(command, BENCH, metric_env, work, 'metrics_tracking', 'metrics')
paper = read(paper_path)
assert paper['columns'] == COLUMNS
assert all(math.isfinite(v) for v in paper['values']) and len(paper['values']) == 14
if bench == 'intent':
assert paper['complete'] is True
full = read(cache / 'metrics.json')
assert full['tracker_executed']['num_samples'] == 2048 and full['tracker_executed']['paper_eligible'] is True
text = full['generated_target']['text']
assert text['evaluator_checkpoint_sha256'] == text['retrieval_evaluator_checkpoint_sha256'] == protocol['evaluator_sha256']
assert text['retrieval_seed'] == 0 and text['retrieval_seeds'] == [0]
else:
assert paper['num_samples'] == 1024
full = read(cache / 'benchmark.json')
assert full['tracker_executed']['num_samples'] == 1024
assert full['tracker_executed']['selected_indices'] == list(range(1024))
report = dict(status='complete', size=job['size'], checkpoint_step=job['step'],
checkpoint_sha256=job['checkpoint_sha256'], protocol_sha256=digest(work / 'protocol.json'),
columns=COLUMNS, values=paper['values'], paper_path=str(paper_path), paper_sha256=digest(paper_path),
generated_sha256=digest(generated), num_samples=protocol['num_samples'], completed_at=time.time())
save(work / 'report.json', report)
save(work / 'status.json', dict(status='complete', phase='complete', samples_complete=protocol['num_samples'],
tracker_samples_complete=protocol['num_samples'], updated_at=time.time()))
print(job['size'], job['step'], bench, paper['values'], flush=True)
# <modify>+两个 bench 都完成后只删除本队列创建的权重硬链接,保留协议、生成数据和全部指标。
weights.unlink(missing_ok=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', default='1')
parser.add_argument('--once', action='store_true')
parser.add_argument('--prepare-only', action='store_true')
args = parser.parse_args()
OUT.mkdir(parents=True, exist_ok=True)
lock = (OUT / 'queue.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
while True:
# Finish the first queued checkpoint of every size before pinning its next latest backup.
for size, part in [('S', 'split_sm'), ('M', 'split_sm'), ('XL', 'split_xl')]:
unfinished = [p.parent for p in sorted((OUT / 'jobs').glob(f'{size}_*/job.json'))
if any(read(p.parent / b / 'report.json').get('status') != 'complete' for b in ('omg', 'intent'))]
if not unfinished:
try:
prepare(size, read(GROUP / part / 'backups' / size / 'latest.json'))
except Exception as error:
print('Prepare failed:', size, repr(error), flush=True)
save(OUT / f'prepare_error_{size}.json', dict(error=repr(error), updated_at=time.time()))
if args.prepare_only:
print(json.dumps(dashboard_tables(), ensure_ascii=False, indent=2))
break
pending = [p.parent for p in sorted((OUT / 'jobs').glob('*/job.json'))
if any(read(p.parent / b / 'report.json').get('status') != 'complete' for b in ('omg', 'intent'))]
pending.sort(key=lambda p: read(p / 'job.json')['created_at'])
for jobdir in pending:
try:
evaluate(jobdir, args.gpu)
except Exception as error:
print('Evaluation failed:', jobdir.name, repr(error), flush=True)
for bench in ('omg', 'intent'):
status = read(jobdir / bench / 'status.json')
if status.get('status') == 'running':
save(jobdir / bench / 'status.json', dict(status, status='error', error=repr(error), updated_at=time.time()))
if args.once:
break
save(OUT / 'worker.json', dict(status='idle', gpu=args.gpu, updated_at=time.time()))
time.sleep(60)