Download scale/protocol/render_cases.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 19.7 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/protocol/render_cases.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/scale/protocol/render_cases.py
-
curl -L -o render_cases.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/scale/protocol/render_cases.py
19.7 kB
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import time | |
| import traceback | |
| from pathlib import Path | |
| # <modify> Select software or NVIDIA EGL before importing MuJoCo and OpenGL. | |
| # os.environ['CUDA_VISIBLE_DEVICES'] = '' | |
| # os.environ['MUJOCO_GL'] = 'egl' | |
| # os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1' | |
| # os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe' | |
| # os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json' | |
| # os.environ['LP_NUM_THREADS'] = '2' | |
| # os.nice(10) | |
| # import mujoco | |
| # import numpy as np | |
| # from OpenGL import GL | |
| # from PIL import Image, ImageDraw, ImageFont | |
| ROOT = Path(__file__).resolve().parent | |
| OUTPUT = ROOT / 'output' | |
| XML = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml') | |
| def configure_renderer(renderer, gpu_index, nvidia_egl_vendor): | |
| os.environ['MUJOCO_GL'] = 'egl' | |
| os.environ['PYOPENGL_PLATFORM'] = 'egl' | |
| if renderer == 'software': | |
| os.environ['CUDA_VISIBLE_DEVICES'] = '' | |
| os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1' | |
| os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe' | |
| os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json' | |
| os.environ['LP_NUM_THREADS'] = '2' | |
| os.environ.pop('MUJOCO_EGL_DEVICE_ID', None) | |
| else: | |
| if gpu_index < 0: | |
| raise ValueError('--gpu-index must be nonnegative') | |
| if not nvidia_egl_vendor.is_file(): | |
| raise FileNotFoundError(f'NVIDIA EGL vendor file is missing: {nvidia_egl_vendor}. Use --renderer software or supply --nvidia-egl-vendor; CUDA alone does not provide OpenGL.') | |
| library = json.loads(nvidia_egl_vendor.read_text()).get('ICD', {}).get('library_path', '') | |
| if 'nvidia' not in library.lower(): | |
| raise ValueError(f'Expected an NVIDIA EGL library in {nvidia_egl_vendor}, got {library!r}') | |
| for name in ('LIBGL_ALWAYS_SOFTWARE', 'MESA_LOADER_DRIVER_OVERRIDE', 'GALLIUM_DRIVER', 'LP_NUM_THREADS', 'LIBGL_ALWAYS_INDIRECT'): | |
| os.environ.pop(name, None) | |
| os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = str(nvidia_egl_vendor.resolve()) | |
| os.environ['__GLX_VENDOR_LIBRARY_NAME'] = 'nvidia' | |
| # <modify> MuJoCo indexes eglQueryDevicesEXT directly; CUDA_VISIBLE_DEVICES does not remap this index. | |
| os.environ['MUJOCO_EGL_DEVICE_ID'] = str(gpu_index) | |
| def save_json(path, value): | |
| temp = path.with_suffix('.tmp') | |
| temp.write_text(json.dumps(value, indent=2, allow_nan=False)) | |
| temp.replace(path) | |
| # <modify> Record the measured GL device per artifact, without changing camera or motion rendering. | |
| # def render(case, model, data, renderer, font): | |
| def render(case, model, data, renderer, font, backend_info, max_frames=0): | |
| directory = OUTPUT / case['directory'] | |
| started = time.time() | |
| status_path = directory / 'render_status.json' | |
| status = dict(status='rendering', render_started_at=started, render_updated_at=started) | |
| status.update(backend_info) | |
| status['smoke_max_frames'] = max_frames | |
| # <modify> Keep render identity tied to the exact prediction protocol, not only its checkpoint step. | |
| status.update(inference_protocol_version=case['inference_protocol_version'], gallery_version=case['gallery_version']) | |
| # <modify> Keep contact-label provenance when rendering a corrected checkpoint. | |
| status['root40_contact_protocol'] = case.get('root40_contact_protocol', 'legacy_joint_delta_v1') | |
| save_json(status_path, status) | |
| try: | |
| with np.load(directory / 'gt.npz') as file: | |
| gt = file['qpos_36'] | |
| names = file['joint_names'].tolist() | |
| assert names == [mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, i) for i in range(1, model.njnt)] | |
| with np.load(directory / 'prediction.npz') as file: | |
| prediction = file['qpos_36'] | |
| # <modify> Both corrected protocols require exact prediction provenance. | |
| # if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2': | |
| # <modify> Validate v4 provenance without changing GMR camera, geometry, or motion. | |
| # if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'): | |
| if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'): | |
| assert str(file['inference_protocol_version'].item()) == case['inference_protocol_version'] | |
| assert str(file['gallery_version'].item()) == case['gallery_version'] | |
| contact_protocol = str(file['root40_contact_protocol'].item()) if 'root40_contact_protocol' in file else 'legacy_joint_delta_v1' | |
| assert contact_protocol == case.get('root40_contact_protocol', 'legacy_joint_delta_v1') | |
| if case['inference_protocol_version'].endswith('_projection_fp32_v4'): | |
| assert str(file['action_input_projection_dtype'].item()) == 'float32' | |
| assert str(file['action_head_remaining_dtype'].item()) == 'bfloat16' | |
| assert str(file['vlm_dtype'].item()) == 'bfloat16' | |
| assert str(file['sampler_dtype'].item()) == 'float32' | |
| assert gt.shape == prediction.shape and gt.shape[1] == model.nq == 36 | |
| # <modify> Optional short smoke runs only use explicitly marked, separate output directories. | |
| if max_frames: | |
| gt = gt[:max_frames] | |
| prediction = prediction[:max_frames] | |
| points = np.concatenate([prediction[:, :3], gt[:, :3]]) | |
| low = points.min(axis=0) - np.array([0.7, 0.7, 0.85]) | |
| high = points.max(axis=0) + np.array([0.7, 0.7, 0.7]) | |
| low[2] = min(low[2], 0) | |
| camera = mujoco.MjvCamera() | |
| mujoco.mjv_defaultCamera(camera) | |
| camera.azimuth = -140 | |
| camera.elevation = -20 | |
| camera.lookat[:] = [0, 0, 0.7] | |
| camera.distance = 3.0 | |
| status['camera'] = dict(mode='XY-follow only; fixed world Z=0.7, scale and angles; shared-coordinate XY inset', distance=camera.distance, azimuth=camera.azimuth, elevation=camera.elevation) | |
| center = (low[:2] + high[:2]) / 2 | |
| span = max(float(np.max(high[:2] - low[:2])), 2.0) | |
| tracks = [(poses[:, :2] - center) / span * 118 for poses in (prediction, gt)] | |
| track_colors = [(118, 213, 173), (115, 175, 245)] | |
| small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 14) | |
| temp = directory / 'comparison.partial.mp4' | |
| command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-y', '-threads', '1', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', '1280x480', '-r', str(case['fps']), '-i', 'pipe:0'] | |
| with_audio = case['modality'] in ('music', 'speech') | |
| if with_audio: | |
| command += ['-threads', '1', '-i', str(directory / case['input_media'])] | |
| command += ['-map', '0:v:0', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', '-threads', '2', '-pix_fmt', 'yuv420p', '-movflags', '+faststart'] | |
| if with_audio: | |
| command += ['-map', '1:a:0', '-af', 'apad', '-c:a', 'aac', '-b:a', '128k', '-shortest'] | |
| command += [str(temp)] | |
| with (directory / 'ffmpeg.log').open('ab') as log: | |
| process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=log) | |
| try: | |
| for frame_id, (pred, truth) in enumerate(zip(prediction, gt)): | |
| images = [] | |
| for pose in (pred, truth): | |
| data.qpos[:] = pose | |
| mujoco.mj_forward(model, data) | |
| camera.lookat[:2] = pose[:2] | |
| renderer.update_scene(data, camera=camera) | |
| renderer.scene.flags[mujoco.mjtRndFlag.mjRND_SHADOW] = False | |
| renderer.scene.flags[mujoco.mjtRndFlag.mjRND_REFLECTION] = False | |
| images.append(renderer.render()) | |
| frame = Image.fromarray(np.concatenate(images, axis=1)) | |
| draw = ImageDraw.Draw(frame) | |
| draw.rectangle((0, 0, 1280, 34), fill=(16, 23, 32)) | |
| # <modify> Label the actual inference checkpoint rather than the historical fixed step. | |
| # draw.text((14, 4), 'Prediction | ckpt 2000', font=font, fill=(118, 213, 173)) | |
| # <modify> A visible protocol suffix distinguishes corrected and historical videos at the same checkpoint. | |
| # draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]}', font=font, fill=(118, 213, 173)) | |
| # <modify> Show the actual video-progress protocol without changing rendering or motion. | |
| # protocol_label = 'root40_v2' if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2' else 'original_v1' | |
| protocol_label = {'root40_current_heading_fp32_v2': 'root40_v2', | |
| 'root40_current_heading_fp32_video_local_progress_v3': 'root40_v3', | |
| # <modify> Label the exact new inference precision on rendered videos. | |
| 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4': 'root40_v4 input FP32', | |
| 'original_cpu_bf16_rtc_v1': 'original_v1'}[case['inference_protocol_version']] | |
| draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]} | {protocol_label}', font=font, fill=(118, 213, 173)) | |
| draw.text((654, 4), 'Ground truth', font=font, fill=(115, 175, 245)) | |
| draw.text((1030, 4), f'{frame_id / case["fps"]:.2f} / {len(gt) / case["fps"]:.2f} s', font=font, fill='white') | |
| draw.line((640, 0, 640, 480), fill=(31, 42, 53), width=2) | |
| for offset, pose in [(0, pred), (640, truth)]: | |
| draw.rectangle((offset, 449, offset + 639, 479), fill=(16, 23, 32)) | |
| color = (255, 146, 146) if pose[2] < 0 else (222, 232, 240) | |
| draw.text((offset + 10, 454), f'Root XYZ: {pose[0]:+.2f}, {pose[1]:+.2f}, {pose[2]:+.2f} m | XY-follow view', font=small_font, fill=color) | |
| draw.rectangle((offset + 482, 42, offset + 632, 198), fill=(16, 23, 32)) | |
| draw.text((offset + 489, 46), 'World XY paths', font=small_font, fill=(210, 223, 234)) | |
| draw.text((offset + 489, 179), f'Width {span:.1f} m', font=small_font, fill=(147, 163, 181)) | |
| for track, track_color in zip(tracks, track_colors): | |
| coords = [(float(offset + 557 + xy[0]), float(121 - xy[1])) for xy in track] | |
| draw.line(coords, fill=track_color, width=2) | |
| x, y = coords[frame_id] | |
| draw.ellipse((x - 4, y - 4, x + 4, y + 4), fill=track_color, outline='white') | |
| if frame_id == 0: | |
| frame.save(directory / 'poster.jpg', quality=90) | |
| process.stdin.write(np.asarray(frame).tobytes()) | |
| if frame_id % 30 == 0: | |
| status.update(rendered_frames=frame_id + 1, render_total_frames=len(gt), render_updated_at=time.time()) | |
| save_json(status_path, status) | |
| finally: | |
| process.stdin.close() | |
| code = process.wait(timeout=60) | |
| if code: | |
| raise RuntimeError(f'FFmpeg failed with code {code}; see {directory / "ffmpeg.log"}') | |
| temp.replace(directory / 'comparison.mp4') | |
| status.update(status='ready', rendered_frames=len(gt), render_seconds=time.time() - started, render_updated_at=time.time()) | |
| print(f'{case["id"]}: rendered {len(gt)} frames in {status["render_seconds"]:.1f}s', flush=True) | |
| except Exception: | |
| status.update(status='error', error=traceback.format_exc(), render_updated_at=time.time()) | |
| print(status['error'], flush=True) | |
| save_json(status_path, status) | |
| def main(): | |
| # <modify> Keep original output unchanged and allow checkpoint-specific render jobs. | |
| # <modify> Delay imports until the renderer environment has been configured. | |
| # global OUTPUT | |
| global OUTPUT, mujoco, np, GL, Image, ImageDraw, ImageFont | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--watch', action='store_true') | |
| parser.add_argument('--output-dir', type=Path, default=ROOT / 'output') | |
| parser.add_argument('--renderer', choices=('software', 'nvidia', 'gpu'), default='software') | |
| parser.add_argument('--gpu-index', type=int, default=0, help='EGL device index; not remapped by CUDA_VISIBLE_DEVICES') | |
| parser.add_argument('--nvidia-egl-vendor', type=Path, default=Path('/usr/share/glvnd/egl_vendor.d/10_nvidia.json')) | |
| parser.add_argument('--case-ids', nargs='+', help='Only render these manifest case IDs') | |
| parser.add_argument('--max-frames', type=int, default=0, help='Short render; requires manifest render_smoke_test=true') | |
| args = parser.parse_args() | |
| OUTPUT = args.output_dir.resolve(strict=True) | |
| manifest = json.loads((OUTPUT / 'manifest.json').read_text()) | |
| if args.max_frames < 0: | |
| parser.error('--max-frames must be nonnegative') | |
| if args.max_frames and manifest.get('render_smoke_test') is not True: | |
| parser.error('--max-frames requires a separate smoke output with manifest render_smoke_test=true') | |
| selected_ids = set(args.case_ids or [case['id'] for case in manifest['cases']]) | |
| unknown_ids = selected_ids - {case['id'] for case in manifest['cases']} | |
| if unknown_ids: | |
| parser.error(f'Unknown case IDs: {sorted(unknown_ids)}') | |
| configure_renderer(args.renderer, args.gpu_index, args.nvidia_egl_vendor) | |
| os.nice(10) | |
| import mujoco | |
| import numpy as np | |
| from OpenGL import GL | |
| from PIL import Image, ImageDraw, ImageFont | |
| for case in manifest['cases']: | |
| case['checkpoint_step'] = manifest['checkpoint_step'] | |
| case['inference_protocol_version'] = manifest.get('inference_protocol_version', 'original_cpu_bf16_rtc_v1') | |
| case['gallery_version'] = manifest.get('gallery_version', f'ckpt_{manifest["checkpoint_step"]}') | |
| case['root40_contact_protocol'] = manifest.get('root40_contact_protocol', 'legacy_joint_delta_v1') | |
| model = mujoco.MjModel.from_xml_path(str(XML)) | |
| model.vis.quality.offsamples = 0 | |
| model.vis.quality.shadowsize = 512 | |
| data = mujoco.MjData(model) | |
| renderer = mujoco.Renderer(model, height=480, width=640) | |
| backend = GL.glGetString(GL.GL_RENDERER).decode() | |
| # <modify> Never label software fallback as GPU acceleration. | |
| # if 'llvmpipe' not in backend.lower(): | |
| # raise RuntimeError(f'Refusing a non-software renderer during training: {backend}') | |
| vendor = GL.glGetString(GL.GL_VENDOR).decode() | |
| is_software = 'llvmpipe' in backend.lower() | |
| is_nvidia = 'nvidia' in vendor.lower() and not is_software | |
| if (args.renderer == 'software' and not is_software) or (args.renderer != 'software' and not is_nvidia): | |
| renderer.close() | |
| raise RuntimeError(f'Requested {args.renderer}, got GL vendor={vendor!r}, renderer={backend!r}') | |
| backend_info = dict(renderer_requested=args.renderer, render_backend='software' if is_software else 'nvidia', gl_renderer=backend, gl_vendor=vendor, egl_device_id=None if is_software else args.gpu_index) | |
| print('Renderer:', backend, flush=True) | |
| font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 20) | |
| attempted = set() | |
| try: | |
| while True: | |
| for case in manifest['cases']: | |
| if case['id'] not in selected_ids: | |
| continue | |
| dest = OUTPUT / case['directory'] | |
| # <modify> Retry interrupted renders unless both the video and its ready marker are present. | |
| # if (dest / 'prediction.npz').exists() and not (dest / 'comparison.mp4').exists() and case['id'] not in attempted: | |
| marker = dest / 'render_status.json' | |
| # <modify> Reject mixed-protocol completed renders instead of quietly reusing them. | |
| # complete = (dest / 'comparison.mp4').exists() and marker.exists() and json.loads(marker.read_text()).get('status') == 'ready' | |
| cached = json.loads(marker.read_text()) if marker.exists() else {} | |
| # <modify> Never reuse a cached render from another v3/v2 protocol. | |
| # if cached and case['inference_protocol_version'] == 'root40_current_heading_fp32_v2': | |
| # <modify> Do not reuse a BF16 render for a projection-FP32 job. | |
| # if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'): | |
| if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'): | |
| assert cached.get('inference_protocol_version') == case['inference_protocol_version'] | |
| assert cached.get('gallery_version') == case['gallery_version'] | |
| assert cached.get('root40_contact_protocol', 'legacy_joint_delta_v1') == case['root40_contact_protocol'] | |
| complete = (dest / 'comparison.mp4').exists() and cached.get('status') == 'ready' | |
| # <modify> Keep completed CPU renders unchanged, including their actual/unknown historical backend. | |
| if complete and cached.get('render_backend'): | |
| cached_renderer = str(cached.get('gl_renderer', '')).lower() | |
| cached_vendor = str(cached.get('gl_vendor', '')).lower() | |
| if not cached_renderer or (cached['render_backend'] == 'software' and 'llvmpipe' not in cached_renderer) or (cached['render_backend'] == 'nvidia' and ('nvidia' not in cached_vendor or 'llvmpipe' in cached_renderer)): | |
| raise RuntimeError(f'Inconsistent cached GL renderer metadata for {case["id"]}') | |
| if complete and cached.get('smoke_max_frames', 0) != args.max_frames: | |
| raise RuntimeError(f'Incompatible smoke render already exists for {case["id"]}') | |
| if (dest / 'prediction.npz').exists() and not complete and case['id'] not in attempted: | |
| attempted.add(case['id']) | |
| # render(case, model, data, renderer, font) | |
| render(case, model, data, renderer, font, backend_info, args.max_frames) | |
| cached = json.loads(marker.read_text()) | |
| if cached.get('status') == 'ready': | |
| manifest.setdefault('render_backends', {})[case['id']] = {key: cached.get(key) for key in ('render_backend', 'gl_renderer', 'gl_vendor', 'egl_device_id')} | |
| save_json(OUTPUT / 'manifest.json', manifest) | |
| if not args.watch: | |
| break | |
| status_path = OUTPUT / 'inference_status.json' | |
| if status_path.exists(): | |
| status = json.loads(status_path.read_text()) | |
| proc = Path('/proc') / str(status.get('pid', -1)) | |
| if status['status'] == 'finished' or not proc.exists(): | |
| break | |
| time.sleep(5) | |
| finally: | |
| renderer.close() | |
| if __name__ == '__main__': | |
| main() | |