import argparse import json import os import subprocess import time import traceback from pathlib import Path # Select software or NVIDIA EGL before importing MuJoCo and OpenGL. # os.environ['CUDA_VISIBLE_DEVICES'] = '' # os.environ['MUJOCO_GL'] = 'egl' # os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1' # os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe' # os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json' # os.environ['LP_NUM_THREADS'] = '2' # os.nice(10) # import mujoco # import numpy as np # from OpenGL import GL # from PIL import Image, ImageDraw, ImageFont ROOT = Path(__file__).resolve().parent OUTPUT = ROOT / 'output' XML = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml') def configure_renderer(renderer, gpu_index, nvidia_egl_vendor): os.environ['MUJOCO_GL'] = 'egl' os.environ['PYOPENGL_PLATFORM'] = 'egl' if renderer == 'software': os.environ['CUDA_VISIBLE_DEVICES'] = '' os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1' os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe' os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json' os.environ['LP_NUM_THREADS'] = '2' os.environ.pop('MUJOCO_EGL_DEVICE_ID', None) else: if gpu_index < 0: raise ValueError('--gpu-index must be nonnegative') if not nvidia_egl_vendor.is_file(): raise FileNotFoundError(f'NVIDIA EGL vendor file is missing: {nvidia_egl_vendor}. Use --renderer software or supply --nvidia-egl-vendor; CUDA alone does not provide OpenGL.') library = json.loads(nvidia_egl_vendor.read_text()).get('ICD', {}).get('library_path', '') if 'nvidia' not in library.lower(): raise ValueError(f'Expected an NVIDIA EGL library in {nvidia_egl_vendor}, got {library!r}') for name in ('LIBGL_ALWAYS_SOFTWARE', 'MESA_LOADER_DRIVER_OVERRIDE', 'GALLIUM_DRIVER', 'LP_NUM_THREADS', 'LIBGL_ALWAYS_INDIRECT'): os.environ.pop(name, None) os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = str(nvidia_egl_vendor.resolve()) os.environ['__GLX_VENDOR_LIBRARY_NAME'] = 'nvidia' # MuJoCo indexes eglQueryDevicesEXT directly; CUDA_VISIBLE_DEVICES does not remap this index. os.environ['MUJOCO_EGL_DEVICE_ID'] = str(gpu_index) def save_json(path, value): temp = path.with_suffix('.tmp') temp.write_text(json.dumps(value, indent=2, allow_nan=False)) temp.replace(path) # Record the measured GL device per artifact, without changing camera or motion rendering. # def render(case, model, data, renderer, font): def render(case, model, data, renderer, font, backend_info, max_frames=0): directory = OUTPUT / case['directory'] started = time.time() status_path = directory / 'render_status.json' status = dict(status='rendering', render_started_at=started, render_updated_at=started) status.update(backend_info) status['smoke_max_frames'] = max_frames # Keep render identity tied to the exact prediction protocol, not only its checkpoint step. status.update(inference_protocol_version=case['inference_protocol_version'], gallery_version=case['gallery_version']) # Keep contact-label provenance when rendering a corrected checkpoint. status['root40_contact_protocol'] = case.get('root40_contact_protocol', 'legacy_joint_delta_v1') save_json(status_path, status) try: with np.load(directory / 'gt.npz') as file: gt = file['qpos_36'] names = file['joint_names'].tolist() assert names == [mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, i) for i in range(1, model.njnt)] with np.load(directory / 'prediction.npz') as file: prediction = file['qpos_36'] # Both corrected protocols require exact prediction provenance. # if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2': # Validate v4 provenance without changing GMR camera, geometry, or motion. # if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'): if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'): assert str(file['inference_protocol_version'].item()) == case['inference_protocol_version'] assert str(file['gallery_version'].item()) == case['gallery_version'] contact_protocol = str(file['root40_contact_protocol'].item()) if 'root40_contact_protocol' in file else 'legacy_joint_delta_v1' assert contact_protocol == case.get('root40_contact_protocol', 'legacy_joint_delta_v1') if case['inference_protocol_version'].endswith('_projection_fp32_v4'): assert str(file['action_input_projection_dtype'].item()) == 'float32' assert str(file['action_head_remaining_dtype'].item()) == 'bfloat16' assert str(file['vlm_dtype'].item()) == 'bfloat16' assert str(file['sampler_dtype'].item()) == 'float32' assert gt.shape == prediction.shape and gt.shape[1] == model.nq == 36 # Optional short smoke runs only use explicitly marked, separate output directories. if max_frames: gt = gt[:max_frames] prediction = prediction[:max_frames] points = np.concatenate([prediction[:, :3], gt[:, :3]]) low = points.min(axis=0) - np.array([0.7, 0.7, 0.85]) high = points.max(axis=0) + np.array([0.7, 0.7, 0.7]) low[2] = min(low[2], 0) camera = mujoco.MjvCamera() mujoco.mjv_defaultCamera(camera) camera.azimuth = -140 camera.elevation = -20 camera.lookat[:] = [0, 0, 0.7] camera.distance = 3.0 status['camera'] = dict(mode='XY-follow only; fixed world Z=0.7, scale and angles; shared-coordinate XY inset', distance=camera.distance, azimuth=camera.azimuth, elevation=camera.elevation) center = (low[:2] + high[:2]) / 2 span = max(float(np.max(high[:2] - low[:2])), 2.0) tracks = [(poses[:, :2] - center) / span * 118 for poses in (prediction, gt)] track_colors = [(118, 213, 173), (115, 175, 245)] small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 14) temp = directory / 'comparison.partial.mp4' command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-y', '-threads', '1', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', '1280x480', '-r', str(case['fps']), '-i', 'pipe:0'] with_audio = case['modality'] in ('music', 'speech') if with_audio: command += ['-threads', '1', '-i', str(directory / case['input_media'])] command += ['-map', '0:v:0', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', '-threads', '2', '-pix_fmt', 'yuv420p', '-movflags', '+faststart'] if with_audio: command += ['-map', '1:a:0', '-af', 'apad', '-c:a', 'aac', '-b:a', '128k', '-shortest'] command += [str(temp)] with (directory / 'ffmpeg.log').open('ab') as log: process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=log) try: for frame_id, (pred, truth) in enumerate(zip(prediction, gt)): images = [] for pose in (pred, truth): data.qpos[:] = pose mujoco.mj_forward(model, data) camera.lookat[:2] = pose[:2] renderer.update_scene(data, camera=camera) renderer.scene.flags[mujoco.mjtRndFlag.mjRND_SHADOW] = False renderer.scene.flags[mujoco.mjtRndFlag.mjRND_REFLECTION] = False images.append(renderer.render()) frame = Image.fromarray(np.concatenate(images, axis=1)) draw = ImageDraw.Draw(frame) draw.rectangle((0, 0, 1280, 34), fill=(16, 23, 32)) # Label the actual inference checkpoint rather than the historical fixed step. # draw.text((14, 4), 'Prediction | ckpt 2000', font=font, fill=(118, 213, 173)) # A visible protocol suffix distinguishes corrected and historical videos at the same checkpoint. # draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]}', font=font, fill=(118, 213, 173)) # Show the actual video-progress protocol without changing rendering or motion. # protocol_label = 'root40_v2' if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2' else 'original_v1' protocol_label = {'root40_current_heading_fp32_v2': 'root40_v2', 'root40_current_heading_fp32_video_local_progress_v3': 'root40_v3', # Label the exact new inference precision on rendered videos. 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4': 'root40_v4 input FP32', 'original_cpu_bf16_rtc_v1': 'original_v1'}[case['inference_protocol_version']] draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]} | {protocol_label}', font=font, fill=(118, 213, 173)) draw.text((654, 4), 'Ground truth', font=font, fill=(115, 175, 245)) draw.text((1030, 4), f'{frame_id / case["fps"]:.2f} / {len(gt) / case["fps"]:.2f} s', font=font, fill='white') draw.line((640, 0, 640, 480), fill=(31, 42, 53), width=2) for offset, pose in [(0, pred), (640, truth)]: draw.rectangle((offset, 449, offset + 639, 479), fill=(16, 23, 32)) color = (255, 146, 146) if pose[2] < 0 else (222, 232, 240) draw.text((offset + 10, 454), f'Root XYZ: {pose[0]:+.2f}, {pose[1]:+.2f}, {pose[2]:+.2f} m | XY-follow view', font=small_font, fill=color) draw.rectangle((offset + 482, 42, offset + 632, 198), fill=(16, 23, 32)) draw.text((offset + 489, 46), 'World XY paths', font=small_font, fill=(210, 223, 234)) draw.text((offset + 489, 179), f'Width {span:.1f} m', font=small_font, fill=(147, 163, 181)) for track, track_color in zip(tracks, track_colors): coords = [(float(offset + 557 + xy[0]), float(121 - xy[1])) for xy in track] draw.line(coords, fill=track_color, width=2) x, y = coords[frame_id] draw.ellipse((x - 4, y - 4, x + 4, y + 4), fill=track_color, outline='white') if frame_id == 0: frame.save(directory / 'poster.jpg', quality=90) process.stdin.write(np.asarray(frame).tobytes()) if frame_id % 30 == 0: status.update(rendered_frames=frame_id + 1, render_total_frames=len(gt), render_updated_at=time.time()) save_json(status_path, status) finally: process.stdin.close() code = process.wait(timeout=60) if code: raise RuntimeError(f'FFmpeg failed with code {code}; see {directory / "ffmpeg.log"}') temp.replace(directory / 'comparison.mp4') status.update(status='ready', rendered_frames=len(gt), render_seconds=time.time() - started, render_updated_at=time.time()) print(f'{case["id"]}: rendered {len(gt)} frames in {status["render_seconds"]:.1f}s', flush=True) except Exception: status.update(status='error', error=traceback.format_exc(), render_updated_at=time.time()) print(status['error'], flush=True) save_json(status_path, status) def main(): # Keep original output unchanged and allow checkpoint-specific render jobs. # Delay imports until the renderer environment has been configured. # global OUTPUT global OUTPUT, mujoco, np, GL, Image, ImageDraw, ImageFont parser = argparse.ArgumentParser() parser.add_argument('--watch', action='store_true') parser.add_argument('--output-dir', type=Path, default=ROOT / 'output') parser.add_argument('--renderer', choices=('software', 'nvidia', 'gpu'), default='software') parser.add_argument('--gpu-index', type=int, default=0, help='EGL device index; not remapped by CUDA_VISIBLE_DEVICES') parser.add_argument('--nvidia-egl-vendor', type=Path, default=Path('/usr/share/glvnd/egl_vendor.d/10_nvidia.json')) parser.add_argument('--case-ids', nargs='+', help='Only render these manifest case IDs') parser.add_argument('--max-frames', type=int, default=0, help='Short render; requires manifest render_smoke_test=true') args = parser.parse_args() OUTPUT = args.output_dir.resolve(strict=True) manifest = json.loads((OUTPUT / 'manifest.json').read_text()) if args.max_frames < 0: parser.error('--max-frames must be nonnegative') if args.max_frames and manifest.get('render_smoke_test') is not True: parser.error('--max-frames requires a separate smoke output with manifest render_smoke_test=true') selected_ids = set(args.case_ids or [case['id'] for case in manifest['cases']]) unknown_ids = selected_ids - {case['id'] for case in manifest['cases']} if unknown_ids: parser.error(f'Unknown case IDs: {sorted(unknown_ids)}') configure_renderer(args.renderer, args.gpu_index, args.nvidia_egl_vendor) os.nice(10) import mujoco import numpy as np from OpenGL import GL from PIL import Image, ImageDraw, ImageFont for case in manifest['cases']: case['checkpoint_step'] = manifest['checkpoint_step'] case['inference_protocol_version'] = manifest.get('inference_protocol_version', 'original_cpu_bf16_rtc_v1') case['gallery_version'] = manifest.get('gallery_version', f'ckpt_{manifest["checkpoint_step"]}') case['root40_contact_protocol'] = manifest.get('root40_contact_protocol', 'legacy_joint_delta_v1') model = mujoco.MjModel.from_xml_path(str(XML)) model.vis.quality.offsamples = 0 model.vis.quality.shadowsize = 512 data = mujoco.MjData(model) renderer = mujoco.Renderer(model, height=480, width=640) backend = GL.glGetString(GL.GL_RENDERER).decode() # Never label software fallback as GPU acceleration. # if 'llvmpipe' not in backend.lower(): # raise RuntimeError(f'Refusing a non-software renderer during training: {backend}') vendor = GL.glGetString(GL.GL_VENDOR).decode() is_software = 'llvmpipe' in backend.lower() is_nvidia = 'nvidia' in vendor.lower() and not is_software if (args.renderer == 'software' and not is_software) or (args.renderer != 'software' and not is_nvidia): renderer.close() raise RuntimeError(f'Requested {args.renderer}, got GL vendor={vendor!r}, renderer={backend!r}') backend_info = dict(renderer_requested=args.renderer, render_backend='software' if is_software else 'nvidia', gl_renderer=backend, gl_vendor=vendor, egl_device_id=None if is_software else args.gpu_index) print('Renderer:', backend, flush=True) font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 20) attempted = set() try: while True: for case in manifest['cases']: if case['id'] not in selected_ids: continue dest = OUTPUT / case['directory'] # Retry interrupted renders unless both the video and its ready marker are present. # if (dest / 'prediction.npz').exists() and not (dest / 'comparison.mp4').exists() and case['id'] not in attempted: marker = dest / 'render_status.json' # Reject mixed-protocol completed renders instead of quietly reusing them. # complete = (dest / 'comparison.mp4').exists() and marker.exists() and json.loads(marker.read_text()).get('status') == 'ready' cached = json.loads(marker.read_text()) if marker.exists() else {} # Never reuse a cached render from another v3/v2 protocol. # if cached and case['inference_protocol_version'] == 'root40_current_heading_fp32_v2': # Do not reuse a BF16 render for a projection-FP32 job. # if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'): if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'): assert cached.get('inference_protocol_version') == case['inference_protocol_version'] assert cached.get('gallery_version') == case['gallery_version'] assert cached.get('root40_contact_protocol', 'legacy_joint_delta_v1') == case['root40_contact_protocol'] complete = (dest / 'comparison.mp4').exists() and cached.get('status') == 'ready' # Keep completed CPU renders unchanged, including their actual/unknown historical backend. if complete and cached.get('render_backend'): cached_renderer = str(cached.get('gl_renderer', '')).lower() cached_vendor = str(cached.get('gl_vendor', '')).lower() if not cached_renderer or (cached['render_backend'] == 'software' and 'llvmpipe' not in cached_renderer) or (cached['render_backend'] == 'nvidia' and ('nvidia' not in cached_vendor or 'llvmpipe' in cached_renderer)): raise RuntimeError(f'Inconsistent cached GL renderer metadata for {case["id"]}') if complete and cached.get('smoke_max_frames', 0) != args.max_frames: raise RuntimeError(f'Incompatible smoke render already exists for {case["id"]}') if (dest / 'prediction.npz').exists() and not complete and case['id'] not in attempted: attempted.add(case['id']) # render(case, model, data, renderer, font) render(case, model, data, renderer, font, backend_info, args.max_frames) cached = json.loads(marker.read_text()) if cached.get('status') == 'ready': manifest.setdefault('render_backends', {})[case['id']] = {key: cached.get(key) for key in ('render_backend', 'gl_renderer', 'gl_vendor', 'egl_device_id')} save_json(OUTPUT / 'manifest.json', manifest) if not args.watch: break status_path = OUTPUT / 'inference_status.json' if status_path.exists(): status = json.loads(status_path.read_text()) proc = Path('/proc') / str(status.get('pid', -1)) if status['status'] == 'finished' or not proc.exists(): break time.sleep(5) finally: renderer.close() if __name__ == '__main__': main()