Tinghong-Ye22's picture
Add files using upload-large-folder tool
f55ddc0 verified
Raw History Blame Contribute Delete
19.7 kB
import argparse
import json
import os
import subprocess
import time
import traceback
from pathlib import Path
# <modify> Select software or NVIDIA EGL before importing MuJoCo and OpenGL.
# os.environ['CUDA_VISIBLE_DEVICES'] = ''
# os.environ['MUJOCO_GL'] = 'egl'
# os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1'
# os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe'
# os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json'
# os.environ['LP_NUM_THREADS'] = '2'
# os.nice(10)
# import mujoco
# import numpy as np
# from OpenGL import GL
# from PIL import Image, ImageDraw, ImageFont
ROOT = Path(__file__).resolve().parent
OUTPUT = ROOT / 'output'
XML = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml')
def configure_renderer(renderer, gpu_index, nvidia_egl_vendor):
os.environ['MUJOCO_GL'] = 'egl'
os.environ['PYOPENGL_PLATFORM'] = 'egl'
if renderer == 'software':
os.environ['CUDA_VISIBLE_DEVICES'] = ''
os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1'
os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe'
os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json'
os.environ['LP_NUM_THREADS'] = '2'
os.environ.pop('MUJOCO_EGL_DEVICE_ID', None)
else:
if gpu_index < 0:
raise ValueError('--gpu-index must be nonnegative')
if not nvidia_egl_vendor.is_file():
raise FileNotFoundError(f'NVIDIA EGL vendor file is missing: {nvidia_egl_vendor}. Use --renderer software or supply --nvidia-egl-vendor; CUDA alone does not provide OpenGL.')
library = json.loads(nvidia_egl_vendor.read_text()).get('ICD', {}).get('library_path', '')
if 'nvidia' not in library.lower():
raise ValueError(f'Expected an NVIDIA EGL library in {nvidia_egl_vendor}, got {library!r}')
for name in ('LIBGL_ALWAYS_SOFTWARE', 'MESA_LOADER_DRIVER_OVERRIDE', 'GALLIUM_DRIVER', 'LP_NUM_THREADS', 'LIBGL_ALWAYS_INDIRECT'):
os.environ.pop(name, None)
os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = str(nvidia_egl_vendor.resolve())
os.environ['__GLX_VENDOR_LIBRARY_NAME'] = 'nvidia'
# <modify> MuJoCo indexes eglQueryDevicesEXT directly; CUDA_VISIBLE_DEVICES does not remap this index.
os.environ['MUJOCO_EGL_DEVICE_ID'] = str(gpu_index)
def save_json(path, value):
temp = path.with_suffix('.tmp')
temp.write_text(json.dumps(value, indent=2, allow_nan=False))
temp.replace(path)
# <modify> Record the measured GL device per artifact, without changing camera or motion rendering.
# def render(case, model, data, renderer, font):
def render(case, model, data, renderer, font, backend_info, max_frames=0):
directory = OUTPUT / case['directory']
started = time.time()
status_path = directory / 'render_status.json'
status = dict(status='rendering', render_started_at=started, render_updated_at=started)
status.update(backend_info)
status['smoke_max_frames'] = max_frames
# <modify> Keep render identity tied to the exact prediction protocol, not only its checkpoint step.
status.update(inference_protocol_version=case['inference_protocol_version'], gallery_version=case['gallery_version'])
# <modify> Keep contact-label provenance when rendering a corrected checkpoint.
status['root40_contact_protocol'] = case.get('root40_contact_protocol', 'legacy_joint_delta_v1')
save_json(status_path, status)
try:
with np.load(directory / 'gt.npz') as file:
gt = file['qpos_36']
names = file['joint_names'].tolist()
assert names == [mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, i) for i in range(1, model.njnt)]
with np.load(directory / 'prediction.npz') as file:
prediction = file['qpos_36']
# <modify> Both corrected protocols require exact prediction provenance.
# if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2':
# <modify> Validate v4 provenance without changing GMR camera, geometry, or motion.
# if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'):
if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'):
assert str(file['inference_protocol_version'].item()) == case['inference_protocol_version']
assert str(file['gallery_version'].item()) == case['gallery_version']
contact_protocol = str(file['root40_contact_protocol'].item()) if 'root40_contact_protocol' in file else 'legacy_joint_delta_v1'
assert contact_protocol == case.get('root40_contact_protocol', 'legacy_joint_delta_v1')
if case['inference_protocol_version'].endswith('_projection_fp32_v4'):
assert str(file['action_input_projection_dtype'].item()) == 'float32'
assert str(file['action_head_remaining_dtype'].item()) == 'bfloat16'
assert str(file['vlm_dtype'].item()) == 'bfloat16'
assert str(file['sampler_dtype'].item()) == 'float32'
assert gt.shape == prediction.shape and gt.shape[1] == model.nq == 36
# <modify> Optional short smoke runs only use explicitly marked, separate output directories.
if max_frames:
gt = gt[:max_frames]
prediction = prediction[:max_frames]
points = np.concatenate([prediction[:, :3], gt[:, :3]])
low = points.min(axis=0) - np.array([0.7, 0.7, 0.85])
high = points.max(axis=0) + np.array([0.7, 0.7, 0.7])
low[2] = min(low[2], 0)
camera = mujoco.MjvCamera()
mujoco.mjv_defaultCamera(camera)
camera.azimuth = -140
camera.elevation = -20
camera.lookat[:] = [0, 0, 0.7]
camera.distance = 3.0
status['camera'] = dict(mode='XY-follow only; fixed world Z=0.7, scale and angles; shared-coordinate XY inset', distance=camera.distance, azimuth=camera.azimuth, elevation=camera.elevation)
center = (low[:2] + high[:2]) / 2
span = max(float(np.max(high[:2] - low[:2])), 2.0)
tracks = [(poses[:, :2] - center) / span * 118 for poses in (prediction, gt)]
track_colors = [(118, 213, 173), (115, 175, 245)]
small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 14)
temp = directory / 'comparison.partial.mp4'
command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-y', '-threads', '1', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', '1280x480', '-r', str(case['fps']), '-i', 'pipe:0']
with_audio = case['modality'] in ('music', 'speech')
if with_audio:
command += ['-threads', '1', '-i', str(directory / case['input_media'])]
command += ['-map', '0:v:0', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', '-threads', '2', '-pix_fmt', 'yuv420p', '-movflags', '+faststart']
if with_audio:
command += ['-map', '1:a:0', '-af', 'apad', '-c:a', 'aac', '-b:a', '128k', '-shortest']
command += [str(temp)]
with (directory / 'ffmpeg.log').open('ab') as log:
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=log)
try:
for frame_id, (pred, truth) in enumerate(zip(prediction, gt)):
images = []
for pose in (pred, truth):
data.qpos[:] = pose
mujoco.mj_forward(model, data)
camera.lookat[:2] = pose[:2]
renderer.update_scene(data, camera=camera)
renderer.scene.flags[mujoco.mjtRndFlag.mjRND_SHADOW] = False
renderer.scene.flags[mujoco.mjtRndFlag.mjRND_REFLECTION] = False
images.append(renderer.render())
frame = Image.fromarray(np.concatenate(images, axis=1))
draw = ImageDraw.Draw(frame)
draw.rectangle((0, 0, 1280, 34), fill=(16, 23, 32))
# <modify> Label the actual inference checkpoint rather than the historical fixed step.
# draw.text((14, 4), 'Prediction | ckpt 2000', font=font, fill=(118, 213, 173))
# <modify> A visible protocol suffix distinguishes corrected and historical videos at the same checkpoint.
# draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]}', font=font, fill=(118, 213, 173))
# <modify> Show the actual video-progress protocol without changing rendering or motion.
# protocol_label = 'root40_v2' if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2' else 'original_v1'
protocol_label = {'root40_current_heading_fp32_v2': 'root40_v2',
'root40_current_heading_fp32_video_local_progress_v3': 'root40_v3',
# <modify> Label the exact new inference precision on rendered videos.
'root40_current_heading_fp32_video_local_progress_projection_fp32_v4': 'root40_v4 input FP32',
'original_cpu_bf16_rtc_v1': 'original_v1'}[case['inference_protocol_version']]
draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]} | {protocol_label}', font=font, fill=(118, 213, 173))
draw.text((654, 4), 'Ground truth', font=font, fill=(115, 175, 245))
draw.text((1030, 4), f'{frame_id / case["fps"]:.2f} / {len(gt) / case["fps"]:.2f} s', font=font, fill='white')
draw.line((640, 0, 640, 480), fill=(31, 42, 53), width=2)
for offset, pose in [(0, pred), (640, truth)]:
draw.rectangle((offset, 449, offset + 639, 479), fill=(16, 23, 32))
color = (255, 146, 146) if pose[2] < 0 else (222, 232, 240)
draw.text((offset + 10, 454), f'Root XYZ: {pose[0]:+.2f}, {pose[1]:+.2f}, {pose[2]:+.2f} m | XY-follow view', font=small_font, fill=color)
draw.rectangle((offset + 482, 42, offset + 632, 198), fill=(16, 23, 32))
draw.text((offset + 489, 46), 'World XY paths', font=small_font, fill=(210, 223, 234))
draw.text((offset + 489, 179), f'Width {span:.1f} m', font=small_font, fill=(147, 163, 181))
for track, track_color in zip(tracks, track_colors):
coords = [(float(offset + 557 + xy[0]), float(121 - xy[1])) for xy in track]
draw.line(coords, fill=track_color, width=2)
x, y = coords[frame_id]
draw.ellipse((x - 4, y - 4, x + 4, y + 4), fill=track_color, outline='white')
if frame_id == 0:
frame.save(directory / 'poster.jpg', quality=90)
process.stdin.write(np.asarray(frame).tobytes())
if frame_id % 30 == 0:
status.update(rendered_frames=frame_id + 1, render_total_frames=len(gt), render_updated_at=time.time())
save_json(status_path, status)
finally:
process.stdin.close()
code = process.wait(timeout=60)
if code:
raise RuntimeError(f'FFmpeg failed with code {code}; see {directory / "ffmpeg.log"}')
temp.replace(directory / 'comparison.mp4')
status.update(status='ready', rendered_frames=len(gt), render_seconds=time.time() - started, render_updated_at=time.time())
print(f'{case["id"]}: rendered {len(gt)} frames in {status["render_seconds"]:.1f}s', flush=True)
except Exception:
status.update(status='error', error=traceback.format_exc(), render_updated_at=time.time())
print(status['error'], flush=True)
save_json(status_path, status)
def main():
# <modify> Keep original output unchanged and allow checkpoint-specific render jobs.
# <modify> Delay imports until the renderer environment has been configured.
# global OUTPUT
global OUTPUT, mujoco, np, GL, Image, ImageDraw, ImageFont
parser = argparse.ArgumentParser()
parser.add_argument('--watch', action='store_true')
parser.add_argument('--output-dir', type=Path, default=ROOT / 'output')
parser.add_argument('--renderer', choices=('software', 'nvidia', 'gpu'), default='software')
parser.add_argument('--gpu-index', type=int, default=0, help='EGL device index; not remapped by CUDA_VISIBLE_DEVICES')
parser.add_argument('--nvidia-egl-vendor', type=Path, default=Path('/usr/share/glvnd/egl_vendor.d/10_nvidia.json'))
parser.add_argument('--case-ids', nargs='+', help='Only render these manifest case IDs')
parser.add_argument('--max-frames', type=int, default=0, help='Short render; requires manifest render_smoke_test=true')
args = parser.parse_args()
OUTPUT = args.output_dir.resolve(strict=True)
manifest = json.loads((OUTPUT / 'manifest.json').read_text())
if args.max_frames < 0:
parser.error('--max-frames must be nonnegative')
if args.max_frames and manifest.get('render_smoke_test') is not True:
parser.error('--max-frames requires a separate smoke output with manifest render_smoke_test=true')
selected_ids = set(args.case_ids or [case['id'] for case in manifest['cases']])
unknown_ids = selected_ids - {case['id'] for case in manifest['cases']}
if unknown_ids:
parser.error(f'Unknown case IDs: {sorted(unknown_ids)}')
configure_renderer(args.renderer, args.gpu_index, args.nvidia_egl_vendor)
os.nice(10)
import mujoco
import numpy as np
from OpenGL import GL
from PIL import Image, ImageDraw, ImageFont
for case in manifest['cases']:
case['checkpoint_step'] = manifest['checkpoint_step']
case['inference_protocol_version'] = manifest.get('inference_protocol_version', 'original_cpu_bf16_rtc_v1')
case['gallery_version'] = manifest.get('gallery_version', f'ckpt_{manifest["checkpoint_step"]}')
case['root40_contact_protocol'] = manifest.get('root40_contact_protocol', 'legacy_joint_delta_v1')
model = mujoco.MjModel.from_xml_path(str(XML))
model.vis.quality.offsamples = 0
model.vis.quality.shadowsize = 512
data = mujoco.MjData(model)
renderer = mujoco.Renderer(model, height=480, width=640)
backend = GL.glGetString(GL.GL_RENDERER).decode()
# <modify> Never label software fallback as GPU acceleration.
# if 'llvmpipe' not in backend.lower():
# raise RuntimeError(f'Refusing a non-software renderer during training: {backend}')
vendor = GL.glGetString(GL.GL_VENDOR).decode()
is_software = 'llvmpipe' in backend.lower()
is_nvidia = 'nvidia' in vendor.lower() and not is_software
if (args.renderer == 'software' and not is_software) or (args.renderer != 'software' and not is_nvidia):
renderer.close()
raise RuntimeError(f'Requested {args.renderer}, got GL vendor={vendor!r}, renderer={backend!r}')
backend_info = dict(renderer_requested=args.renderer, render_backend='software' if is_software else 'nvidia', gl_renderer=backend, gl_vendor=vendor, egl_device_id=None if is_software else args.gpu_index)
print('Renderer:', backend, flush=True)
font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 20)
attempted = set()
try:
while True:
for case in manifest['cases']:
if case['id'] not in selected_ids:
continue
dest = OUTPUT / case['directory']
# <modify> Retry interrupted renders unless both the video and its ready marker are present.
# if (dest / 'prediction.npz').exists() and not (dest / 'comparison.mp4').exists() and case['id'] not in attempted:
marker = dest / 'render_status.json'
# <modify> Reject mixed-protocol completed renders instead of quietly reusing them.
# complete = (dest / 'comparison.mp4').exists() and marker.exists() and json.loads(marker.read_text()).get('status') == 'ready'
cached = json.loads(marker.read_text()) if marker.exists() else {}
# <modify> Never reuse a cached render from another v3/v2 protocol.
# if cached and case['inference_protocol_version'] == 'root40_current_heading_fp32_v2':
# <modify> Do not reuse a BF16 render for a projection-FP32 job.
# if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'):
if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'):
assert cached.get('inference_protocol_version') == case['inference_protocol_version']
assert cached.get('gallery_version') == case['gallery_version']
assert cached.get('root40_contact_protocol', 'legacy_joint_delta_v1') == case['root40_contact_protocol']
complete = (dest / 'comparison.mp4').exists() and cached.get('status') == 'ready'
# <modify> Keep completed CPU renders unchanged, including their actual/unknown historical backend.
if complete and cached.get('render_backend'):
cached_renderer = str(cached.get('gl_renderer', '')).lower()
cached_vendor = str(cached.get('gl_vendor', '')).lower()
if not cached_renderer or (cached['render_backend'] == 'software' and 'llvmpipe' not in cached_renderer) or (cached['render_backend'] == 'nvidia' and ('nvidia' not in cached_vendor or 'llvmpipe' in cached_renderer)):
raise RuntimeError(f'Inconsistent cached GL renderer metadata for {case["id"]}')
if complete and cached.get('smoke_max_frames', 0) != args.max_frames:
raise RuntimeError(f'Incompatible smoke render already exists for {case["id"]}')
if (dest / 'prediction.npz').exists() and not complete and case['id'] not in attempted:
attempted.add(case['id'])
# render(case, model, data, renderer, font)
render(case, model, data, renderer, font, backend_info, args.max_frames)
cached = json.loads(marker.read_text())
if cached.get('status') == 'ready':
manifest.setdefault('render_backends', {})[case['id']] = {key: cached.get(key) for key in ('render_backend', 'gl_renderer', 'gl_vendor', 'egl_device_id')}
save_json(OUTPUT / 'manifest.json', manifest)
if not args.watch:
break
status_path = OUTPUT / 'inference_status.json'
if status_path.exists():
status = json.loads(status_path.read_text())
proc = Path('/proc') / str(status.get('pid', -1))
if status['status'] == 'finished' or not proc.exists():
break
time.sleep(5)
finally:
renderer.close()
if __name__ == '__main__':
main()