File size: 19,747 Bytes
f55ddc0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 | import argparse
import json
import os
import subprocess
import time
import traceback
from pathlib import Path
# <modify> Select software or NVIDIA EGL before importing MuJoCo and OpenGL.
# os.environ['CUDA_VISIBLE_DEVICES'] = ''
# os.environ['MUJOCO_GL'] = 'egl'
# os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1'
# os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe'
# os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json'
# os.environ['LP_NUM_THREADS'] = '2'
# os.nice(10)
# import mujoco
# import numpy as np
# from OpenGL import GL
# from PIL import Image, ImageDraw, ImageFont
ROOT = Path(__file__).resolve().parent
OUTPUT = ROOT / 'output'
XML = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml')
def configure_renderer(renderer, gpu_index, nvidia_egl_vendor):
os.environ['MUJOCO_GL'] = 'egl'
os.environ['PYOPENGL_PLATFORM'] = 'egl'
if renderer == 'software':
os.environ['CUDA_VISIBLE_DEVICES'] = ''
os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1'
os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe'
os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json'
os.environ['LP_NUM_THREADS'] = '2'
os.environ.pop('MUJOCO_EGL_DEVICE_ID', None)
else:
if gpu_index < 0:
raise ValueError('--gpu-index must be nonnegative')
if not nvidia_egl_vendor.is_file():
raise FileNotFoundError(f'NVIDIA EGL vendor file is missing: {nvidia_egl_vendor}. Use --renderer software or supply --nvidia-egl-vendor; CUDA alone does not provide OpenGL.')
library = json.loads(nvidia_egl_vendor.read_text()).get('ICD', {}).get('library_path', '')
if 'nvidia' not in library.lower():
raise ValueError(f'Expected an NVIDIA EGL library in {nvidia_egl_vendor}, got {library!r}')
for name in ('LIBGL_ALWAYS_SOFTWARE', 'MESA_LOADER_DRIVER_OVERRIDE', 'GALLIUM_DRIVER', 'LP_NUM_THREADS', 'LIBGL_ALWAYS_INDIRECT'):
os.environ.pop(name, None)
os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = str(nvidia_egl_vendor.resolve())
os.environ['__GLX_VENDOR_LIBRARY_NAME'] = 'nvidia'
# <modify> MuJoCo indexes eglQueryDevicesEXT directly; CUDA_VISIBLE_DEVICES does not remap this index.
os.environ['MUJOCO_EGL_DEVICE_ID'] = str(gpu_index)
def save_json(path, value):
temp = path.with_suffix('.tmp')
temp.write_text(json.dumps(value, indent=2, allow_nan=False))
temp.replace(path)
# <modify> Record the measured GL device per artifact, without changing camera or motion rendering.
# def render(case, model, data, renderer, font):
def render(case, model, data, renderer, font, backend_info, max_frames=0):
directory = OUTPUT / case['directory']
started = time.time()
status_path = directory / 'render_status.json'
status = dict(status='rendering', render_started_at=started, render_updated_at=started)
status.update(backend_info)
status['smoke_max_frames'] = max_frames
# <modify> Keep render identity tied to the exact prediction protocol, not only its checkpoint step.
status.update(inference_protocol_version=case['inference_protocol_version'], gallery_version=case['gallery_version'])
# <modify> Keep contact-label provenance when rendering a corrected checkpoint.
status['root40_contact_protocol'] = case.get('root40_contact_protocol', 'legacy_joint_delta_v1')
save_json(status_path, status)
try:
with np.load(directory / 'gt.npz') as file:
gt = file['qpos_36']
names = file['joint_names'].tolist()
assert names == [mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, i) for i in range(1, model.njnt)]
with np.load(directory / 'prediction.npz') as file:
prediction = file['qpos_36']
# <modify> Both corrected protocols require exact prediction provenance.
# if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2':
# <modify> Validate v4 provenance without changing GMR camera, geometry, or motion.
# if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'):
if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'):
assert str(file['inference_protocol_version'].item()) == case['inference_protocol_version']
assert str(file['gallery_version'].item()) == case['gallery_version']
contact_protocol = str(file['root40_contact_protocol'].item()) if 'root40_contact_protocol' in file else 'legacy_joint_delta_v1'
assert contact_protocol == case.get('root40_contact_protocol', 'legacy_joint_delta_v1')
if case['inference_protocol_version'].endswith('_projection_fp32_v4'):
assert str(file['action_input_projection_dtype'].item()) == 'float32'
assert str(file['action_head_remaining_dtype'].item()) == 'bfloat16'
assert str(file['vlm_dtype'].item()) == 'bfloat16'
assert str(file['sampler_dtype'].item()) == 'float32'
assert gt.shape == prediction.shape and gt.shape[1] == model.nq == 36
# <modify> Optional short smoke runs only use explicitly marked, separate output directories.
if max_frames:
gt = gt[:max_frames]
prediction = prediction[:max_frames]
points = np.concatenate([prediction[:, :3], gt[:, :3]])
low = points.min(axis=0) - np.array([0.7, 0.7, 0.85])
high = points.max(axis=0) + np.array([0.7, 0.7, 0.7])
low[2] = min(low[2], 0)
camera = mujoco.MjvCamera()
mujoco.mjv_defaultCamera(camera)
camera.azimuth = -140
camera.elevation = -20
camera.lookat[:] = [0, 0, 0.7]
camera.distance = 3.0
status['camera'] = dict(mode='XY-follow only; fixed world Z=0.7, scale and angles; shared-coordinate XY inset', distance=camera.distance, azimuth=camera.azimuth, elevation=camera.elevation)
center = (low[:2] + high[:2]) / 2
span = max(float(np.max(high[:2] - low[:2])), 2.0)
tracks = [(poses[:, :2] - center) / span * 118 for poses in (prediction, gt)]
track_colors = [(118, 213, 173), (115, 175, 245)]
small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 14)
temp = directory / 'comparison.partial.mp4'
command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-y', '-threads', '1', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', '1280x480', '-r', str(case['fps']), '-i', 'pipe:0']
with_audio = case['modality'] in ('music', 'speech')
if with_audio:
command += ['-threads', '1', '-i', str(directory / case['input_media'])]
command += ['-map', '0:v:0', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', '-threads', '2', '-pix_fmt', 'yuv420p', '-movflags', '+faststart']
if with_audio:
command += ['-map', '1:a:0', '-af', 'apad', '-c:a', 'aac', '-b:a', '128k', '-shortest']
command += [str(temp)]
with (directory / 'ffmpeg.log').open('ab') as log:
process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=log)
try:
for frame_id, (pred, truth) in enumerate(zip(prediction, gt)):
images = []
for pose in (pred, truth):
data.qpos[:] = pose
mujoco.mj_forward(model, data)
camera.lookat[:2] = pose[:2]
renderer.update_scene(data, camera=camera)
renderer.scene.flags[mujoco.mjtRndFlag.mjRND_SHADOW] = False
renderer.scene.flags[mujoco.mjtRndFlag.mjRND_REFLECTION] = False
images.append(renderer.render())
frame = Image.fromarray(np.concatenate(images, axis=1))
draw = ImageDraw.Draw(frame)
draw.rectangle((0, 0, 1280, 34), fill=(16, 23, 32))
# <modify> Label the actual inference checkpoint rather than the historical fixed step.
# draw.text((14, 4), 'Prediction | ckpt 2000', font=font, fill=(118, 213, 173))
# <modify> A visible protocol suffix distinguishes corrected and historical videos at the same checkpoint.
# draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]}', font=font, fill=(118, 213, 173))
# <modify> Show the actual video-progress protocol without changing rendering or motion.
# protocol_label = 'root40_v2' if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2' else 'original_v1'
protocol_label = {'root40_current_heading_fp32_v2': 'root40_v2',
'root40_current_heading_fp32_video_local_progress_v3': 'root40_v3',
# <modify> Label the exact new inference precision on rendered videos.
'root40_current_heading_fp32_video_local_progress_projection_fp32_v4': 'root40_v4 input FP32',
'original_cpu_bf16_rtc_v1': 'original_v1'}[case['inference_protocol_version']]
draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]} | {protocol_label}', font=font, fill=(118, 213, 173))
draw.text((654, 4), 'Ground truth', font=font, fill=(115, 175, 245))
draw.text((1030, 4), f'{frame_id / case["fps"]:.2f} / {len(gt) / case["fps"]:.2f} s', font=font, fill='white')
draw.line((640, 0, 640, 480), fill=(31, 42, 53), width=2)
for offset, pose in [(0, pred), (640, truth)]:
draw.rectangle((offset, 449, offset + 639, 479), fill=(16, 23, 32))
color = (255, 146, 146) if pose[2] < 0 else (222, 232, 240)
draw.text((offset + 10, 454), f'Root XYZ: {pose[0]:+.2f}, {pose[1]:+.2f}, {pose[2]:+.2f} m | XY-follow view', font=small_font, fill=color)
draw.rectangle((offset + 482, 42, offset + 632, 198), fill=(16, 23, 32))
draw.text((offset + 489, 46), 'World XY paths', font=small_font, fill=(210, 223, 234))
draw.text((offset + 489, 179), f'Width {span:.1f} m', font=small_font, fill=(147, 163, 181))
for track, track_color in zip(tracks, track_colors):
coords = [(float(offset + 557 + xy[0]), float(121 - xy[1])) for xy in track]
draw.line(coords, fill=track_color, width=2)
x, y = coords[frame_id]
draw.ellipse((x - 4, y - 4, x + 4, y + 4), fill=track_color, outline='white')
if frame_id == 0:
frame.save(directory / 'poster.jpg', quality=90)
process.stdin.write(np.asarray(frame).tobytes())
if frame_id % 30 == 0:
status.update(rendered_frames=frame_id + 1, render_total_frames=len(gt), render_updated_at=time.time())
save_json(status_path, status)
finally:
process.stdin.close()
code = process.wait(timeout=60)
if code:
raise RuntimeError(f'FFmpeg failed with code {code}; see {directory / "ffmpeg.log"}')
temp.replace(directory / 'comparison.mp4')
status.update(status='ready', rendered_frames=len(gt), render_seconds=time.time() - started, render_updated_at=time.time())
print(f'{case["id"]}: rendered {len(gt)} frames in {status["render_seconds"]:.1f}s', flush=True)
except Exception:
status.update(status='error', error=traceback.format_exc(), render_updated_at=time.time())
print(status['error'], flush=True)
save_json(status_path, status)
def main():
# <modify> Keep original output unchanged and allow checkpoint-specific render jobs.
# <modify> Delay imports until the renderer environment has been configured.
# global OUTPUT
global OUTPUT, mujoco, np, GL, Image, ImageDraw, ImageFont
parser = argparse.ArgumentParser()
parser.add_argument('--watch', action='store_true')
parser.add_argument('--output-dir', type=Path, default=ROOT / 'output')
parser.add_argument('--renderer', choices=('software', 'nvidia', 'gpu'), default='software')
parser.add_argument('--gpu-index', type=int, default=0, help='EGL device index; not remapped by CUDA_VISIBLE_DEVICES')
parser.add_argument('--nvidia-egl-vendor', type=Path, default=Path('/usr/share/glvnd/egl_vendor.d/10_nvidia.json'))
parser.add_argument('--case-ids', nargs='+', help='Only render these manifest case IDs')
parser.add_argument('--max-frames', type=int, default=0, help='Short render; requires manifest render_smoke_test=true')
args = parser.parse_args()
OUTPUT = args.output_dir.resolve(strict=True)
manifest = json.loads((OUTPUT / 'manifest.json').read_text())
if args.max_frames < 0:
parser.error('--max-frames must be nonnegative')
if args.max_frames and manifest.get('render_smoke_test') is not True:
parser.error('--max-frames requires a separate smoke output with manifest render_smoke_test=true')
selected_ids = set(args.case_ids or [case['id'] for case in manifest['cases']])
unknown_ids = selected_ids - {case['id'] for case in manifest['cases']}
if unknown_ids:
parser.error(f'Unknown case IDs: {sorted(unknown_ids)}')
configure_renderer(args.renderer, args.gpu_index, args.nvidia_egl_vendor)
os.nice(10)
import mujoco
import numpy as np
from OpenGL import GL
from PIL import Image, ImageDraw, ImageFont
for case in manifest['cases']:
case['checkpoint_step'] = manifest['checkpoint_step']
case['inference_protocol_version'] = manifest.get('inference_protocol_version', 'original_cpu_bf16_rtc_v1')
case['gallery_version'] = manifest.get('gallery_version', f'ckpt_{manifest["checkpoint_step"]}')
case['root40_contact_protocol'] = manifest.get('root40_contact_protocol', 'legacy_joint_delta_v1')
model = mujoco.MjModel.from_xml_path(str(XML))
model.vis.quality.offsamples = 0
model.vis.quality.shadowsize = 512
data = mujoco.MjData(model)
renderer = mujoco.Renderer(model, height=480, width=640)
backend = GL.glGetString(GL.GL_RENDERER).decode()
# <modify> Never label software fallback as GPU acceleration.
# if 'llvmpipe' not in backend.lower():
# raise RuntimeError(f'Refusing a non-software renderer during training: {backend}')
vendor = GL.glGetString(GL.GL_VENDOR).decode()
is_software = 'llvmpipe' in backend.lower()
is_nvidia = 'nvidia' in vendor.lower() and not is_software
if (args.renderer == 'software' and not is_software) or (args.renderer != 'software' and not is_nvidia):
renderer.close()
raise RuntimeError(f'Requested {args.renderer}, got GL vendor={vendor!r}, renderer={backend!r}')
backend_info = dict(renderer_requested=args.renderer, render_backend='software' if is_software else 'nvidia', gl_renderer=backend, gl_vendor=vendor, egl_device_id=None if is_software else args.gpu_index)
print('Renderer:', backend, flush=True)
font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 20)
attempted = set()
try:
while True:
for case in manifest['cases']:
if case['id'] not in selected_ids:
continue
dest = OUTPUT / case['directory']
# <modify> Retry interrupted renders unless both the video and its ready marker are present.
# if (dest / 'prediction.npz').exists() and not (dest / 'comparison.mp4').exists() and case['id'] not in attempted:
marker = dest / 'render_status.json'
# <modify> Reject mixed-protocol completed renders instead of quietly reusing them.
# complete = (dest / 'comparison.mp4').exists() and marker.exists() and json.loads(marker.read_text()).get('status') == 'ready'
cached = json.loads(marker.read_text()) if marker.exists() else {}
# <modify> Never reuse a cached render from another v3/v2 protocol.
# if cached and case['inference_protocol_version'] == 'root40_current_heading_fp32_v2':
# <modify> Do not reuse a BF16 render for a projection-FP32 job.
# if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'):
if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'):
assert cached.get('inference_protocol_version') == case['inference_protocol_version']
assert cached.get('gallery_version') == case['gallery_version']
assert cached.get('root40_contact_protocol', 'legacy_joint_delta_v1') == case['root40_contact_protocol']
complete = (dest / 'comparison.mp4').exists() and cached.get('status') == 'ready'
# <modify> Keep completed CPU renders unchanged, including their actual/unknown historical backend.
if complete and cached.get('render_backend'):
cached_renderer = str(cached.get('gl_renderer', '')).lower()
cached_vendor = str(cached.get('gl_vendor', '')).lower()
if not cached_renderer or (cached['render_backend'] == 'software' and 'llvmpipe' not in cached_renderer) or (cached['render_backend'] == 'nvidia' and ('nvidia' not in cached_vendor or 'llvmpipe' in cached_renderer)):
raise RuntimeError(f'Inconsistent cached GL renderer metadata for {case["id"]}')
if complete and cached.get('smoke_max_frames', 0) != args.max_frames:
raise RuntimeError(f'Incompatible smoke render already exists for {case["id"]}')
if (dest / 'prediction.npz').exists() and not complete and case['id'] not in attempted:
attempted.add(case['id'])
# render(case, model, data, renderer, font)
render(case, model, data, renderer, font, backend_info, args.max_frames)
cached = json.loads(marker.read_text())
if cached.get('status') == 'ready':
manifest.setdefault('render_backends', {})[case['id']] = {key: cached.get(key) for key in ('render_backend', 'gl_renderer', 'gl_vendor', 'egl_device_id')}
save_json(OUTPUT / 'manifest.json', manifest)
if not args.watch:
break
status_path = OUTPUT / 'inference_status.json'
if status_path.exists():
status = json.loads(status_path.read_text())
proc = Path('/proc') / str(status.get('pid', -1))
if status['status'] == 'finished' or not proc.exists():
break
time.sleep(5)
finally:
renderer.close()
if __name__ == '__main__':
main()
|