File size: 19,747 Bytes
f55ddc0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
import argparse
import json
import os
import subprocess
import time
import traceback
from pathlib import Path

# <modify> Select software or NVIDIA EGL before importing MuJoCo and OpenGL.
# os.environ['CUDA_VISIBLE_DEVICES'] = ''
# os.environ['MUJOCO_GL'] = 'egl'
# os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1'
# os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe'
# os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json'
# os.environ['LP_NUM_THREADS'] = '2'
# os.nice(10)
# import mujoco
# import numpy as np
# from OpenGL import GL
# from PIL import Image, ImageDraw, ImageFont

ROOT = Path(__file__).resolve().parent
OUTPUT = ROOT / 'output'
XML = Path('/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/user_workspace/buwendong/user/code/GMR/assets/unitree_g1/g1_mocap_29dof.xml')


def configure_renderer(renderer, gpu_index, nvidia_egl_vendor):
    os.environ['MUJOCO_GL'] = 'egl'
    os.environ['PYOPENGL_PLATFORM'] = 'egl'
    if renderer == 'software':
        os.environ['CUDA_VISIBLE_DEVICES'] = ''
        os.environ['LIBGL_ALWAYS_SOFTWARE'] = '1'
        os.environ['MESA_LOADER_DRIVER_OVERRIDE'] = 'llvmpipe'
        os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = '/usr/share/glvnd/egl_vendor.d/50_mesa.json'
        os.environ['LP_NUM_THREADS'] = '2'
        os.environ.pop('MUJOCO_EGL_DEVICE_ID', None)
    else:
        if gpu_index < 0:
            raise ValueError('--gpu-index must be nonnegative')
        if not nvidia_egl_vendor.is_file():
            raise FileNotFoundError(f'NVIDIA EGL vendor file is missing: {nvidia_egl_vendor}. Use --renderer software or supply --nvidia-egl-vendor; CUDA alone does not provide OpenGL.')
        library = json.loads(nvidia_egl_vendor.read_text()).get('ICD', {}).get('library_path', '')
        if 'nvidia' not in library.lower():
            raise ValueError(f'Expected an NVIDIA EGL library in {nvidia_egl_vendor}, got {library!r}')
        for name in ('LIBGL_ALWAYS_SOFTWARE', 'MESA_LOADER_DRIVER_OVERRIDE', 'GALLIUM_DRIVER', 'LP_NUM_THREADS', 'LIBGL_ALWAYS_INDIRECT'):
            os.environ.pop(name, None)
        os.environ['__EGL_VENDOR_LIBRARY_FILENAMES'] = str(nvidia_egl_vendor.resolve())
        os.environ['__GLX_VENDOR_LIBRARY_NAME'] = 'nvidia'
        # <modify> MuJoCo indexes eglQueryDevicesEXT directly; CUDA_VISIBLE_DEVICES does not remap this index.
        os.environ['MUJOCO_EGL_DEVICE_ID'] = str(gpu_index)


def save_json(path, value):
    temp = path.with_suffix('.tmp')
    temp.write_text(json.dumps(value, indent=2, allow_nan=False))
    temp.replace(path)


# <modify> Record the measured GL device per artifact, without changing camera or motion rendering.
# def render(case, model, data, renderer, font):
def render(case, model, data, renderer, font, backend_info, max_frames=0):
    directory = OUTPUT / case['directory']
    started = time.time()
    status_path = directory / 'render_status.json'
    status = dict(status='rendering', render_started_at=started, render_updated_at=started)
    status.update(backend_info)
    status['smoke_max_frames'] = max_frames
    # <modify> Keep render identity tied to the exact prediction protocol, not only its checkpoint step.
    status.update(inference_protocol_version=case['inference_protocol_version'], gallery_version=case['gallery_version'])
    # <modify> Keep contact-label provenance when rendering a corrected checkpoint.
    status['root40_contact_protocol'] = case.get('root40_contact_protocol', 'legacy_joint_delta_v1')
    save_json(status_path, status)
    try:
        with np.load(directory / 'gt.npz') as file:
            gt = file['qpos_36']
            names = file['joint_names'].tolist()
        assert names == [mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, i) for i in range(1, model.njnt)]
        with np.load(directory / 'prediction.npz') as file:
            prediction = file['qpos_36']
            # <modify> Both corrected protocols require exact prediction provenance.
            # if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2':
            # <modify> Validate v4 provenance without changing GMR camera, geometry, or motion.
            # if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'):
            if case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'):
                assert str(file['inference_protocol_version'].item()) == case['inference_protocol_version']
                assert str(file['gallery_version'].item()) == case['gallery_version']
                contact_protocol = str(file['root40_contact_protocol'].item()) if 'root40_contact_protocol' in file else 'legacy_joint_delta_v1'
                assert contact_protocol == case.get('root40_contact_protocol', 'legacy_joint_delta_v1')
                if case['inference_protocol_version'].endswith('_projection_fp32_v4'):
                    assert str(file['action_input_projection_dtype'].item()) == 'float32'
                    assert str(file['action_head_remaining_dtype'].item()) == 'bfloat16'
                    assert str(file['vlm_dtype'].item()) == 'bfloat16'
                    assert str(file['sampler_dtype'].item()) == 'float32'
        assert gt.shape == prediction.shape and gt.shape[1] == model.nq == 36
        # <modify> Optional short smoke runs only use explicitly marked, separate output directories.
        if max_frames:
            gt = gt[:max_frames]
            prediction = prediction[:max_frames]
        points = np.concatenate([prediction[:, :3], gt[:, :3]])
        low = points.min(axis=0) - np.array([0.7, 0.7, 0.85])
        high = points.max(axis=0) + np.array([0.7, 0.7, 0.7])
        low[2] = min(low[2], 0)
        camera = mujoco.MjvCamera()
        mujoco.mjv_defaultCamera(camera)
        camera.azimuth = -140
        camera.elevation = -20
        camera.lookat[:] = [0, 0, 0.7]
        camera.distance = 3.0
        status['camera'] = dict(mode='XY-follow only; fixed world Z=0.7, scale and angles; shared-coordinate XY inset', distance=camera.distance, azimuth=camera.azimuth, elevation=camera.elevation)
        center = (low[:2] + high[:2]) / 2
        span = max(float(np.max(high[:2] - low[:2])), 2.0)
        tracks = [(poses[:, :2] - center) / span * 118 for poses in (prediction, gt)]
        track_colors = [(118, 213, 173), (115, 175, 245)]
        small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 14)
        temp = directory / 'comparison.partial.mp4'
        command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-y', '-threads', '1', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', '1280x480', '-r', str(case['fps']), '-i', 'pipe:0']
        with_audio = case['modality'] in ('music', 'speech')
        if with_audio:
            command += ['-threads', '1', '-i', str(directory / case['input_media'])]
        command += ['-map', '0:v:0', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '21', '-threads', '2', '-pix_fmt', 'yuv420p', '-movflags', '+faststart']
        if with_audio:
            command += ['-map', '1:a:0', '-af', 'apad', '-c:a', 'aac', '-b:a', '128k', '-shortest']
        command += [str(temp)]
        with (directory / 'ffmpeg.log').open('ab') as log:
            process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=log)
            try:
                for frame_id, (pred, truth) in enumerate(zip(prediction, gt)):
                    images = []
                    for pose in (pred, truth):
                        data.qpos[:] = pose
                        mujoco.mj_forward(model, data)
                        camera.lookat[:2] = pose[:2]
                        renderer.update_scene(data, camera=camera)
                        renderer.scene.flags[mujoco.mjtRndFlag.mjRND_SHADOW] = False
                        renderer.scene.flags[mujoco.mjtRndFlag.mjRND_REFLECTION] = False
                        images.append(renderer.render())
                    frame = Image.fromarray(np.concatenate(images, axis=1))
                    draw = ImageDraw.Draw(frame)
                    draw.rectangle((0, 0, 1280, 34), fill=(16, 23, 32))
                    # <modify> Label the actual inference checkpoint rather than the historical fixed step.
                    # draw.text((14, 4), 'Prediction | ckpt 2000', font=font, fill=(118, 213, 173))
                    # <modify> A visible protocol suffix distinguishes corrected and historical videos at the same checkpoint.
                    # draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]}', font=font, fill=(118, 213, 173))
                    # <modify> Show the actual video-progress protocol without changing rendering or motion.
                    # protocol_label = 'root40_v2' if case['inference_protocol_version'] == 'root40_current_heading_fp32_v2' else 'original_v1'
                    protocol_label = {'root40_current_heading_fp32_v2': 'root40_v2',
                                      'root40_current_heading_fp32_video_local_progress_v3': 'root40_v3',
                                      # <modify> Label the exact new inference precision on rendered videos.
                                      'root40_current_heading_fp32_video_local_progress_projection_fp32_v4': 'root40_v4 input FP32',
                                      'original_cpu_bf16_rtc_v1': 'original_v1'}[case['inference_protocol_version']]
                    draw.text((14, 4), f'Prediction | ckpt {case["checkpoint_step"]} | {protocol_label}', font=font, fill=(118, 213, 173))
                    draw.text((654, 4), 'Ground truth', font=font, fill=(115, 175, 245))
                    draw.text((1030, 4), f'{frame_id / case["fps"]:.2f} / {len(gt) / case["fps"]:.2f} s', font=font, fill='white')
                    draw.line((640, 0, 640, 480), fill=(31, 42, 53), width=2)
                    for offset, pose in [(0, pred), (640, truth)]:
                        draw.rectangle((offset, 449, offset + 639, 479), fill=(16, 23, 32))
                        color = (255, 146, 146) if pose[2] < 0 else (222, 232, 240)
                        draw.text((offset + 10, 454), f'Root XYZ: {pose[0]:+.2f}, {pose[1]:+.2f}, {pose[2]:+.2f} m | XY-follow view', font=small_font, fill=color)
                        draw.rectangle((offset + 482, 42, offset + 632, 198), fill=(16, 23, 32))
                        draw.text((offset + 489, 46), 'World XY paths', font=small_font, fill=(210, 223, 234))
                        draw.text((offset + 489, 179), f'Width {span:.1f} m', font=small_font, fill=(147, 163, 181))
                        for track, track_color in zip(tracks, track_colors):
                            coords = [(float(offset + 557 + xy[0]), float(121 - xy[1])) for xy in track]
                            draw.line(coords, fill=track_color, width=2)
                            x, y = coords[frame_id]
                            draw.ellipse((x - 4, y - 4, x + 4, y + 4), fill=track_color, outline='white')
                    if frame_id == 0:
                        frame.save(directory / 'poster.jpg', quality=90)
                    process.stdin.write(np.asarray(frame).tobytes())
                    if frame_id % 30 == 0:
                        status.update(rendered_frames=frame_id + 1, render_total_frames=len(gt), render_updated_at=time.time())
                        save_json(status_path, status)
            finally:
                process.stdin.close()
                code = process.wait(timeout=60)
            if code:
                raise RuntimeError(f'FFmpeg failed with code {code}; see {directory / "ffmpeg.log"}')
        temp.replace(directory / 'comparison.mp4')
        status.update(status='ready', rendered_frames=len(gt), render_seconds=time.time() - started, render_updated_at=time.time())
        print(f'{case["id"]}: rendered {len(gt)} frames in {status["render_seconds"]:.1f}s', flush=True)
    except Exception:
        status.update(status='error', error=traceback.format_exc(), render_updated_at=time.time())
        print(status['error'], flush=True)
    save_json(status_path, status)


def main():
    # <modify> Keep original output unchanged and allow checkpoint-specific render jobs.
    # <modify> Delay imports until the renderer environment has been configured.
    # global OUTPUT
    global OUTPUT, mujoco, np, GL, Image, ImageDraw, ImageFont
    parser = argparse.ArgumentParser()
    parser.add_argument('--watch', action='store_true')
    parser.add_argument('--output-dir', type=Path, default=ROOT / 'output')
    parser.add_argument('--renderer', choices=('software', 'nvidia', 'gpu'), default='software')
    parser.add_argument('--gpu-index', type=int, default=0, help='EGL device index; not remapped by CUDA_VISIBLE_DEVICES')
    parser.add_argument('--nvidia-egl-vendor', type=Path, default=Path('/usr/share/glvnd/egl_vendor.d/10_nvidia.json'))
    parser.add_argument('--case-ids', nargs='+', help='Only render these manifest case IDs')
    parser.add_argument('--max-frames', type=int, default=0, help='Short render; requires manifest render_smoke_test=true')
    args = parser.parse_args()
    OUTPUT = args.output_dir.resolve(strict=True)
    manifest = json.loads((OUTPUT / 'manifest.json').read_text())
    if args.max_frames < 0:
        parser.error('--max-frames must be nonnegative')
    if args.max_frames and manifest.get('render_smoke_test') is not True:
        parser.error('--max-frames requires a separate smoke output with manifest render_smoke_test=true')
    selected_ids = set(args.case_ids or [case['id'] for case in manifest['cases']])
    unknown_ids = selected_ids - {case['id'] for case in manifest['cases']}
    if unknown_ids:
        parser.error(f'Unknown case IDs: {sorted(unknown_ids)}')
    configure_renderer(args.renderer, args.gpu_index, args.nvidia_egl_vendor)
    os.nice(10)
    import mujoco
    import numpy as np
    from OpenGL import GL
    from PIL import Image, ImageDraw, ImageFont
    for case in manifest['cases']:
        case['checkpoint_step'] = manifest['checkpoint_step']
        case['inference_protocol_version'] = manifest.get('inference_protocol_version', 'original_cpu_bf16_rtc_v1')
        case['gallery_version'] = manifest.get('gallery_version', f'ckpt_{manifest["checkpoint_step"]}')
        case['root40_contact_protocol'] = manifest.get('root40_contact_protocol', 'legacy_joint_delta_v1')
    model = mujoco.MjModel.from_xml_path(str(XML))
    model.vis.quality.offsamples = 0
    model.vis.quality.shadowsize = 512
    data = mujoco.MjData(model)
    renderer = mujoco.Renderer(model, height=480, width=640)
    backend = GL.glGetString(GL.GL_RENDERER).decode()
    # <modify> Never label software fallback as GPU acceleration.
    # if 'llvmpipe' not in backend.lower():
    #     raise RuntimeError(f'Refusing a non-software renderer during training: {backend}')
    vendor = GL.glGetString(GL.GL_VENDOR).decode()
    is_software = 'llvmpipe' in backend.lower()
    is_nvidia = 'nvidia' in vendor.lower() and not is_software
    if (args.renderer == 'software' and not is_software) or (args.renderer != 'software' and not is_nvidia):
        renderer.close()
        raise RuntimeError(f'Requested {args.renderer}, got GL vendor={vendor!r}, renderer={backend!r}')
    backend_info = dict(renderer_requested=args.renderer, render_backend='software' if is_software else 'nvidia', gl_renderer=backend, gl_vendor=vendor, egl_device_id=None if is_software else args.gpu_index)
    print('Renderer:', backend, flush=True)
    font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 20)
    attempted = set()
    try:
        while True:
            for case in manifest['cases']:
                if case['id'] not in selected_ids:
                    continue
                dest = OUTPUT / case['directory']
                # <modify> Retry interrupted renders unless both the video and its ready marker are present.
                # if (dest / 'prediction.npz').exists() and not (dest / 'comparison.mp4').exists() and case['id'] not in attempted:
                marker = dest / 'render_status.json'
                # <modify> Reject mixed-protocol completed renders instead of quietly reusing them.
                # complete = (dest / 'comparison.mp4').exists() and marker.exists() and json.loads(marker.read_text()).get('status') == 'ready'
                cached = json.loads(marker.read_text()) if marker.exists() else {}
                # <modify> Never reuse a cached render from another v3/v2 protocol.
                # if cached and case['inference_protocol_version'] == 'root40_current_heading_fp32_v2':
                # <modify> Do not reuse a BF16 render for a projection-FP32 job.
                # if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3'):
                if cached and case['inference_protocol_version'] in ('root40_current_heading_fp32_v2', 'root40_current_heading_fp32_video_local_progress_v3', 'root40_current_heading_fp32_video_local_progress_projection_fp32_v4'):
                    assert cached.get('inference_protocol_version') == case['inference_protocol_version']
                    assert cached.get('gallery_version') == case['gallery_version']
                    assert cached.get('root40_contact_protocol', 'legacy_joint_delta_v1') == case['root40_contact_protocol']
                complete = (dest / 'comparison.mp4').exists() and cached.get('status') == 'ready'
                # <modify> Keep completed CPU renders unchanged, including their actual/unknown historical backend.
                if complete and cached.get('render_backend'):
                    cached_renderer = str(cached.get('gl_renderer', '')).lower()
                    cached_vendor = str(cached.get('gl_vendor', '')).lower()
                    if not cached_renderer or (cached['render_backend'] == 'software' and 'llvmpipe' not in cached_renderer) or (cached['render_backend'] == 'nvidia' and ('nvidia' not in cached_vendor or 'llvmpipe' in cached_renderer)):
                        raise RuntimeError(f'Inconsistent cached GL renderer metadata for {case["id"]}')
                if complete and cached.get('smoke_max_frames', 0) != args.max_frames:
                    raise RuntimeError(f'Incompatible smoke render already exists for {case["id"]}')
                if (dest / 'prediction.npz').exists() and not complete and case['id'] not in attempted:
                    attempted.add(case['id'])
                    # render(case, model, data, renderer, font)
                    render(case, model, data, renderer, font, backend_info, args.max_frames)
                    cached = json.loads(marker.read_text())
                if cached.get('status') == 'ready':
                    manifest.setdefault('render_backends', {})[case['id']] = {key: cached.get(key) for key in ('render_backend', 'gl_renderer', 'gl_vendor', 'egl_device_id')}
                    save_json(OUTPUT / 'manifest.json', manifest)
            if not args.watch:
                break
            status_path = OUTPUT / 'inference_status.json'
            if status_path.exists():
                status = json.loads(status_path.read_text())
                proc = Path('/proc') / str(status.get('pid', -1))
                if status['status'] == 'finished' or not proc.exists():
                    break
            time.sleep(5)
    finally:
        renderer.close()


if __name__ == '__main__':
    main()