"""Paired matrix-shooter clips -> teacher fine-tuning windows. usage: torchrun --nproc_per_node 8 -m sftrain.matrix_data DATA_DIR OUT_DIR [--cache /workspace/sftrain-cache] DATA_DIR is a datagen/matrix_capture.py run (/clip_*/ with original, cartoon, war and punk .mp4 and cond/). Each clip gives four 81-frame windows starting at frames 0, 52, 104 and 156 (21 latents each, the bidirectional teacher's window). Per window: OUT_DIR//clip_NNNNNN/sSSS/ cond/000000..000020.png graybox frame per latent: the window's first frame, then first frame + 4L - 3 (lossless where the capture kept a PNG, else decoded from original.mp4) cartoon.pt war.pt punk.pt [48, 21, 34, 60] bf16 VAE latents of the styled frames (one causal encode per window, so latent 0 is the window's first frame as a still, like a session's reference) meta.json OUT_DIR/prompts.json holds the static prompt per (map, look), and their UMT5 embeddings go to the text cache. OUT_DIR/split.json holds the held-out clips (every clip number divisible by 20, 5% per map). """ import argparse import json import os import time from pathlib import Path import cv2 import numpy as np import torch import torch.distributed as dist from sftrain import conditions LOOKS = ('cartoon', 'war', 'punk') STARTS = (0, 52, 104, 156) WINDOW = 21 FRAMES = 1 + 4 * (WINDOW - 1) MAPS = { 'paris': 'The scene is a compact Paris Seine district: the Eiffel Tower, Notre-Dame, the Arc de Triomphe, the Louvre courtyard ' 'and glass pyramid, cream stone Haussmann facades with dark roofs, trees and paved plazas.', 'tsinghua': 'The scene is the historic Tsinghua University campus: the white Second Gate, the red-brick Grand Auditorium with a ' 'copper dome, Tsinghua Xuetang, the old library and the observatory, lawns and tree-lined paths.', } STYLES = { 'cartoon': 'Cartoon anime look: bright cream stone and clean colour blocks with fine brick lines, soft sunny daylight and fluffy ' 'clouds, outlined toy-like foliage. Chibi two-head-tall plush characters with round heads and anime eyes: the player ' 'a mint-green bunny, enemies berry-coloured bats with red markers. Pink star-shaped bullets with short trails.', 'war': 'World War II weathered realism: muted grey-brown stone and old red brick with soot, cracks, bullet holes, rust and ' 'boarded windows, dusty overcast sepia light. Chibi soldiers: the player in a sand wide-brim helmet with a big field ' 'pack and crossed straps, enemies in iron-grey helmets with goggles and long coats. Thin orange-yellow tracer rounds, ' 'small muzzle flashes and gun smoke.', 'punk': 'Cyberpunk neon night: deep blue night sky with stars and rain, wet reflective streets, saturated cyan, magenta and ' 'warm gold neon signs, holographic panels, lit windows and light strips on every building. Chibi mech characters with ' 'glowing cyan visors: the player a dark blue-white mech, enemies in heavy armour. Cyan ion bolts with glowing trails.', } # matrix-shooter-js-20261030-sakura-v2: new characters in every look and a sakura dusk look (from its pilot frames). GAME_STYLES = {'sakura': { 'cartoon': 'Cartoon anime look: bright cream stone and clean colour blocks with fine brick lines, soft sunny daylight and fluffy ' 'clouds, outlined toy-like foliage. Chibi two-head-tall characters with big anime eyes: the player a pink-haired girl ' 'in a dark sailor dress with a white collar, enemies brown-haired boys in navy uniforms with red markers. Pink ' 'star-shaped bullets with short trails.', 'war': 'World War II weathered realism: muted grey-brown stone and old red brick with soot, cracks and boarded windows, dusty ' 'overcast sepia light. Chibi soldiers with round heads in sand-coloured uniforms with crossed straps and field packs, ' 'enemies in iron grey. Thin orange-yellow tracer rounds, small muzzle flashes and gun smoke.', 'punk': 'Cyberpunk neon night: deep blue night sky with lit skyscrapers and stars, wet reflective streets, saturated cyan, ' 'magenta and warm gold neon signs, lit windows and light strips on every building. Chibi armoured characters: the ' 'player with silver hair in a dark blue-white suit with a glowing cyan ring, enemies in dark armour with cyan visors. ' 'Cyan ion bolts with glowing trails.', 'sakura': 'Sakura dusk look: flat saturated toon colours, a pastel pink and lavender dusk sky with soft clouds, large pink ' 'cherry-blossom trees with drifting petals, yellow, teal and orange-red buildings with bright blue windows, a ' 'red-and-white lattice tower. Chibi robot characters: the player a round glossy blue robot with a white-and-red ' 'eye, enemies white robots with red trim and cat ears. Glowing bullets with short trails.', }} def prompt(map_id, look, styles=STYLES): return ('A third-person shooter game frame seen from a follow camera behind the player\'s right shoulder. ' f'{MAPS[map_id]} {styles[look]} Squad gunfights with bullets in flight, muzzle flashes and hit sparks. ' 'Follow the layout, camera and character positions of the control frames exactly. Every surface keeps this look ' 'in every frame. No HUD and no on-screen text.') def read_video(path, tries=3): """All frames, RGB. The network volume under load occasionally hands back an unreadable file: retry, then raise.""" for attempt in range(tries): cap = cv2.VideoCapture(str(path)) frames = [] while True: ok, frame = cap.read() if not ok: break frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) cap.release() if frames: return np.stack(frames) time.sleep(5 * (attempt + 1)) raise ValueError(f'no frames read from {path}') def main(): parser = argparse.ArgumentParser() parser.add_argument('data', type=Path) parser.add_argument('out', type=Path) parser.add_argument('--cache', type=Path, default=Path('/workspace/sftrain-cache')) parser.add_argument('--game', choices=('matrix', 'sakura'), default='matrix', help='which package the capture came from') args = parser.parse_args() rank, world = int(os.environ.get('RANK', 0)), int(os.environ.get('WORLD_SIZE', 1)) # Several processes may share a GPU (video decoding is the bottleneck); they only meet at a barrier. device = torch.device('cuda', int(os.environ.get('LOCAL_RANK', 0)) % torch.cuda.device_count()) torch.cuda.set_device(device) if world > 1: dist.init_process_group('gloo') args.out.mkdir(parents=True, exist_ok=True) clips = sorted(c for m in MAPS for c in (args.data / m).glob('clip_*') if (c / 'meta.json').exists()) # the styled looks this capture rendered (the sakura package adds 'sakura'); its prompts follow --game looks = tuple(look for look in json.loads((clips[0] / 'meta.json').read_text()).get('looks', ('original',) + LOOKS) if look != 'original') styles = {**STYLES, **GAME_STYLES.get(args.game, {})} if rank == 0: prompts = {f'{m}/{look}': prompt(m, look, styles) for m in MAPS for look in looks} for text in list(prompts.values()) + [conditions.NEGATIVE_PROMPT]: conditions.text_embedding(args.cache, text, device) (args.out / 'prompts.json').write_text(json.dumps(prompts, indent=1)) held = sorted(f'{c.parent.name}/{c.name}' for c in clips if int(c.name.split('_')[1]) % 20 == 0) (args.out / 'split.json').write_text(json.dumps(dict(rule='clip number % 20 == 0', held_out=held), indent=1)) vae = conditions.vae_encoder(device) done = 0 for clip in clips[rank::world]: target = args.out / clip.parent.name / clip.name if (target / f's{STARTS[-1]:03d}' / 'meta.json').exists(): continue meta = json.loads((clip / 'meta.json').read_text()) cond_index = {f: i for i, f in enumerate(meta['cond_frames'])} try: gray = read_video(clip / 'original.mp4') styled = {look: read_video(clip / f'{look}.mp4') for look in looks} except ValueError as e: # skip the clip (a rerun picks it up), keep the batch going print(json.dumps(dict(rank=rank, skipped=str(clip), error=str(e))), flush=True) continue for start in STARTS: seg = target / f's{start:03d}' (seg / 'cond').mkdir(parents=True, exist_ok=True) picks = [start] + [start + 4 * l - 3 for l in range(1, WINDOW)] for i, f in enumerate(picks): png = clip / 'cond' / f'{cond_index[f]:06d}.png' if f in cond_index else None image = cv2.imread(str(png)) if png else cv2.cvtColor(gray[f], cv2.COLOR_RGB2BGR) cv2.imwrite(str(seg / 'cond' / f'{i:06d}.png'), image) for look in looks: pixels = torch.from_numpy(styled[look][start:start + FRAMES]).to(device) pixels = pixels.permute(0, 3, 1, 2).float().div(127.5).sub(1).unsqueeze(0) # [1, 81, 3, H, W] with torch.no_grad(): latent = vae(pixels, autoregressive_index=0, cache=vae.initialize_autoregressive_cache()) torch.save(latent[0].permute(1, 0, 2, 3).to(torch.bfloat16).cpu().contiguous(), seg / f'{look}.pt') (seg / 'meta.json').write_text(json.dumps(dict(map=clip.parent.name, clip=clip.name, start=start, frames=picks, latents=WINDOW, looks=looks, job=meta), indent=1)) done += 1 if rank == 0 and done % 5 == 0: print(json.dumps(dict(rank=rank, done=done, of=len(clips[rank::world]))), flush=True) if world > 1: dist.barrier() dist.destroy_process_group() if rank == 0: print('MATRIX_DATA_DONE', flush=True) if __name__ == '__main__': main()