Download native-eiffel/sftrain/matrix_data.py from peih/matrix-shooter-train: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/peih/matrix-shooter-train/resolve/main/native-eiffel/sftrain/matrix_data.py
- Command line
-
hf download hf://peih/matrix-shooter-train/native-eiffel/sftrain/matrix_data.py
-
curl -L -o matrix_data.py https://huggingface.co/peih/matrix-shooter-train/resolve/main/native-eiffel/sftrain/matrix_data.py
10.3 kB
| """Paired matrix-shooter clips -> teacher fine-tuning windows. | |
| usage: torchrun --nproc_per_node 8 -m sftrain.matrix_data DATA_DIR OUT_DIR [--cache /workspace/sftrain-cache] | |
| DATA_DIR is a datagen/matrix_capture.py run (<map>/clip_*/ with original, | |
| cartoon, war and punk .mp4 and cond/). Each clip gives four 81-frame windows | |
| starting at frames 0, 52, 104 and 156 (21 latents each, the bidirectional | |
| teacher's window). Per window: | |
| OUT_DIR/<map>/clip_NNNNNN/sSSS/ | |
| cond/000000..000020.png graybox frame per latent: the window's first | |
| frame, then first frame + 4L - 3 (lossless where the | |
| capture kept a PNG, else decoded from original.mp4) | |
| cartoon.pt war.pt punk.pt [48, 21, 34, 60] bf16 VAE latents of the styled | |
| frames (one causal encode per window, so latent 0 | |
| is the window's first frame as a still, like a | |
| session's reference) | |
| meta.json | |
| OUT_DIR/prompts.json holds the static prompt per (map, look), and their UMT5 | |
| embeddings go to the text cache. OUT_DIR/split.json holds the held-out clips | |
| (every clip number divisible by 20, 5% per map). | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| from pathlib import Path | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| import torch.distributed as dist | |
| from sftrain import conditions | |
| LOOKS = ('cartoon', 'war', 'punk') | |
| STARTS = (0, 52, 104, 156) | |
| WINDOW = 21 | |
| FRAMES = 1 + 4 * (WINDOW - 1) | |
| MAPS = { | |
| 'paris': 'The scene is a compact Paris Seine district: the Eiffel Tower, Notre-Dame, the Arc de Triomphe, the Louvre courtyard ' | |
| 'and glass pyramid, cream stone Haussmann facades with dark roofs, trees and paved plazas.', | |
| 'tsinghua': 'The scene is the historic Tsinghua University campus: the white Second Gate, the red-brick Grand Auditorium with a ' | |
| 'copper dome, Tsinghua Xuetang, the old library and the observatory, lawns and tree-lined paths.', | |
| } | |
| STYLES = { | |
| 'cartoon': 'Cartoon anime look: bright cream stone and clean colour blocks with fine brick lines, soft sunny daylight and fluffy ' | |
| 'clouds, outlined toy-like foliage. Chibi two-head-tall plush characters with round heads and anime eyes: the player ' | |
| 'a mint-green bunny, enemies berry-coloured bats with red markers. Pink star-shaped bullets with short trails.', | |
| 'war': 'World War II weathered realism: muted grey-brown stone and old red brick with soot, cracks, bullet holes, rust and ' | |
| 'boarded windows, dusty overcast sepia light. Chibi soldiers: the player in a sand wide-brim helmet with a big field ' | |
| 'pack and crossed straps, enemies in iron-grey helmets with goggles and long coats. Thin orange-yellow tracer rounds, ' | |
| 'small muzzle flashes and gun smoke.', | |
| 'punk': 'Cyberpunk neon night: deep blue night sky with stars and rain, wet reflective streets, saturated cyan, magenta and ' | |
| 'warm gold neon signs, holographic panels, lit windows and light strips on every building. Chibi mech characters with ' | |
| 'glowing cyan visors: the player a dark blue-white mech, enemies in heavy armour. Cyan ion bolts with glowing trails.', | |
| } | |
| # matrix-shooter-js-20261030-sakura-v2: new characters in every look and a sakura dusk look (from its pilot frames). | |
| GAME_STYLES = {'sakura': { | |
| 'cartoon': 'Cartoon anime look: bright cream stone and clean colour blocks with fine brick lines, soft sunny daylight and fluffy ' | |
| 'clouds, outlined toy-like foliage. Chibi two-head-tall characters with big anime eyes: the player a pink-haired girl ' | |
| 'in a dark sailor dress with a white collar, enemies brown-haired boys in navy uniforms with red markers. Pink ' | |
| 'star-shaped bullets with short trails.', | |
| 'war': 'World War II weathered realism: muted grey-brown stone and old red brick with soot, cracks and boarded windows, dusty ' | |
| 'overcast sepia light. Chibi soldiers with round heads in sand-coloured uniforms with crossed straps and field packs, ' | |
| 'enemies in iron grey. Thin orange-yellow tracer rounds, small muzzle flashes and gun smoke.', | |
| 'punk': 'Cyberpunk neon night: deep blue night sky with lit skyscrapers and stars, wet reflective streets, saturated cyan, ' | |
| 'magenta and warm gold neon signs, lit windows and light strips on every building. Chibi armoured characters: the ' | |
| 'player with silver hair in a dark blue-white suit with a glowing cyan ring, enemies in dark armour with cyan visors. ' | |
| 'Cyan ion bolts with glowing trails.', | |
| 'sakura': 'Sakura dusk look: flat saturated toon colours, a pastel pink and lavender dusk sky with soft clouds, large pink ' | |
| 'cherry-blossom trees with drifting petals, yellow, teal and orange-red buildings with bright blue windows, a ' | |
| 'red-and-white lattice tower. Chibi robot characters: the player a round glossy blue robot with a white-and-red ' | |
| 'eye, enemies white robots with red trim and cat ears. Glowing bullets with short trails.', | |
| }} | |
| def prompt(map_id, look, styles=STYLES): | |
| return ('A third-person shooter game frame seen from a follow camera behind the player\'s right shoulder. ' | |
| f'{MAPS[map_id]} {styles[look]} Squad gunfights with bullets in flight, muzzle flashes and hit sparks. ' | |
| 'Follow the layout, camera and character positions of the control frames exactly. Every surface keeps this look ' | |
| 'in every frame. No HUD and no on-screen text.') | |
| def read_video(path, tries=3): | |
| """All frames, RGB. The network volume under load occasionally hands back an unreadable file: retry, then raise.""" | |
| for attempt in range(tries): | |
| cap = cv2.VideoCapture(str(path)) | |
| frames = [] | |
| while True: | |
| ok, frame = cap.read() | |
| if not ok: | |
| break | |
| frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) | |
| cap.release() | |
| if frames: | |
| return np.stack(frames) | |
| time.sleep(5 * (attempt + 1)) | |
| raise ValueError(f'no frames read from {path}') | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('data', type=Path) | |
| parser.add_argument('out', type=Path) | |
| parser.add_argument('--cache', type=Path, default=Path('/workspace/sftrain-cache')) | |
| parser.add_argument('--game', choices=('matrix', 'sakura'), default='matrix', help='which package the capture came from') | |
| args = parser.parse_args() | |
| rank, world = int(os.environ.get('RANK', 0)), int(os.environ.get('WORLD_SIZE', 1)) | |
| # Several processes may share a GPU (video decoding is the bottleneck); they only meet at a barrier. | |
| device = torch.device('cuda', int(os.environ.get('LOCAL_RANK', 0)) % torch.cuda.device_count()) | |
| torch.cuda.set_device(device) | |
| if world > 1: | |
| dist.init_process_group('gloo') | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| clips = sorted(c for m in MAPS for c in (args.data / m).glob('clip_*') if (c / 'meta.json').exists()) | |
| # the styled looks this capture rendered (the sakura package adds 'sakura'); its prompts follow --game | |
| looks = tuple(look for look in json.loads((clips[0] / 'meta.json').read_text()).get('looks', ('original',) + LOOKS) | |
| if look != 'original') | |
| styles = {**STYLES, **GAME_STYLES.get(args.game, {})} | |
| if rank == 0: | |
| prompts = {f'{m}/{look}': prompt(m, look, styles) for m in MAPS for look in looks} | |
| for text in list(prompts.values()) + [conditions.NEGATIVE_PROMPT]: | |
| conditions.text_embedding(args.cache, text, device) | |
| (args.out / 'prompts.json').write_text(json.dumps(prompts, indent=1)) | |
| held = sorted(f'{c.parent.name}/{c.name}' for c in clips if int(c.name.split('_')[1]) % 20 == 0) | |
| (args.out / 'split.json').write_text(json.dumps(dict(rule='clip number % 20 == 0', held_out=held), indent=1)) | |
| vae = conditions.vae_encoder(device) | |
| done = 0 | |
| for clip in clips[rank::world]: | |
| target = args.out / clip.parent.name / clip.name | |
| if (target / f's{STARTS[-1]:03d}' / 'meta.json').exists(): | |
| continue | |
| meta = json.loads((clip / 'meta.json').read_text()) | |
| cond_index = {f: i for i, f in enumerate(meta['cond_frames'])} | |
| try: | |
| gray = read_video(clip / 'original.mp4') | |
| styled = {look: read_video(clip / f'{look}.mp4') for look in looks} | |
| except ValueError as e: # skip the clip (a rerun picks it up), keep the batch going | |
| print(json.dumps(dict(rank=rank, skipped=str(clip), error=str(e))), flush=True) | |
| continue | |
| for start in STARTS: | |
| seg = target / f's{start:03d}' | |
| (seg / 'cond').mkdir(parents=True, exist_ok=True) | |
| picks = [start] + [start + 4 * l - 3 for l in range(1, WINDOW)] | |
| for i, f in enumerate(picks): | |
| png = clip / 'cond' / f'{cond_index[f]:06d}.png' if f in cond_index else None | |
| image = cv2.imread(str(png)) if png else cv2.cvtColor(gray[f], cv2.COLOR_RGB2BGR) | |
| cv2.imwrite(str(seg / 'cond' / f'{i:06d}.png'), image) | |
| for look in looks: | |
| pixels = torch.from_numpy(styled[look][start:start + FRAMES]).to(device) | |
| pixels = pixels.permute(0, 3, 1, 2).float().div(127.5).sub(1).unsqueeze(0) # [1, 81, 3, H, W] | |
| with torch.no_grad(): | |
| latent = vae(pixels, autoregressive_index=0, cache=vae.initialize_autoregressive_cache()) | |
| torch.save(latent[0].permute(1, 0, 2, 3).to(torch.bfloat16).cpu().contiguous(), seg / f'{look}.pt') | |
| (seg / 'meta.json').write_text(json.dumps(dict(map=clip.parent.name, clip=clip.name, start=start, frames=picks, | |
| latents=WINDOW, looks=looks, job=meta), indent=1)) | |
| done += 1 | |
| if rank == 0 and done % 5 == 0: | |
| print(json.dumps(dict(rank=rank, done=done, of=len(clips[rank::world]))), flush=True) | |
| if world > 1: | |
| dist.barrier() | |
| dist.destroy_process_group() | |
| if rank == 0: | |
| print('MATRIX_DATA_DONE', flush=True) | |
| if __name__ == '__main__': | |
| main() | |