Download native-eiffel/sftrain/sample.py from peih/matrix-shooter-train: direct link, hf CLI and curl.
- Browser
- Download file 5.97 kB
-
https://huggingface.co/peih/matrix-shooter-train/resolve/main/native-eiffel/sftrain/sample.py
- Command line
-
hf download hf://peih/matrix-shooter-train/native-eiffel/sftrain/sample.py
-
curl -L -o sample.py https://huggingface.co/peih/matrix-shooter-train/resolve/main/native-eiffel/sftrain/sample.py
5.97 kB
| """Roll a clip out with the student (or sample its window with the teacher) and write proxy | generated. | |
| usage: python -m sftrain.sample CLIP_DIR OUT.mp4 [--chunk 3 --steps 4] [--lora ckpt/student_lora.safetensors] | |
| [--latents 60] [--teacher --teacher-steps 40 --cfg 3] [--seed 0] | |
| Student: Eden's AR sampler (sink 6 + window, cache-slot RoPE) at the given chunk | |
| size and step count. With --chunk 3 --steps 4 and no LoRA this is the published | |
| dit-5b in its native setting, the check that this forward matches Eden. | |
| Teacher: dit-5b-bidir over the whole window, Euler on the shift-5 flow, frame 0 | |
| stamped with the reference, CFG against the negative prompt (EdenLive's bidir | |
| profile samples it at guidance 3; --cfg 1 turns it off). | |
| --model overrides the weights directory; a graybox-latent teacher (or a student with | |
| --student-patch from a gray distillation) also gets the clip's gray.pt as control. | |
| """ | |
| import argparse | |
| import json | |
| import subprocess | |
| from pathlib import Path | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| from sftrain import conditions, dit | |
| from sftrain.prepare import text_key | |
| from safetensors.torch import load_file | |
| from sftrain.train import load_lora, add_lora, sampling_sigmas | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('clip', type=Path) | |
| parser.add_argument('out', type=Path) | |
| parser.add_argument('--chunk', type=int, default=1) | |
| parser.add_argument('--steps', type=int, default=1) | |
| parser.add_argument('--lora', type=Path) | |
| parser.add_argument('--lora-rank', type=int, default=128) | |
| parser.add_argument('--latents', type=int, default=60) | |
| parser.add_argument('--teacher', action='store_true') | |
| parser.add_argument('--teacher-steps', type=int, default=40) | |
| parser.add_argument('--cfg', type=float, default=conditions.TEACHER_CFG) | |
| parser.add_argument('--cache', type=Path, default=Path('/workspace/sftrain-cache')) | |
| parser.add_argument('--model-root', type=Path, default=conditions.MODEL_ROOT) | |
| parser.add_argument('--model', type=Path, help='weights directory instead of MODEL_ROOT/dit-5b[-bidir]') | |
| parser.add_argument('--student-patch', type=Path, help='student_patch.safetensors of a gray distillation') | |
| parser.add_argument('--seed', type=int, default=0) | |
| args = parser.parse_args() | |
| device = torch.device('cuda') | |
| torch.manual_seed(args.seed) | |
| meta = json.loads((args.clip / 'meta.json').read_text()) | |
| style = json.loads((args.clip / 'style.json').read_text()) | |
| T = min(args.latents, meta['latents']) | |
| proxy = np.stack([cv2.cvtColor(cv2.imread(str(args.clip / 'cond' / f'{i:06d}.png')), cv2.COLOR_BGR2RGB) | |
| for i in range(T)]) | |
| ref = torch.load(args.clip / 'ref_latent.pt').to(device).unsqueeze(0) | |
| text = torch.load(args.cache / 'text' / f'{text_key(style["static_prompt"])}.pt').to(device).unsqueeze(0) | |
| dino = conditions.control_encoder(device) | |
| with torch.no_grad(): | |
| control = conditions.encode_control(dino, torch.from_numpy(proxy)) | |
| del dino | |
| model = dit.load_dit(str(args.model or args.model_root / ('dit-5b-bidir' if args.teacher else 'dit-5b')), device) | |
| if args.lora: | |
| add_lora(model, args.lora_rank) | |
| load_lora(model, args.lora) | |
| if args.student_patch: | |
| model.patch_embedding = dit.TrainablePatchEmbedding(model.patch_embedding, dit.GRAY_CHANNELS) | |
| model.patch_embedding.load(load_file(str(args.student_patch))) | |
| if dit.takes_gray(model): | |
| control = torch.cat([control, torch.load(args.clip / 'gray.pt')[:, :T].to(device, control.dtype).unsqueeze(0)], 1) | |
| rope = dit.SlotRope(model, conditions.LATENT_H // 2, conditions.LATENT_W // 2, max(dit.ARENA, T) + 1, device) | |
| with torch.no_grad(): | |
| ctx = dit.embed_text(model, text) | |
| if args.teacher: | |
| sigmas = sampling_sigmas(args.teacher_steps, 5.0) + [0.0] | |
| x = torch.randn(1, dit.LATENT_CHANNELS, T, conditions.LATENT_H, conditions.LATENT_W, device=device) | |
| if args.cfg != 1: | |
| neg = dit.embed_text(model, conditions.text_embedding(args.cache, conditions.NEGATIVE_PROMPT, device) | |
| .to(device).unsqueeze(0)) | |
| for s, s_next in zip(sigmas[:-1], sigmas[1:]): | |
| x[:, :, :1] = ref | |
| t = torch.full((1, T), s * 1000, device=device) | |
| t[:, 0] = 0 | |
| v, _ = dit.forward(model, x, control, t, ctx, rope) | |
| if args.cfg != 1: | |
| v_neg, _ = dit.forward(model, x, control, t, neg, rope) | |
| v = v_neg + args.cfg * (v - v_neg) | |
| x = x + (s_next - s) * v | |
| x[:, :, :1] = ref | |
| latents = x | |
| else: | |
| latents = dit.rollout(model, rope, ref, control, ctx, sampling_sigmas(args.steps, 5.0), args.chunk) | |
| del model | |
| torch.cuda.empty_cache() | |
| video = conditions.decode(conditions.vae_decoder(device), latents) | |
| frames = len(video) | |
| # One proxy frame per latent: frame 0 once, then each latent's frame for its 4 decoded frames. | |
| index = [0] + [1 + (i - 1) // 4 for i in range(1, frames)] | |
| side = np.concatenate([proxy[np.minimum(index, T - 1)], video], axis=2) | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| writer = subprocess.Popen(['ffmpeg', '-y', '-hide_banner', '-loglevel', 'error', '-f', 'rawvideo', '-pix_fmt', 'rgb24', | |
| '-s', f'{side.shape[2]}x{side.shape[1]}', '-r', '24', '-i', '-', '-an', '-c:v', 'libx264', | |
| '-crf', '20', '-pix_fmt', 'yuv420p', str(args.out)], stdin=subprocess.PIPE) | |
| writer.stdin.write(side.tobytes()) | |
| writer.stdin.close() | |
| writer.wait() | |
| print(json.dumps(dict(out=str(args.out), latents=T, frames=frames, look=style['look'], | |
| mode=f'teacher-cfg{args.cfg:g}' if args.teacher else f'chunk{args.chunk}-steps{args.steps}'))) | |
| if __name__ == '__main__': | |
| main() | |