peih's picture
backup native-eiffel
5dd27d0 verified
Raw History Blame Contribute Delete
5.97 kB
"""Roll a clip out with the student (or sample its window with the teacher) and write proxy | generated.
usage: python -m sftrain.sample CLIP_DIR OUT.mp4 [--chunk 3 --steps 4] [--lora ckpt/student_lora.safetensors]
[--latents 60] [--teacher --teacher-steps 40 --cfg 3] [--seed 0]
Student: Eden's AR sampler (sink 6 + window, cache-slot RoPE) at the given chunk
size and step count. With --chunk 3 --steps 4 and no LoRA this is the published
dit-5b in its native setting, the check that this forward matches Eden.
Teacher: dit-5b-bidir over the whole window, Euler on the shift-5 flow, frame 0
stamped with the reference, CFG against the negative prompt (EdenLive's bidir
profile samples it at guidance 3; --cfg 1 turns it off).
--model overrides the weights directory; a graybox-latent teacher (or a student with
--student-patch from a gray distillation) also gets the clip's gray.pt as control.
"""
import argparse
import json
import subprocess
from pathlib import Path
import cv2
import numpy as np
import torch
from sftrain import conditions, dit
from sftrain.prepare import text_key
from safetensors.torch import load_file
from sftrain.train import load_lora, add_lora, sampling_sigmas
def main():
parser = argparse.ArgumentParser()
parser.add_argument('clip', type=Path)
parser.add_argument('out', type=Path)
parser.add_argument('--chunk', type=int, default=1)
parser.add_argument('--steps', type=int, default=1)
parser.add_argument('--lora', type=Path)
parser.add_argument('--lora-rank', type=int, default=128)
parser.add_argument('--latents', type=int, default=60)
parser.add_argument('--teacher', action='store_true')
parser.add_argument('--teacher-steps', type=int, default=40)
parser.add_argument('--cfg', type=float, default=conditions.TEACHER_CFG)
parser.add_argument('--cache', type=Path, default=Path('/workspace/sftrain-cache'))
parser.add_argument('--model-root', type=Path, default=conditions.MODEL_ROOT)
parser.add_argument('--model', type=Path, help='weights directory instead of MODEL_ROOT/dit-5b[-bidir]')
parser.add_argument('--student-patch', type=Path, help='student_patch.safetensors of a gray distillation')
parser.add_argument('--seed', type=int, default=0)
args = parser.parse_args()
device = torch.device('cuda')
torch.manual_seed(args.seed)
meta = json.loads((args.clip / 'meta.json').read_text())
style = json.loads((args.clip / 'style.json').read_text())
T = min(args.latents, meta['latents'])
proxy = np.stack([cv2.cvtColor(cv2.imread(str(args.clip / 'cond' / f'{i:06d}.png')), cv2.COLOR_BGR2RGB)
for i in range(T)])
ref = torch.load(args.clip / 'ref_latent.pt').to(device).unsqueeze(0)
text = torch.load(args.cache / 'text' / f'{text_key(style["static_prompt"])}.pt').to(device).unsqueeze(0)
dino = conditions.control_encoder(device)
with torch.no_grad():
control = conditions.encode_control(dino, torch.from_numpy(proxy))
del dino
model = dit.load_dit(str(args.model or args.model_root / ('dit-5b-bidir' if args.teacher else 'dit-5b')), device)
if args.lora:
add_lora(model, args.lora_rank)
load_lora(model, args.lora)
if args.student_patch:
model.patch_embedding = dit.TrainablePatchEmbedding(model.patch_embedding, dit.GRAY_CHANNELS)
model.patch_embedding.load(load_file(str(args.student_patch)))
if dit.takes_gray(model):
control = torch.cat([control, torch.load(args.clip / 'gray.pt')[:, :T].to(device, control.dtype).unsqueeze(0)], 1)
rope = dit.SlotRope(model, conditions.LATENT_H // 2, conditions.LATENT_W // 2, max(dit.ARENA, T) + 1, device)
with torch.no_grad():
ctx = dit.embed_text(model, text)
if args.teacher:
sigmas = sampling_sigmas(args.teacher_steps, 5.0) + [0.0]
x = torch.randn(1, dit.LATENT_CHANNELS, T, conditions.LATENT_H, conditions.LATENT_W, device=device)
if args.cfg != 1:
neg = dit.embed_text(model, conditions.text_embedding(args.cache, conditions.NEGATIVE_PROMPT, device)
.to(device).unsqueeze(0))
for s, s_next in zip(sigmas[:-1], sigmas[1:]):
x[:, :, :1] = ref
t = torch.full((1, T), s * 1000, device=device)
t[:, 0] = 0
v, _ = dit.forward(model, x, control, t, ctx, rope)
if args.cfg != 1:
v_neg, _ = dit.forward(model, x, control, t, neg, rope)
v = v_neg + args.cfg * (v - v_neg)
x = x + (s_next - s) * v
x[:, :, :1] = ref
latents = x
else:
latents = dit.rollout(model, rope, ref, control, ctx, sampling_sigmas(args.steps, 5.0), args.chunk)
del model
torch.cuda.empty_cache()
video = conditions.decode(conditions.vae_decoder(device), latents)
frames = len(video)
# One proxy frame per latent: frame 0 once, then each latent's frame for its 4 decoded frames.
index = [0] + [1 + (i - 1) // 4 for i in range(1, frames)]
side = np.concatenate([proxy[np.minimum(index, T - 1)], video], axis=2)
args.out.parent.mkdir(parents=True, exist_ok=True)
writer = subprocess.Popen(['ffmpeg', '-y', '-hide_banner', '-loglevel', 'error', '-f', 'rawvideo', '-pix_fmt', 'rgb24',
'-s', f'{side.shape[2]}x{side.shape[1]}', '-r', '24', '-i', '-', '-an', '-c:v', 'libx264',
'-crf', '20', '-pix_fmt', 'yuv420p', str(args.out)], stdin=subprocess.PIPE)
writer.stdin.write(side.tobytes())
writer.stdin.close()
writer.wait()
print(json.dumps(dict(out=str(args.out), latents=T, frames=frames, look=style['look'],
mode=f'teacher-cfg{args.cfg:g}' if args.teacher else f'chunk{args.chunk}-steps{args.steps}')))
if __name__ == '__main__':
main()