Terrarioum-AI / scripts /infer /prepare_i2v_inputs.py
papy61's picture
Ajout du moteur AlayaWorld - Terrarioum
bf825fe verified
Raw History Blame Contribute Delete
5.75 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Turn CLI arguments into the three inputs custom_i2v expects:
images/ + captions.json + pose.jsonl (plus an extrinsics .npz).
Extrinsics, pick one:
--extrinsics FILE bring your own camera-to-world poses: .npz (key cam_c2w, [N,4,4])
or .npy/.txt ([N,16] or [N,4,4])
--synth-frames N synthesize a trajectory from --forward / --yaw / --pitch
Intrinsics are optional. Without them a placeholder fx=fy=0.5 is used and the warp falls back
to ViGeo-fitted intrinsics.
--intrinsic fx fy cx cy normalized intrinsics (fx/W, fy/H, cx/W, cy/H)
Example:
python scripts/infer/prepare_i2v_inputs.py --image in.png --prompt "walking forward" \
--synth-frames 256 --forward 0.0049 --yaw 0.15 --out outputs/infer_i2v_camera/inputs
"""
from __future__ import annotations
import argparse
import json
import math
import shutil
from pathlib import Path
import numpy as np
def synth_trajectory(n: int, forward: float, yaw_deg: float, pitch_deg: float) -> np.ndarray:
"""Build c2w [n,4,4]: each frame advances `forward` along its own -z axis and accumulates yaw/pitch."""
poses = np.zeros((n, 4, 4), dtype=np.float32)
R = np.eye(3, dtype=np.float32)
t = np.zeros(3, dtype=np.float32)
for i in range(n):
poses[i, :3, :3] = R
poses[i, :3, 3] = t
poses[i, 3, 3] = 1.0
ya, pa = math.radians(yaw_deg), math.radians(pitch_deg)
Ry = np.array([[math.cos(ya), 0, math.sin(ya)],
[0, 1, 0],
[-math.sin(ya), 0, math.cos(ya)]], dtype=np.float32)
Rx = np.array([[1, 0, 0], [0, math.cos(pa), -math.sin(pa)], [0, math.sin(pa), math.cos(pa)]], dtype=np.float32)
R = R @ Ry @ Rx
t = t + R @ np.array([0, 0, forward], dtype=np.float32) # +z is forward (matches the training pose convention)
return poses
def load_extrinsics(path: Path) -> np.ndarray:
if path.suffix == ".npz":
z = np.load(path)
key = "cam_c2w" if "cam_c2w" in z.files else z.files[0]
arr = np.asarray(z[key], dtype=np.float32)
elif path.suffix == ".npy":
arr = np.asarray(np.load(path), dtype=np.float32)
else:
arr = np.loadtxt(path, dtype=np.float32)
if arr.ndim == 2 and arr.shape[1] == 16:
arr = arr.reshape(-1, 4, 4)
if arr.ndim != 3 or arr.shape[1:] != (4, 4):
raise ValueError(f"extrinsics must be [N,4,4] or [N,16], got {arr.shape}")
return arr
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--image", required=True, help="single first frame image")
ap.add_argument("--prompt", required=True, help="text prompt")
ap.add_argument("--out", default="outputs/infer_i2v_camera/inputs")
ap.add_argument("--extrinsics", default=None, help="file holding your own camera-to-world trajectory")
ap.add_argument("--synth-frames", type=int, default=None,
help="length of the synthesized trajectory; derived from --rounds when omitted. "
"The trajectory is tiled cyclically, so a short one jumps back to its start.")
ap.add_argument("--rounds", type=int, default=5,
help="only used to size the trajectory; the actual round count is written to the config by generate_video.sh")
ap.add_argument("--forward", type=float, default=0.0049, help="per-frame translation (same scale as training poses)")
ap.add_argument("--yaw", type=float, default=0.0, help="per-frame yaw in degrees")
ap.add_argument("--pitch", type=float, default=0.0, help="per-frame pitch in degrees")
ap.add_argument("--intrinsic", type=float, nargs=4, default=None,
metavar=("FX", "FY", "CX", "CY"), help="normalized intrinsics; omit to use the placeholder")
args = ap.parse_args()
out = Path(args.out)
(out / "images").mkdir(parents=True, exist_ok=True)
img = Path(args.image)
shutil.copy(img, out / "images" / img.name)
if args.extrinsics:
cam = load_extrinsics(Path(args.extrinsics))
how = f"provided by {args.extrinsics}"
else:
# 4 latents per round x stride 8 = 32 pixel frames; 3x margin, at least 512
n = args.synth_frames or max(512, args.rounds * 32 * 3)
cam = synth_trajectory(n, args.forward, args.yaw, args.pitch)
how = f"synthesized {n} frames (forward={args.forward}/frame, yaw={args.yaw} deg, pitch={args.pitch} deg)"
npz = out / "cam_c2w.npz"
save = {"cam_c2w": cam.astype(np.float32)}
entry = {"pose_path": str(npz.resolve()), "assigned_image": img.name, "note": how}
if args.intrinsic:
fx, fy, cx, cy = args.intrinsic
save["intrinsic"] = np.array([[fx, 0.0, cx], [0.0, fy, cy], [0.0, 0.0, 1.0]], dtype=np.float32)
entry["intrinsic"] = [fx, fy, cx, cy]
np.savez(npz, **save)
(out / "captions.json").write_text(json.dumps({img.name: args.prompt}, ensure_ascii=False, indent=2),
encoding="utf-8")
with open(out / "pose.jsonl", "w", encoding="utf-8") as f:
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
step = float(np.linalg.norm(np.diff(cam[:, :3, 3], axis=0), axis=1).mean()) if len(cam) > 1 else 0.0
print(f"[prepare] image : {out / 'images' / img.name}")
print(f"[prepare] prompt : {args.prompt[:70]}{'...' if len(args.prompt) > 70 else ''}")
print(f"[prepare] extrinsics: {cam.shape} ({how}), mean inter-frame translation {step:.4f}")
print(f"[prepare] intrinsic : {'real ' + str(args.intrinsic) if args.intrinsic else 'placeholder fx=fy=0.5 (ViGeo-fitted fallback)'}")
print(f"[prepare] inputs written to {out}")
if __name__ == "__main__":
main()