maliang / scripts /teaser /edit_app_f0.py
teawhite's picture
scripts/teaser: teaser / edit / bossswap / infinite / cover code
68e2686 verified
Raw History Blame Contribute Delete
10.7 kB
#!/usr/bin/env python3
"""teaser · 编辑 case「外观编辑」的首帧编辑器(img env:$M/envs/img/bin/python,diffusers main,GPU)。
语义不动、只改首帧:Qwen-Image-2.1-Turbo(f0gen.Qwen21,8 步)对底子首帧(832×480)做图像编辑,输出同尺寸 png + json。
三种编辑(jobs 行的 kind)
--------------------------
* global :整图编辑(时间 / 天气 / 季节:night / dusk / snow / autumn / spring / lava …)。参考图 = 原首帧放大到 1664×960,
提示词 = job.prompt + KEEP(布局、物体、角色位置姿势不变,不增删)。不钉回(夜晚角色也要变暗)。
可选 job.best_of = N:N 个种子里留结构边缘 F1(对原图)最高的一张。
* char :角色局部编辑(主角换盔甲 / 换英雄、Boss 换造型)。target = player | boss:该角色帧 0 掩码(ids[0] 的部件 id)的外接框
(f0lib.mask_box,外扩 0.6、至少 160 px)裁出 → 放到 ≈1 MP → Qwen21 编辑 → 缩回、框内「离角色 ≥ 8 px 的背景」逐通道
均值 / 方差匹配回原图(去偏色)→ 只在掩码膨胀 r = max(10, 0.35·角色框短边) px、羽化 3 px 内贴回,**掩码外逐像素钉回原图**;
另一个角色的掩码(膨胀 2 px)也钉回原图(Boss 修补不会盖掉贴身的主角)。
* ground :地面换材质。整图编辑「只把 <类描述> 换成 <新材质>」,然后只在 ids[0] ∈ classes 的掩码(腐蚀 1 px、羽化 2 px)内用编辑结果,
掩码外逐像素钉回原图。
每张写 <out>.png + <out>.json(job、提示词、种子、秒数、显存、struct_metrics(原图, 编辑)、掩码像素数、贴回参数);
char / ground 另存 <out>_raw.png(贴回前的编辑原样,查问题用)。已有 png + json ⇒ 跳过(可续跑,--redo 重做)。
jobs.jsonl 行(edit_app.py f0jobs 生成):{"out", "src"(原首帧 png), "ids"(npz,取帧 0), "kind", "prompt", "seed",
["target"], ["classes"], ...任意标签}。
usage(env:HF_HOME=/opt/dlami/nvme/zhiyangdeng/.cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=$M/.tmp):
CUDA_VISIBLE_DEVICES=6 $M/envs/img/bin/python scripts/teaser/edit_app_f0.py --jobs outputs/teaser/edit/app/f0/jobs.jsonl \\
[--shard 0 --nshards 1] [--only <glob>] [--redo]
"""
from __future__ import annotations
import argparse
import fnmatch
import json
import os
import sys
import time
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
os.environ.setdefault("HF_HOME", "/opt/dlami/nvme/zhiyangdeng/.cache/huggingface")
import numpy as np # noqa: E402
from PIL import Image # noqa: E402
M = "/opt/dlami/nvme/zhiyangdeng/maliang"
sys.path[:0] = [f"{M}/scripts/teaser", f"{M}/code/gc25d", f"{M}/scripts/genbench3"]
import f0lib as L # noqa: E402
import firstframe_flux as FF # noqa: E402
GEN = (1664, 960)
KEEP = (" Keep exactly the same composition, camera angle, layout and every building, wall, rock, plant and object, with the "
"same positions, outlines and sizes; keep both characters exactly where they are with the same poses, sizes, "
"facing directions and weapons. Do not add or remove anything; no text, no UI, no border.")
CHAR_KEEP = (" Keep exactly the same position, pose, size, facing direction and limb directions, and keep the weapon in the "
"same hand. Keep everything else in the image exactly unchanged; do not add any other characters or objects.")
def rp(p: str) -> str:
return p if os.path.isabs(p) else os.path.join(M, p)
def load_ids0(path: str) -> np.ndarray:
with np.load(rp(path)) as z:
a = z["ids"] if "ids" in z else z[list(z.keys())[0]]
return np.ascontiguousarray(a[0] if a.ndim == 3 else a)
def up(img: np.ndarray, size=GEN) -> np.ndarray:
import cv2
return cv2.resize(img, size, interpolation=cv2.INTER_LANCZOS4)
def color_match(out: np.ndarray, ref: np.ndarray, bg: np.ndarray) -> np.ndarray:
"""out 的 bg 区域逐通道均值 / 方差对齐到 ref 的同一区域(f0gen.charfix 的去偏色)。"""
if bg.sum() < 200:
return out
o, r = out.astype(np.float32), ref.astype(np.float32)
mu_o, sd_o = o[bg].mean(0), o[bg].std(0) + 1e-3
mu_r, sd_r = r[bg].mean(0), r[bg].std(0) + 1e-3
return np.clip((o - mu_o) * np.clip(sd_r / sd_o, 0.7, 1.4) + mu_r, 0, 255)
def edit_global(q21, img: np.ndarray, j: dict):
"""job.best_of = N(默认 1):种子 seed, seed+100, … 各编辑一次,留 struct_metrics(原图, 编辑).edge_f1 最高的
(下雪等编辑偶尔会凭空加墙 / 篱笆 —— 结构边缘对不上原图 ⇒ F1 低)。"""
p = j["prompt"].strip() + KEEP
best, cands = None, []
for k in range(int(j.get("best_of", 1))):
sd = int(j["seed"]) + 100 * k
out = L.area_down(q21(p, [up(img)], GEN, sd))
f1 = float(FF.struct_metrics(img, out)["edge_f1"])
cands.append({"seed": sd, "edge_f1": round(f1, 4)})
if best is None or f1 > best[1]:
best = (out, f1, sd)
info = {"prompt": p}
if len(cands) > 1:
info.update(best_of=len(cands), candidates=cands, picked_seed=best[2])
return best[0], info
def edit_char(q21, img: np.ndarray, ids0: np.ndarray, j: dict):
import cv2
who = j["target"]
mk = np.isin(ids0, L.PLAYER if who == "player" else L.BOSS)
if mk.sum() < L.MIN_PX:
raise ValueError(f"帧 0 里看不到 {who}({int(mk.sum())} px)")
x0, y0, x1, y1 = L.mask_box(mk, margin=0.6, min_side=160)
crop = img[y0:y1, x0:x1]
cw, ch = L.W, L.H
import math
s = math.sqrt(1024 * 1024 / ((x1 - x0) * (y1 - y0)))
cw, ch = max(32, round((x1 - x0) * s / 32) * 32), max(32, round((y1 - y0) * s / 32) * 32)
p = j["prompt"].strip() + CHAR_KEEP
out = q21(p, [cv2.resize(crop, (cw, ch), interpolation=cv2.INTER_LANCZOS4)], (cw, ch), int(j["seed"]))
out = cv2.resize(out, (x1 - x0, y1 - y0), interpolation=cv2.INTER_AREA)
other = np.isin(ids0, L.PLAYER + L.BOSS)[y0:y1, x0:x1]
bg = cv2.dilate(other.astype(np.uint8), np.ones((17, 17), np.uint8)) == 0
out = color_match(out, crop, bg)
ys, xs = np.nonzero(mk)
r = max(10, round(float(j.get("dilate_frac", 0.35)) * min(xs.max() - xs.min() + 1, ys.max() - ys.min() + 1)))
a = L.soft_mask(mk, r, 3.0)
other_mk = np.isin(ids0, L.BOSS if who == "player" else L.PLAYER) # 另一个角色钉住(膨胀 2 px),不被贴回区盖掉
if other_mk.sum():
a = a * (1.0 - L.soft_mask(other_mk & ~mk, 2, 1.0))
a = a[y0:y1, x0:x1, None]
raw = img.copy()
raw[y0:y1, x0:x1] = out.astype(np.uint8)
res = img.copy()
res[y0:y1, x0:x1] = np.clip(a * out + (1 - a) * crop, 0, 255).astype(np.uint8)
return res, {"prompt": p, "box": [int(x0), int(y0), int(x1), int(y1)], "edit_size": [cw, ch], "dilate_px": int(r),
"mask_px": int(mk.sum()), "raw": raw}
def edit_ground(q21, img: np.ndarray, ids0: np.ndarray, j: dict):
import cv2
cls = tuple(int(c) for c in j["classes"])
mk = np.isin(ids0, cls)
if mk.sum() < 2000:
raise ValueError(f"帧 0 里类 {cls} 太少({int(mk.sum())} px)")
p = j["prompt"].strip() + (" Only the ground surface changes; every building, wall, rock, plant, object and both "
"characters stay exactly the same." + KEEP)
out = L.area_down(q21(p, [up(img)], GEN, int(j["seed"])))
er = cv2.erode(mk.astype(np.uint8), np.ones((3, 3), np.uint8)).astype(np.float32)
a = np.maximum(cv2.GaussianBlur(er, (0, 0), 2.0), er)[..., None]
res = np.clip(a * out + (1 - a) * img, 0, 255).astype(np.uint8)
return res, {"prompt": p, "classes": list(cls), "mask_px": int(mk.sum()), "raw": out}
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
ap.add_argument("--jobs", required=True)
ap.add_argument("--shard", type=int, default=0)
ap.add_argument("--nshards", type=int, default=1)
ap.add_argument("--only", default=None, help="只做 out 文件名匹配这个 glob 的行")
ap.add_argument("--redo", action="store_true")
a = ap.parse_args()
import torch
from f0gen import Qwen21
rows = [json.loads(s) for s in open(rp(a.jobs)) if s.strip()]
rows = [r for i, r in enumerate(rows) if i % a.nshards == a.shard]
if a.only:
rows = [r for r in rows if fnmatch.fnmatch(os.path.basename(r["out"]), a.only)]
todo = [r for r in rows if a.redo or not (os.path.exists(rp(r["out"])) and
os.path.exists(os.path.splitext(rp(r["out"]))[0] + ".json"))]
print(f"[edit_app_f0] {len(todo)}/{len(rows)} 待编辑", flush=True)
if not todo:
return
t0 = time.time()
q21 = Qwen21("cuda")
print(f"[edit_app_f0] qwen21 加载 {time.time() - t0:.0f}s", flush=True)
for j in todo:
out_p = rp(j["out"])
os.makedirs(os.path.dirname(out_p), exist_ok=True)
img = L.area_down(np.asarray(Image.open(rp(j["src"])).convert("RGB")))
ids0 = load_ids0(j["ids"])
torch.cuda.reset_peak_memory_stats()
t1 = time.time()
try:
if j["kind"] == "global":
res, info = edit_global(q21, img, j)
elif j["kind"] == "char":
res, info = edit_char(q21, img, ids0, j)
elif j["kind"] == "ground":
res, info = edit_ground(q21, img, ids0, j)
else:
raise ValueError(f"未知 kind {j['kind']!r}")
except Exception as e: # noqa: BLE001
print(f"[edit_app_f0] FAIL {out_p}: {e!r}", flush=True)
json.dump({"job": j, "error": repr(e)}, open(os.path.splitext(out_p)[0] + ".err.json", "w"), indent=1)
continue
raw = info.pop("raw", None)
if raw is not None:
Image.fromarray(raw).save(os.path.splitext(out_p)[0] + "_raw.png")
Image.fromarray(res).save(out_p)
info.update(job=j, sec=round(time.time() - t1, 2), peak_gib=round(torch.cuda.max_memory_reserved() / 2 ** 30, 1),
struct=FF.struct_metrics(img, res), layout=FF.layout_metrics(res, ids0),
layout_src=FF.layout_metrics(img, ids0))
json.dump(info, open(os.path.splitext(out_p)[0] + ".json", "w"), indent=1)
print(f"[edit_app_f0] {out_p} {info['sec']}s struct={info['struct']}", flush=True)
print(f"[edit_app_f0] 完成 {len(todo)} 张 · {time.time() - t0:.0f}s", flush=True)
if __name__ == "__main__":
main()