#!/usr/bin/env python3 """teaser · 编辑 case「外观编辑」的首帧编辑器(img env:$M/envs/img/bin/python,diffusers main,GPU)。 语义不动、只改首帧:Qwen-Image-2.1-Turbo(f0gen.Qwen21,8 步)对底子首帧(832×480)做图像编辑,输出同尺寸 png + json。 三种编辑(jobs 行的 kind) -------------------------- * global :整图编辑(时间 / 天气 / 季节:night / dusk / snow / autumn / spring / lava …)。参考图 = 原首帧放大到 1664×960, 提示词 = job.prompt + KEEP(布局、物体、角色位置姿势不变,不增删)。不钉回(夜晚角色也要变暗)。 可选 job.best_of = N:N 个种子里留结构边缘 F1(对原图)最高的一张。 * char :角色局部编辑(主角换盔甲 / 换英雄、Boss 换造型)。target = player | boss:该角色帧 0 掩码(ids[0] 的部件 id)的外接框 (f0lib.mask_box,外扩 0.6、至少 160 px)裁出 → 放到 ≈1 MP → Qwen21 编辑 → 缩回、框内「离角色 ≥ 8 px 的背景」逐通道 均值 / 方差匹配回原图(去偏色)→ 只在掩码膨胀 r = max(10, 0.35·角色框短边) px、羽化 3 px 内贴回,**掩码外逐像素钉回原图**; 另一个角色的掩码(膨胀 2 px)也钉回原图(Boss 修补不会盖掉贴身的主角)。 * ground :地面换材质。整图编辑「只把 <类描述> 换成 <新材质>」,然后只在 ids[0] ∈ classes 的掩码(腐蚀 1 px、羽化 2 px)内用编辑结果, 掩码外逐像素钉回原图。 每张写 .png + .json(job、提示词、种子、秒数、显存、struct_metrics(原图, 编辑)、掩码像素数、贴回参数); char / ground 另存 _raw.png(贴回前的编辑原样,查问题用)。已有 png + json ⇒ 跳过(可续跑,--redo 重做)。 jobs.jsonl 行(edit_app.py f0jobs 生成):{"out", "src"(原首帧 png), "ids"(npz,取帧 0), "kind", "prompt", "seed", ["target"], ["classes"], ...任意标签}。 usage(env:HF_HOME=/opt/dlami/nvme/zhiyangdeng/.cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=$M/.tmp): CUDA_VISIBLE_DEVICES=6 $M/envs/img/bin/python scripts/teaser/edit_app_f0.py --jobs outputs/teaser/edit/app/f0/jobs.jsonl \\ [--shard 0 --nshards 1] [--only ] [--redo] """ from __future__ import annotations import argparse import fnmatch import json import os import sys import time os.environ.setdefault("HF_HUB_OFFLINE", "1") os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") os.environ.setdefault("HF_HOME", "/opt/dlami/nvme/zhiyangdeng/.cache/huggingface") import numpy as np # noqa: E402 from PIL import Image # noqa: E402 M = "/opt/dlami/nvme/zhiyangdeng/maliang" sys.path[:0] = [f"{M}/scripts/teaser", f"{M}/code/gc25d", f"{M}/scripts/genbench3"] import f0lib as L # noqa: E402 import firstframe_flux as FF # noqa: E402 GEN = (1664, 960) KEEP = (" Keep exactly the same composition, camera angle, layout and every building, wall, rock, plant and object, with the " "same positions, outlines and sizes; keep both characters exactly where they are with the same poses, sizes, " "facing directions and weapons. Do not add or remove anything; no text, no UI, no border.") CHAR_KEEP = (" Keep exactly the same position, pose, size, facing direction and limb directions, and keep the weapon in the " "same hand. Keep everything else in the image exactly unchanged; do not add any other characters or objects.") def rp(p: str) -> str: return p if os.path.isabs(p) else os.path.join(M, p) def load_ids0(path: str) -> np.ndarray: with np.load(rp(path)) as z: a = z["ids"] if "ids" in z else z[list(z.keys())[0]] return np.ascontiguousarray(a[0] if a.ndim == 3 else a) def up(img: np.ndarray, size=GEN) -> np.ndarray: import cv2 return cv2.resize(img, size, interpolation=cv2.INTER_LANCZOS4) def color_match(out: np.ndarray, ref: np.ndarray, bg: np.ndarray) -> np.ndarray: """out 的 bg 区域逐通道均值 / 方差对齐到 ref 的同一区域(f0gen.charfix 的去偏色)。""" if bg.sum() < 200: return out o, r = out.astype(np.float32), ref.astype(np.float32) mu_o, sd_o = o[bg].mean(0), o[bg].std(0) + 1e-3 mu_r, sd_r = r[bg].mean(0), r[bg].std(0) + 1e-3 return np.clip((o - mu_o) * np.clip(sd_r / sd_o, 0.7, 1.4) + mu_r, 0, 255) def edit_global(q21, img: np.ndarray, j: dict): """job.best_of = N(默认 1):种子 seed, seed+100, … 各编辑一次,留 struct_metrics(原图, 编辑).edge_f1 最高的 (下雪等编辑偶尔会凭空加墙 / 篱笆 —— 结构边缘对不上原图 ⇒ F1 低)。""" p = j["prompt"].strip() + KEEP best, cands = None, [] for k in range(int(j.get("best_of", 1))): sd = int(j["seed"]) + 100 * k out = L.area_down(q21(p, [up(img)], GEN, sd)) f1 = float(FF.struct_metrics(img, out)["edge_f1"]) cands.append({"seed": sd, "edge_f1": round(f1, 4)}) if best is None or f1 > best[1]: best = (out, f1, sd) info = {"prompt": p} if len(cands) > 1: info.update(best_of=len(cands), candidates=cands, picked_seed=best[2]) return best[0], info def edit_char(q21, img: np.ndarray, ids0: np.ndarray, j: dict): import cv2 who = j["target"] mk = np.isin(ids0, L.PLAYER if who == "player" else L.BOSS) if mk.sum() < L.MIN_PX: raise ValueError(f"帧 0 里看不到 {who}({int(mk.sum())} px)") x0, y0, x1, y1 = L.mask_box(mk, margin=0.6, min_side=160) crop = img[y0:y1, x0:x1] cw, ch = L.W, L.H import math s = math.sqrt(1024 * 1024 / ((x1 - x0) * (y1 - y0))) cw, ch = max(32, round((x1 - x0) * s / 32) * 32), max(32, round((y1 - y0) * s / 32) * 32) p = j["prompt"].strip() + CHAR_KEEP out = q21(p, [cv2.resize(crop, (cw, ch), interpolation=cv2.INTER_LANCZOS4)], (cw, ch), int(j["seed"])) out = cv2.resize(out, (x1 - x0, y1 - y0), interpolation=cv2.INTER_AREA) other = np.isin(ids0, L.PLAYER + L.BOSS)[y0:y1, x0:x1] bg = cv2.dilate(other.astype(np.uint8), np.ones((17, 17), np.uint8)) == 0 out = color_match(out, crop, bg) ys, xs = np.nonzero(mk) r = max(10, round(float(j.get("dilate_frac", 0.35)) * min(xs.max() - xs.min() + 1, ys.max() - ys.min() + 1))) a = L.soft_mask(mk, r, 3.0) other_mk = np.isin(ids0, L.BOSS if who == "player" else L.PLAYER) # 另一个角色钉住(膨胀 2 px),不被贴回区盖掉 if other_mk.sum(): a = a * (1.0 - L.soft_mask(other_mk & ~mk, 2, 1.0)) a = a[y0:y1, x0:x1, None] raw = img.copy() raw[y0:y1, x0:x1] = out.astype(np.uint8) res = img.copy() res[y0:y1, x0:x1] = np.clip(a * out + (1 - a) * crop, 0, 255).astype(np.uint8) return res, {"prompt": p, "box": [int(x0), int(y0), int(x1), int(y1)], "edit_size": [cw, ch], "dilate_px": int(r), "mask_px": int(mk.sum()), "raw": raw} def edit_ground(q21, img: np.ndarray, ids0: np.ndarray, j: dict): import cv2 cls = tuple(int(c) for c in j["classes"]) mk = np.isin(ids0, cls) if mk.sum() < 2000: raise ValueError(f"帧 0 里类 {cls} 太少({int(mk.sum())} px)") p = j["prompt"].strip() + (" Only the ground surface changes; every building, wall, rock, plant, object and both " "characters stay exactly the same." + KEEP) out = L.area_down(q21(p, [up(img)], GEN, int(j["seed"]))) er = cv2.erode(mk.astype(np.uint8), np.ones((3, 3), np.uint8)).astype(np.float32) a = np.maximum(cv2.GaussianBlur(er, (0, 0), 2.0), er)[..., None] res = np.clip(a * out + (1 - a) * img, 0, 255).astype(np.uint8) return res, {"prompt": p, "classes": list(cls), "mask_px": int(mk.sum()), "raw": out} def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--jobs", required=True) ap.add_argument("--shard", type=int, default=0) ap.add_argument("--nshards", type=int, default=1) ap.add_argument("--only", default=None, help="只做 out 文件名匹配这个 glob 的行") ap.add_argument("--redo", action="store_true") a = ap.parse_args() import torch from f0gen import Qwen21 rows = [json.loads(s) for s in open(rp(a.jobs)) if s.strip()] rows = [r for i, r in enumerate(rows) if i % a.nshards == a.shard] if a.only: rows = [r for r in rows if fnmatch.fnmatch(os.path.basename(r["out"]), a.only)] todo = [r for r in rows if a.redo or not (os.path.exists(rp(r["out"])) and os.path.exists(os.path.splitext(rp(r["out"]))[0] + ".json"))] print(f"[edit_app_f0] {len(todo)}/{len(rows)} 待编辑", flush=True) if not todo: return t0 = time.time() q21 = Qwen21("cuda") print(f"[edit_app_f0] qwen21 加载 {time.time() - t0:.0f}s", flush=True) for j in todo: out_p = rp(j["out"]) os.makedirs(os.path.dirname(out_p), exist_ok=True) img = L.area_down(np.asarray(Image.open(rp(j["src"])).convert("RGB"))) ids0 = load_ids0(j["ids"]) torch.cuda.reset_peak_memory_stats() t1 = time.time() try: if j["kind"] == "global": res, info = edit_global(q21, img, j) elif j["kind"] == "char": res, info = edit_char(q21, img, ids0, j) elif j["kind"] == "ground": res, info = edit_ground(q21, img, ids0, j) else: raise ValueError(f"未知 kind {j['kind']!r}") except Exception as e: # noqa: BLE001 print(f"[edit_app_f0] FAIL {out_p}: {e!r}", flush=True) json.dump({"job": j, "error": repr(e)}, open(os.path.splitext(out_p)[0] + ".err.json", "w"), indent=1) continue raw = info.pop("raw", None) if raw is not None: Image.fromarray(raw).save(os.path.splitext(out_p)[0] + "_raw.png") Image.fromarray(res).save(out_p) info.update(job=j, sec=round(time.time() - t1, 2), peak_gib=round(torch.cuda.max_memory_reserved() / 2 ** 30, 1), struct=FF.struct_metrics(img, res), layout=FF.layout_metrics(res, ids0), layout_src=FF.layout_metrics(img, ids0)) json.dump(info, open(os.path.splitext(out_p)[0] + ".json", "w"), indent=1) print(f"[edit_app_f0] {out_p} {info['sec']}s struct={info['struct']}", flush=True) print(f"[edit_app_f0] 完成 {len(todo)} 张 · {time.time() - t0:.0f}s", flush=True) if __name__ == "__main__": main()