Download scripts/teaser/edit_app_f0.py from teawhite/maliang: direct link, hf CLI and curl.
- Browser
- Download file 10.7 kB
-
https://huggingface.co/teawhite/maliang/resolve/main/scripts/teaser/edit_app_f0.py
- Command line
-
hf download hf://teawhite/maliang/scripts/teaser/edit_app_f0.py
-
curl -L -o edit_app_f0.py https://huggingface.co/teawhite/maliang/resolve/main/scripts/teaser/edit_app_f0.py
10.7 kB
| #!/usr/bin/env python3 | |
| """teaser · 编辑 case「外观编辑」的首帧编辑器(img env:$M/envs/img/bin/python,diffusers main,GPU)。 | |
| 语义不动、只改首帧:Qwen-Image-2.1-Turbo(f0gen.Qwen21,8 步)对底子首帧(832×480)做图像编辑,输出同尺寸 png + json。 | |
| 三种编辑(jobs 行的 kind) | |
| -------------------------- | |
| * global :整图编辑(时间 / 天气 / 季节:night / dusk / snow / autumn / spring / lava …)。参考图 = 原首帧放大到 1664×960, | |
| 提示词 = job.prompt + KEEP(布局、物体、角色位置姿势不变,不增删)。不钉回(夜晚角色也要变暗)。 | |
| 可选 job.best_of = N:N 个种子里留结构边缘 F1(对原图)最高的一张。 | |
| * char :角色局部编辑(主角换盔甲 / 换英雄、Boss 换造型)。target = player | boss:该角色帧 0 掩码(ids[0] 的部件 id)的外接框 | |
| (f0lib.mask_box,外扩 0.6、至少 160 px)裁出 → 放到 ≈1 MP → Qwen21 编辑 → 缩回、框内「离角色 ≥ 8 px 的背景」逐通道 | |
| 均值 / 方差匹配回原图(去偏色)→ 只在掩码膨胀 r = max(10, 0.35·角色框短边) px、羽化 3 px 内贴回,**掩码外逐像素钉回原图**; | |
| 另一个角色的掩码(膨胀 2 px)也钉回原图(Boss 修补不会盖掉贴身的主角)。 | |
| * ground :地面换材质。整图编辑「只把 <类描述> 换成 <新材质>」,然后只在 ids[0] ∈ classes 的掩码(腐蚀 1 px、羽化 2 px)内用编辑结果, | |
| 掩码外逐像素钉回原图。 | |
| 每张写 <out>.png + <out>.json(job、提示词、种子、秒数、显存、struct_metrics(原图, 编辑)、掩码像素数、贴回参数); | |
| char / ground 另存 <out>_raw.png(贴回前的编辑原样,查问题用)。已有 png + json ⇒ 跳过(可续跑,--redo 重做)。 | |
| jobs.jsonl 行(edit_app.py f0jobs 生成):{"out", "src"(原首帧 png), "ids"(npz,取帧 0), "kind", "prompt", "seed", | |
| ["target"], ["classes"], ...任意标签}。 | |
| usage(env:HF_HOME=/opt/dlami/nvme/zhiyangdeng/.cache/huggingface HF_HUB_OFFLINE=1 TMPDIR=$M/.tmp): | |
| CUDA_VISIBLE_DEVICES=6 $M/envs/img/bin/python scripts/teaser/edit_app_f0.py --jobs outputs/teaser/edit/app/f0/jobs.jsonl \\ | |
| [--shard 0 --nshards 1] [--only <glob>] [--redo] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import fnmatch | |
| import json | |
| import os | |
| import sys | |
| import time | |
| os.environ.setdefault("HF_HUB_OFFLINE", "1") | |
| os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") | |
| os.environ.setdefault("HF_HOME", "/opt/dlami/nvme/zhiyangdeng/.cache/huggingface") | |
| import numpy as np # noqa: E402 | |
| from PIL import Image # noqa: E402 | |
| M = "/opt/dlami/nvme/zhiyangdeng/maliang" | |
| sys.path[:0] = [f"{M}/scripts/teaser", f"{M}/code/gc25d", f"{M}/scripts/genbench3"] | |
| import f0lib as L # noqa: E402 | |
| import firstframe_flux as FF # noqa: E402 | |
| GEN = (1664, 960) | |
| KEEP = (" Keep exactly the same composition, camera angle, layout and every building, wall, rock, plant and object, with the " | |
| "same positions, outlines and sizes; keep both characters exactly where they are with the same poses, sizes, " | |
| "facing directions and weapons. Do not add or remove anything; no text, no UI, no border.") | |
| CHAR_KEEP = (" Keep exactly the same position, pose, size, facing direction and limb directions, and keep the weapon in the " | |
| "same hand. Keep everything else in the image exactly unchanged; do not add any other characters or objects.") | |
| def rp(p: str) -> str: | |
| return p if os.path.isabs(p) else os.path.join(M, p) | |
| def load_ids0(path: str) -> np.ndarray: | |
| with np.load(rp(path)) as z: | |
| a = z["ids"] if "ids" in z else z[list(z.keys())[0]] | |
| return np.ascontiguousarray(a[0] if a.ndim == 3 else a) | |
| def up(img: np.ndarray, size=GEN) -> np.ndarray: | |
| import cv2 | |
| return cv2.resize(img, size, interpolation=cv2.INTER_LANCZOS4) | |
| def color_match(out: np.ndarray, ref: np.ndarray, bg: np.ndarray) -> np.ndarray: | |
| """out 的 bg 区域逐通道均值 / 方差对齐到 ref 的同一区域(f0gen.charfix 的去偏色)。""" | |
| if bg.sum() < 200: | |
| return out | |
| o, r = out.astype(np.float32), ref.astype(np.float32) | |
| mu_o, sd_o = o[bg].mean(0), o[bg].std(0) + 1e-3 | |
| mu_r, sd_r = r[bg].mean(0), r[bg].std(0) + 1e-3 | |
| return np.clip((o - mu_o) * np.clip(sd_r / sd_o, 0.7, 1.4) + mu_r, 0, 255) | |
| def edit_global(q21, img: np.ndarray, j: dict): | |
| """job.best_of = N(默认 1):种子 seed, seed+100, … 各编辑一次,留 struct_metrics(原图, 编辑).edge_f1 最高的 | |
| (下雪等编辑偶尔会凭空加墙 / 篱笆 —— 结构边缘对不上原图 ⇒ F1 低)。""" | |
| p = j["prompt"].strip() + KEEP | |
| best, cands = None, [] | |
| for k in range(int(j.get("best_of", 1))): | |
| sd = int(j["seed"]) + 100 * k | |
| out = L.area_down(q21(p, [up(img)], GEN, sd)) | |
| f1 = float(FF.struct_metrics(img, out)["edge_f1"]) | |
| cands.append({"seed": sd, "edge_f1": round(f1, 4)}) | |
| if best is None or f1 > best[1]: | |
| best = (out, f1, sd) | |
| info = {"prompt": p} | |
| if len(cands) > 1: | |
| info.update(best_of=len(cands), candidates=cands, picked_seed=best[2]) | |
| return best[0], info | |
| def edit_char(q21, img: np.ndarray, ids0: np.ndarray, j: dict): | |
| import cv2 | |
| who = j["target"] | |
| mk = np.isin(ids0, L.PLAYER if who == "player" else L.BOSS) | |
| if mk.sum() < L.MIN_PX: | |
| raise ValueError(f"帧 0 里看不到 {who}({int(mk.sum())} px)") | |
| x0, y0, x1, y1 = L.mask_box(mk, margin=0.6, min_side=160) | |
| crop = img[y0:y1, x0:x1] | |
| cw, ch = L.W, L.H | |
| import math | |
| s = math.sqrt(1024 * 1024 / ((x1 - x0) * (y1 - y0))) | |
| cw, ch = max(32, round((x1 - x0) * s / 32) * 32), max(32, round((y1 - y0) * s / 32) * 32) | |
| p = j["prompt"].strip() + CHAR_KEEP | |
| out = q21(p, [cv2.resize(crop, (cw, ch), interpolation=cv2.INTER_LANCZOS4)], (cw, ch), int(j["seed"])) | |
| out = cv2.resize(out, (x1 - x0, y1 - y0), interpolation=cv2.INTER_AREA) | |
| other = np.isin(ids0, L.PLAYER + L.BOSS)[y0:y1, x0:x1] | |
| bg = cv2.dilate(other.astype(np.uint8), np.ones((17, 17), np.uint8)) == 0 | |
| out = color_match(out, crop, bg) | |
| ys, xs = np.nonzero(mk) | |
| r = max(10, round(float(j.get("dilate_frac", 0.35)) * min(xs.max() - xs.min() + 1, ys.max() - ys.min() + 1))) | |
| a = L.soft_mask(mk, r, 3.0) | |
| other_mk = np.isin(ids0, L.BOSS if who == "player" else L.PLAYER) # 另一个角色钉住(膨胀 2 px),不被贴回区盖掉 | |
| if other_mk.sum(): | |
| a = a * (1.0 - L.soft_mask(other_mk & ~mk, 2, 1.0)) | |
| a = a[y0:y1, x0:x1, None] | |
| raw = img.copy() | |
| raw[y0:y1, x0:x1] = out.astype(np.uint8) | |
| res = img.copy() | |
| res[y0:y1, x0:x1] = np.clip(a * out + (1 - a) * crop, 0, 255).astype(np.uint8) | |
| return res, {"prompt": p, "box": [int(x0), int(y0), int(x1), int(y1)], "edit_size": [cw, ch], "dilate_px": int(r), | |
| "mask_px": int(mk.sum()), "raw": raw} | |
| def edit_ground(q21, img: np.ndarray, ids0: np.ndarray, j: dict): | |
| import cv2 | |
| cls = tuple(int(c) for c in j["classes"]) | |
| mk = np.isin(ids0, cls) | |
| if mk.sum() < 2000: | |
| raise ValueError(f"帧 0 里类 {cls} 太少({int(mk.sum())} px)") | |
| p = j["prompt"].strip() + (" Only the ground surface changes; every building, wall, rock, plant, object and both " | |
| "characters stay exactly the same." + KEEP) | |
| out = L.area_down(q21(p, [up(img)], GEN, int(j["seed"]))) | |
| er = cv2.erode(mk.astype(np.uint8), np.ones((3, 3), np.uint8)).astype(np.float32) | |
| a = np.maximum(cv2.GaussianBlur(er, (0, 0), 2.0), er)[..., None] | |
| res = np.clip(a * out + (1 - a) * img, 0, 255).astype(np.uint8) | |
| return res, {"prompt": p, "classes": list(cls), "mask_px": int(mk.sum()), "raw": out} | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) | |
| ap.add_argument("--jobs", required=True) | |
| ap.add_argument("--shard", type=int, default=0) | |
| ap.add_argument("--nshards", type=int, default=1) | |
| ap.add_argument("--only", default=None, help="只做 out 文件名匹配这个 glob 的行") | |
| ap.add_argument("--redo", action="store_true") | |
| a = ap.parse_args() | |
| import torch | |
| from f0gen import Qwen21 | |
| rows = [json.loads(s) for s in open(rp(a.jobs)) if s.strip()] | |
| rows = [r for i, r in enumerate(rows) if i % a.nshards == a.shard] | |
| if a.only: | |
| rows = [r for r in rows if fnmatch.fnmatch(os.path.basename(r["out"]), a.only)] | |
| todo = [r for r in rows if a.redo or not (os.path.exists(rp(r["out"])) and | |
| os.path.exists(os.path.splitext(rp(r["out"]))[0] + ".json"))] | |
| print(f"[edit_app_f0] {len(todo)}/{len(rows)} 待编辑", flush=True) | |
| if not todo: | |
| return | |
| t0 = time.time() | |
| q21 = Qwen21("cuda") | |
| print(f"[edit_app_f0] qwen21 加载 {time.time() - t0:.0f}s", flush=True) | |
| for j in todo: | |
| out_p = rp(j["out"]) | |
| os.makedirs(os.path.dirname(out_p), exist_ok=True) | |
| img = L.area_down(np.asarray(Image.open(rp(j["src"])).convert("RGB"))) | |
| ids0 = load_ids0(j["ids"]) | |
| torch.cuda.reset_peak_memory_stats() | |
| t1 = time.time() | |
| try: | |
| if j["kind"] == "global": | |
| res, info = edit_global(q21, img, j) | |
| elif j["kind"] == "char": | |
| res, info = edit_char(q21, img, ids0, j) | |
| elif j["kind"] == "ground": | |
| res, info = edit_ground(q21, img, ids0, j) | |
| else: | |
| raise ValueError(f"未知 kind {j['kind']!r}") | |
| except Exception as e: # noqa: BLE001 | |
| print(f"[edit_app_f0] FAIL {out_p}: {e!r}", flush=True) | |
| json.dump({"job": j, "error": repr(e)}, open(os.path.splitext(out_p)[0] + ".err.json", "w"), indent=1) | |
| continue | |
| raw = info.pop("raw", None) | |
| if raw is not None: | |
| Image.fromarray(raw).save(os.path.splitext(out_p)[0] + "_raw.png") | |
| Image.fromarray(res).save(out_p) | |
| info.update(job=j, sec=round(time.time() - t1, 2), peak_gib=round(torch.cuda.max_memory_reserved() / 2 ** 30, 1), | |
| struct=FF.struct_metrics(img, res), layout=FF.layout_metrics(res, ids0), | |
| layout_src=FF.layout_metrics(img, ids0)) | |
| json.dump(info, open(os.path.splitext(out_p)[0] + ".json", "w"), indent=1) | |
| print(f"[edit_app_f0] {out_p} {info['sec']}s struct={info['struct']}", flush=True) | |
| print(f"[edit_app_f0] 完成 {len(todo)} 张 · {time.time() - t0:.0f}s", flush=True) | |
| if __name__ == "__main__": | |
| main() | |