"""teaser · 首帧生成的共用件(纯 numpy / cv2 / PIL,img env 与 maliang venv 都能 import):读地图、派生输入图、拼提示词、 角色框 / 贴回、对比拼图。生成后端在 f0gen.py,扫参在 f0sweep.py。 地图目录(procmap 的输出,outputs/genbench/procmap//、outputs/teaser/maps//、outputs/teaser/f0_sweep/maps//): ids.npz([81,480,832] parts raw id,取帧 0)、map0.png(色图)、map0_shaded.png(按法线打光的体块)、depth0.png(16 bit,近亮)。 派生(load_map 里一次算好,键见 load_map 的 docstring): * flat :firstframe_flux.flux_map 的 "white" 色表(静态类 = geometry 色,玩家蓝 / Boss 红 / 武器白),**不画铺地网格** (v4 的 16 px 网格提示会让首帧和视频里出现网格线); * shaded :map0_shaded 原样(玩家浅蓝、Boss 浅鲑红的胶囊人体 = 「人偶」,qwen21 会照画成人偶 ⇒ 提示词说是占位符 + 角色修补); * sil :shaded 但角色换成深蓝 / 深红实色(保留明暗); * depth8 :depth0 → 8 bit 灰度 RGB;edge:语义类边界(角色部件合并)白线黑底,2 px。 提示词(scene_prompt):只写帧 0 里出现的类(像素 ≥ MIN_PX),「色 = 该场景里的这一类」(scenes.SCENES[s]["cls"]);没出现的 建筑 / 石头 / 植被写「这一带没有、不许加」(genbench4 edit2 的经验);植被 / 道具附形状提示(体块图里的圆球 = 俯视的树冠); 角色句只写帧 0 里可见的(Boss 不可见就只有玩家);结尾禁文字 / UI / 网格 / 边框、禁沿画面边缘加墙。 画风:场景键可写 "<场景>@<画风>"(scenes.SCENES 按需派生,title / light 带画风句);char_style / texture_phrase 让角色描述与 纹理句跟着画风走;restyle_prompt = f0gen 的整图画风重绘提示词(photoreal / 无画风 ⇒ None)。 其它:scene_blockout(体块图按场景色表重新上色,scb* 变体)、dummy_frac(人偶检测规则:角色掩码里「标记色且光滑」的比例)、 charfix_prompt / refine_prompt、VLM_Q + vlm_items(Qwen3-VL-8B 题)、pixelate(可选像素化后处理)、label / grid(拼图); nochar_map / real_chars / soft_mask / mask_box / tint_match / harmonize_prompt 是实例 R(realchar)加的。 """ from __future__ import annotations import glob import json import os import sys import numpy as np from PIL import Image M = "/opt/dlami/nvme/zhiyangdeng/maliang" for _p in (f"{M}/code/gc25d", f"{M}/scripts/teaser"): if _p not in sys.path: sys.path.insert(0, _p) import firstframe_flux as FF # noqa: E402 (只 import numpy / PIL;layout_metrics 要 scipy) from scenes import SCENES # noqa: E402 W, H = 832, 480 PLAYER, BOSS = (10, 12, 13, 14, 15, 16), (11, 17, 18, 19, 20, 21) MIN_PX = FF.MIN_PX # 少于 40 px 的类 / 角色不写进提示词 MAP_ROOTS = (f"{M}/outputs/genbench/procmap", f"{M}/outputs/teaser/maps", f"{M}/outputs/teaser/f0_sweep/maps") # 体块图 / 色图里每类的颜色名(geometry.PALETTE:terrain 棕、paving 浅灰、architecture 橙红、rock 深灰、vegetation 绿、prop 黄) COLOR_NAME = {0: "sky blue", 1: "brown", 2: "light grey", 3: "orange-red", 4: "dark grey", 5: "green", 6: "yellow"} COLOR_NAME_SHP = {**COLOR_NAME, 6: "the small separate objects"} # shp:道具已按场景上色,不写颜色名 SHAPE_HINT = {5: " (round green shapes are tree crowns and bushes seen from above, thin green posts are small trunks or " "stems)", 6: " (small separate objects)", 4: " (grey lumps and boulders)"} SCB_SUBJ = {0: "the empty background", 1: "the flat open ground areas", 2: "the flat paved floors and paths", 3: "the boxy buildings and walls", 4: "the lumpy grey boulders", 5: "the round crowns and posts (plants)", 6: "the small separate objects"} ABSENT = {3: "buildings, walls, towers, gates, huts or fences", 4: "rocks or boulders", 5: "trees or plants"} # ------------------------------------------------------------------ 地图 ---- def find_map(case: str) -> str: for r in MAP_ROOTS: if os.path.isfile(f"{r}/{case}/ids.npz"): return f"{r}/{case}" raise FileNotFoundError(f"找不到地图 {case}(查过 {MAP_ROOTS})") def list_maps(roots=MAP_ROOTS) -> list[str]: """所有齐全的地图目录(ids.npz + map0_shaded.png + depth0.png)。""" out = [] for r in roots: for d in sorted(glob.glob(f"{r}/*/")): d = d.rstrip("/") if all(os.path.isfile(f"{d}/{f}") for f in ("ids.npz", "map0_shaded.png", "depth0.png")): out.append(d) return out def load_ids0(path: str, frame: int = 0) -> np.ndarray: """ids.npz([F,H,W])/ .npy / id png / 色图 png → parts raw id [H,W](firstframe_flux.load_ids 的超集)。""" if path.endswith(".npy"): a = np.load(path) return np.ascontiguousarray(a[frame] if a.ndim == 3 else a).astype(np.uint8) return FF.load_ids(path, frame) def char_masks(ids: np.ndarray) -> dict: p, b = np.isin(ids, PLAYER), np.isin(ids, BOSS) return {"player": p, "boss": b, "has_p": int(p.sum()) >= MIN_PX, "has_b": int(b.sum()) >= MIN_PX, "player_px": int(p.sum()), "boss_px": int(b.sum()), "boss_weapon": int((ids == 21).sum()) >= 20, "player_weapon": int((ids == 16).sum()) >= 10} def sem_edges(ids: np.ndarray, width: int = 2) -> np.ndarray: import cv2 c = ids.copy() c[np.isin(c, PLAYER)] = 10 c[np.isin(c, BOSS)] = 11 e = np.zeros(c.shape, np.uint8) e[:, 1:] |= c[:, 1:] != c[:, :-1] e[1:, :] |= c[1:, :] != c[:-1, :] e = cv2.dilate(e * 255, np.ones((width, width), np.uint8)) return np.repeat(e[..., None], 3, -1) def recolor_chars(rgb: np.ndarray, ids: np.ndarray, player=(40, 70, 230), boss=(200, 30, 30)) -> np.ndarray: """角色像素换成实色(乘原图亮度 / 200 保留明暗)。""" out = rgb.copy() lum = rgb.astype(np.float32).mean(-1, keepdims=True) / 200.0 for m, c in ((np.isin(ids, PLAYER), player), (np.isin(ids, BOSS), boss)): out[m] = np.clip(np.asarray(c, np.float32) * lum[m], 0, 255).astype(np.uint8) return out PALETTE_JSON = f"{M}/outputs/teaser/f0_sweep/palette.json" # f0sweep.py palette 生成:{scene: {"0".."6": [r,g,b]}} _PAL: dict = {} SCB_MIN_LUM = 70.0 def prop_blockout(m: dict, scene: str) -> np.ndarray: """体块图只把道具(类 6)按场景色表重新上色(t1:qwen21 把黄道具原样画成黄方块,其余类照抄颜色问题小);场景没有色表 ⇒ 原样。shp 变体用,提示词里道具那一项不写颜色名(COLOR_NAME_SHP)。""" if not _PAL and os.path.exists(PALETTE_JSON): _PAL.update(json.load(open(PALETTE_JSON))) pal = _PAL.get(base_of(scene), {}) if "6" not in pal: return m["shaded"] ids = m["ids"] mk = ids == 6 out = m["shaded"].copy() if mk.any(): f = np.clip(m["shaded"].astype(np.float32).mean(-1)[mk] / float(np.mean(FF.GEO_COLORS[6])), 0.3, 1.6)[:, None] col = np.asarray(pal["6"], np.float32) + 4.0 col = col * max(1.0, SCB_MIN_LUM / float(col.mean())) out[mk] = np.clip(np.minimum(col, 255)[None] * f, 0, 255).astype(np.uint8) return out def scene_blockout(m: dict, scene: str, mix: float = 1.0) -> np.ndarray: """体块图按场景色表重新上色:静态类像素 = 场景里该类的代表色 × 体块图的明暗(体块图亮度 / 该类 geometry 色亮度, 夹在 [0.3, 1.6]);角色保持原来的浅蓝 / 浅鲑红人偶。t1:qwen21 会把体块图的黄 / 绿原样画出来(墓碑成黄方块), 先换成场景色 ⇒ 照抄颜色也是对的颜色。""" if not _PAL: _PAL.update(json.load(open(PALETTE_JSON))) pal = _PAL[base_of(scene)] ids, sh = m["ids"], m["shaded"].astype(np.float32) lum = sh.mean(-1) out = m["shaded"].copy() for c in range(7): mk = ids == c if not mk.any(): continue base = float(np.mean(FF.GEO_COLORS[c])) f = np.clip(lum[mk] / base, 0.3, 1.6)[:, None] col = np.asarray(pal[str(c)], np.float32) + 4.0 col = col * max(1.0, SCB_MIN_LUM / float(col.mean())) # 暗场景(地牢 / 熔岩)的色提亮到均值 ≥ SCB_MIN_LUM,保色相 col = mix * col + (1 - mix) * np.asarray(FF.GEO_COLORS[c], np.float32) # mix < 1:往标准类色拉回(保类可分) out[mk] = np.clip(np.minimum(col, 255)[None] * f, 0, 255).astype(np.uint8) return out def load_map(map_dir: str) -> dict: """→ {dir, case, ids [H,W], map0, shaded, flat, sil, depth8, edge(都是 uint8 [H,W,3]), chars(char_masks), present, n(类像素计数 [256]), source(source.json 或 {})}。""" import cv2 ids = load_ids0(f"{map_dir}/ids.npz") assert ids.shape == (H, W), ids.shape rd = lambda f: np.asarray(Image.open(f"{map_dir}/{f}").convert("RGB")) # noqa: E731 d16 = cv2.imread(f"{map_dir}/depth0.png", cv2.IMREAD_UNCHANGED) d8 = (d16.astype(np.float32) / max(1, int(d16.max())) * 255).round().astype(np.uint8) if d16.dtype == np.uint16 \ else d16.astype(np.uint8) if d8.ndim == 3: d8 = d8[..., 0] shaded = rd("map0_shaded.png") n = np.bincount(ids.reshape(-1), minlength=256) src = {} if os.path.isfile(f"{map_dir}/source.json"): src = json.load(open(f"{map_dir}/source.json")) return {"dir": map_dir, "case": os.path.basename(map_dir.rstrip("/")), "ids": ids, "map0": rd("map0.png"), "shaded": shaded, "flat": FF.flux_map(ids, "white", tiles=False), "sil": recolor_chars(shaded, ids), "depth8": np.repeat(d8[..., None], 3, -1), "edge": sem_edges(ids), "chars": char_masks(ids), "present": [c for c in range(7) if n[c] >= MIN_PX], "n": n, "source": src} def nochar_map(m: dict) -> dict: """realchar 路线的第一阶段输入:把角色从地图里去掉 —— ids 的角色像素换成最近的静态类(distance_transform 最近邻), 体块图 / 色图在(膨胀 4 px 的)角色区域里 cv2.inpaint(Telea)补成周围的地面 ⇒ 生成器只画背景,角色之后从源 clip 真实帧 贴回(real_chars / f0gen.realchar)。返回新 dict(chars 全空,提示词自动写「没有角色」),原 dict 另存在 m['orig']。""" import cv2 from scipy import ndimage ids = m["ids"] ch = np.isin(ids, PLAYER + BOSS) _, (iy, ix) = ndimage.distance_transform_edt(ch, return_indices=True) ids2 = ids[iy, ix].astype(np.uint8) k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9)) mk = cv2.dilate(ch.astype(np.uint8), k) out = dict(m) out.update(ids=ids2, orig=m, chars=char_masks(ids2), flat=FF.flux_map(ids2, "white", tiles=False), shaded=cv2.inpaint(np.ascontiguousarray(m["shaded"]), mk, 5, cv2.INPAINT_TELEA), sil=cv2.inpaint(np.ascontiguousarray(m["shaded"]), mk, 5, cv2.INPAINT_TELEA), edge=sem_edges(ids2)) n = np.bincount(ids2.reshape(-1), minlength=256) out.update(n=n, present=[c for c in range(7) if n[c] >= MIN_PX]) return out def real_chars(m: dict) -> dict | None: """源 clip 的真实帧 0 + 真实 ids 帧 0(procmap 借的是源 clip 的相机 + 胶囊 ⇒ 角色在同一像素位置,实测 10 张地图里 9 张 IoU = 1.00,plaza P9C 0.83 = 真实帧里玩家被场景挡了一块)→ {rgb, ids, mask(玩家 ∪ Boss), clip_id}; 地图没有 source.json / clip_id 时返回 None。""" cid = (m.get("source") or {}).get("clip_id") if not cid: return None rgb = FF.clip_frame(cid) rid = FF.clip_ids(cid) return {"rgb": rgb, "ids": rid, "mask": np.isin(rid, PLAYER + BOSS), "clip_id": cid} def soft_mask(mask: np.ndarray, dilate: int, feather: float) -> np.ndarray: """bool 掩码 → 膨胀 dilate px(≤ 0 = 腐蚀)+ 高斯羽化 → float32 ∈ [0,1](羽化不往掩码内缩:max(羽化, 硬掩码))。""" import cv2 mk = mask.astype(np.uint8) if dilate > 0: mk = cv2.dilate(mk, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * dilate + 1, 2 * dilate + 1))) elif dilate < 0: mk = cv2.erode(mk, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (-2 * dilate + 1, -2 * dilate + 1))) a = mk.astype(np.float32) if feather > 0: a = np.maximum(cv2.GaussianBlur(a, (0, 0), feather), a) return np.clip(a, 0, 1) def mask_box(mask: np.ndarray, margin: float = 0.6, min_side: int = 160, pad: int = 24): """bool 掩码的外接框(x0,y0,x1,y1),每边外扩 max(pad, margin·边长)、至少 min_side,夹在画面内。""" if mask.sum() < MIN_PX: return None ys, xs = np.nonzero(mask) x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 bw, bh = x1 - x0, y1 - y0 cx, cy = (x0 + x1) / 2, (y0 + y1) / 2 hw = max(bw / 2 + max(pad, margin * bw), min_side / 2) hh = max(bh / 2 + max(pad, margin * bh), min_side / 2) return int(max(0, cx - hw)), int(max(0, cy - hh)), int(min(W, cx + hw)), int(min(H, cy + hh)) def tint_match(fg: np.ndarray, bg_new: np.ndarray, bg_old: np.ndarray, ring: np.ndarray, alpha: float = 0.5): """把真实角色的整体色调往新场景拉:增益 = 新背景环带均值 / 源背景环带均值(逐通道,夹在 [0.55, 1.3]),按 alpha 混合。""" if ring.sum() < 50: return fg g = (bg_new[ring].astype(np.float32).mean(0) + 8) / (bg_old[ring].astype(np.float32).mean(0) + 8) g = np.clip(g, 0.55, 1.3) g = 1 + alpha * (g - 1) return np.clip(fg.astype(np.float32) * g, 0, 255).astype(np.uint8) # ------------------------------------------------------------------ 画风(scenes.LOOKS) ---- def look_of(scene: str) -> str | None: """"<场景>@<画风>" → 画风键;纯场景名 → None(= 默认写实提示词)。""" return SCENES[scene].get("look") def base_of(scene: str) -> str: return scene.split("@", 1)[0] def with_look(scene: str, look: str | None) -> str: """job 行的 look 键并进场景键(scenes.SCENES 认 "<场景>@<画风>");已带 @ 或 look 为空就原样。""" return scene if (not look or "@" in scene) else f"{scene}@{look}" def char_style(scene: str | None) -> str: """角色的画风修饰(LOOKS[look]["char"],如 "pixel-art" / "stylized cartoon");无画风 = 空。""" lk = look_of(scene) if scene else None if not lk: return "" import scenes as _SC return _SC.LOOKS[lk]["char"] + " " def texture_phrase(scene: str | None) -> str: lk = look_of(scene) if scene else None if not lk or lk == "photoreal": return "detailed realistic textures" import scenes as _SC return f"detailed {_SC.LOOKS[lk]['short']} textures" # ------------------------------------------------------------------ 提示词 ---- def legend(m: dict, scene: str, color_name: dict = COLOR_NAME, hints: bool = True) -> str: sc = SCENES[scene] parts = [f"{color_name[c]} = {sc['cls'][c]}" + (SHAPE_HINT.get(c, "") if hints else "") for c in m["present"]] return "; ".join(parts) def absent_text(m: dict) -> str: miss = [ABSENT[c] for c in ABSENT if c not in m["present"]] return (" This part of the level contains no " + ", no ".join(miss) + "; do not add any.") if miss else "" def char_text(m: dict, style: str, scene: str | None = None) -> str: """style: placeholder(体块图里的胶囊人偶 = 占位符)| silhouette(深蓝 / 深红实色剪影)| marker(flat 色图的蓝 / 红人形)。 scene 带画风时角色描述加 LOOKS[look]["char"](像素 / 卡通画风下角色也是像素 / 卡通的)。""" ch = m["chars"] cs = char_style(scene) hero = f"a fully detailed {cs}armored human warrior hero" + (" holding a steel sword" if ch["player_weapon"] else "") boss = f"a huge, fully detailed {cs}monster boss" + (" wielding a big weapon" if ch["boss_weapon"] else "") who = {"placeholder": ("the light-blue mannequin", "the salmon-pink mannequin"), "silhouette": ("the dark blue silhouette", "the dark red silhouette"), "marker": ("the blue humanoid figure", "the red humanoid figure")}[style] items = ([f"{who[0]} becomes the player character, {hero}"] if ch["has_p"] else []) + \ ([f"{who[1]} becomes the boss, {boss}"] if ch["has_b"] else []) if not items: return " There are no characters in this scene: do not draw any people, creatures, figures or statues." n = len(items) lead = {"placeholder": " The smooth capsule mannequins are only placeholders and must not appear in the final image: " if n == 2 else " The smooth capsule mannequin is only a placeholder and must not appear in the final image: ", "silhouette": " ", "marker": " "}[style] txt = lead + "; ".join(items) + ". " txt += ("Each character has exactly the position, pose, size, facing direction and limb directions of its " "placeholder, and looks like a living video game character with natural colours and materials, not a " "mannequin, statue or toy. ") txt += "These are the only two characters; " if n == 2 else "This is the only character; " txt += "do not add any other people or creatures." return txt QUALITY = (" {light} Bright, clear and readable game lighting: every object is easy to see. High-end isometric action RPG " "graphics with {tex} and soft shadows; no text, " "no letters, no UI, no grid lines, no outlines, no border. Do not frame the scene with new walls, cliffs, " "trees or buildings along the image borders; the borders show the same kind of region as the blockout.") NOCOPY = (" The colours of picture 1 are only a code that says what each region is: do not copy them; give every " "region the natural colours and materials it has in {title}.") def scene_prompt(m: dict, scene: str, kind: str = "blockout", chars: str = "placeholder", nocopy: bool = False) -> str: """kind: blockout(体块图 / 剪影图:qwen21 / qie2511)| flatmap(平涂色图:klein / qie)。 nocopy:加 NOCOPY 句(t1:qwen21 常把体块图的黄 / 绿原样画出来 —— 墓地的墓碑成黄方块、糖果的树成绿球)。""" sc = SCENES[scene] if kind == "scb": # 场景色体块图:颜色已经是场景色,色表改成按形状 / 类说明 head = (f"Picture 1 is a simple untextured 3D blockout of one level of an isometric action RPG video game, " f"already painted in the colours of {sc['title']}. Render it as the finished, fully detailed in-game " "screenshot, seen from exactly the same high top-down camera. Keep every region, object and figure " "exactly where it is, with the same outline, size and shape, and add nothing that is not in picture " "1: open ground and floors stay open and are only textured. In picture 1: " + "; ".join(f"{SCB_SUBJ[c]} are {sc['cls'][c]}" for c in m["present"]) + ".") return head + absent_text(m) + char_text(m, chars, scene) + \ QUALITY.format(light=sc["light"], tex=texture_phrase(scene)) if kind == "scb2": # 场景色体块图 + 标准色码体块图(看类) head = (f"Picture 1 is a simple untextured 3D blockout of one level of an isometric action RPG video game, " f"already painted in the colours of {sc['title']}; picture 2 is the same blockout colour-coded by " f"region type ({legend(m, scene)}). Render picture 1 as the finished, fully detailed in-game screenshot " "of " + sc["title"] + ", seen from exactly the same high top-down camera. Keep every region, object " "and figure exactly where it is, with the same outline, size and shape, and add nothing that is not " "in the blockout: open ground and floors stay open and are only textured.") return head + absent_text(m) + char_text(m, chars, scene) + \ QUALITY.format(light=sc["light"], tex=texture_phrase(scene)) what = {"blockout": "a colour-coded 3D blockout", "shp": "a colour-coded 3D blockout", "flatmap": "a flat colour-coded layout map"}[kind] head = (f"Picture 1 is {what} of one level of an isometric action RPG video game. Render it as the finished in-game " f"screenshot of {sc['title']}, seen from exactly the same high top-down camera. Keep every region, object " f"and figure exactly where it is, with the same outline, size and shape, and add nothing that is not in " f"picture 1: open ground and floors stay open and are only textured. Colour code: " f"{legend(m, scene, COLOR_NAME_SHP if kind == 'shp' else COLOR_NAME)}.") return head + (NOCOPY.format(title=sc["title"]) if nocopy else "") + absent_text(m) + char_text(m, chars, scene) + \ QUALITY.format(light=sc["light"], tex=texture_phrase(scene)) def klein_prompt(m: dict, scene: str) -> str: """firstframe_flux.map2frame v3 模板(无网格)+ 场景色表 + 场景句(genbench4 make_scene 的 map 模式改进版)。""" sc = SCENES[scene] txt = ("Turn image 1, a flat color-coded layout map, into a finished screenshot of a 3D action RPG video game seen " "from a high top-down isometric camera. Follow the layout of image 1 exactly: every region, object and " "character keeps its exact position, shape and size, and nothing is added that is not in the map; open " f"ground and floors stay open. Color legend of the layout map: {legend(m, scene)}.") txt += absent_text(m) + char_text(m, "marker", scene) txt += (" Replace every flat color with detailed textured materials, natural lighting and soft shadows; no flat " f"colors, no outlines, no grid lines, no text, no UI, no border. Scene: the whole level is {sc['title']}. " f"{sc['light']}") return txt def zimage_prompt(m: dict, scene: str) -> str: """Z-Image(ControlNet 只给结构):整段场景描述 + 按面积从大到小列出各类在该场景里是什么 + 角色。""" sc = SCENES[scene] order = sorted(m["present"], key=lambda c: -m["n"][c]) ch = m["chars"] chars = ("an armored human warrior hero with a sword fighting a huge monster boss" if ch["has_p"] and ch["has_b"] else "a single armored human warrior hero with a sword" if ch["has_p"] else "no characters") return (f"A screenshot of a high-end isometric action RPG video game seen from a high top-down camera: {sc['title']}. " f"The ground is {sc['cls'][order[0]]}; the level also has " + ", ".join(sc["cls"][c] for c in order[1:]) + f". In the scene: {chars}, exactly two figures at most. {sc['light']} Detailed realistic textures, soft " "shadows, polished game graphics, no text, no UI, no grid lines, no border.") def charfix_prompt(m: dict, scene: str) -> str: ch = m["chars"] items = [] if ch["has_p"]: items.append(f"the light-blue mannequin (or the blue figure) into a fully detailed {char_style(scene)}armored human " "warrior hero" + (" holding a steel sword" if ch["player_weapon"] else "")) if ch["has_b"]: items.append("the salmon-pink mannequin (or the red figure)" + (" and its pink club" if ch["boss_weapon"] else "") + f" into a huge, fully detailed {char_style(scene)}monster boss" + (" wielding a big weapon" if ch["boss_weapon"] else "")) return ("Picture 1 is a close-up crop of a video game screenshot of " + SCENES[scene]["title"] + ". Edit picture 1: " "turn " + ", and turn ".join(items) + ", each in exactly the same position, pose, size and facing " "direction. They must look like living video game characters with natural colours and materials that fit " "the scene, not mannequins, statues or toys. If a character already looks fully detailed, keep it as it " "is. Keep everything else in the image exactly unchanged; do not add any other characters or objects.") def restyle_prompt(scene: str) -> str | None: """画风重绘(f0gen 的 restyle 阶段,qwen21 整图编辑):t4 冒烟 —— 场景提示词里写画风,qwen21 / klein 都还是画成 3D 渲染 (体块图本身是 3D 渲染的样子);先按写实生成再整图「换画风、布局不动」。look 为空 / photoreal ⇒ None(不重绘)。""" lk = look_of(scene) if not lk or lk == "photoreal": return None import scenes as _SC L_ = _SC.LOOKS[lk] return (f"Redraw picture 1 completely in this art style: {L_['prompt']} Every surface, object and character must be " f"in the {L_['short']} style, nothing may stay photorealistic. Keep exactly the same composition, camera, " "layout and every object, with the same positions, outlines and sizes; keep both characters with the same " f"positions, poses, sizes and weapons, drawn as {L_['char']} game characters. Do not add or remove anything; " "no text, no UI, no border.") def pixelate(img: np.ndarray, cell: int = 4, colors: int = 48) -> np.ndarray: """确定性的像素画后处理(look = pixel):INTER_AREA 缩到 1/cell → PIL 中位切分量化到 colors 色(不抖动)→ 最近邻放回。 t4c:qwen21 画风重绘的 pixel 只有一点点像素感,加这一步才是真正的方块像素。""" import cv2 h, w = img.shape[:2] sm = cv2.resize(img, (w // cell, h // cell), interpolation=cv2.INTER_AREA) q = Image.fromarray(sm).quantize(colors=colors, method=Image.Quantize.MEDIANCUT, dither=Image.Dither.NONE) return cv2.resize(np.asarray(q.convert("RGB")), (w, h), interpolation=cv2.INTER_NEAREST) def refine_prompt(scene: str) -> str: """klein 低强度 SDEdit 补细节(f0gen 的 refine 阶段)。""" sc = SCENES[scene] return (f"Make image 1 a highly detailed, polished screenshot of a high-end isometric action RPG set in " f"{sc['title']}: add rich realistic textures and fine material detail to the ground, floors, buildings, " "rocks, plants and objects, and crisp detail to the characters. Keep exactly the same composition, layout, " "objects, characters, poses, camera, colours and lighting; do not add or remove anything; no text, no UI, " "no grid lines, no border.") def harmonize_prompt(scene: str, n_chars: int) -> str: """realchar 的融合编辑:真实角色已贴进生成背景,只修边缘 / 影子 / 光照,不改角色设计。""" sc = SCENES[scene] who = "the two characters" if n_chars == 2 else "the character" return (f"Picture 1 is a close-up crop of a video game screenshot of {sc['title']}; {who} in the middle were pasted " "in from another level. Blend them naturally into this scene: replace any leftover patches or halos of the " "other level's ground around them with the surrounding ground of this scene, add soft natural contact " f"shadows under them that match the scene lighting ({sc['light'].rstrip('.')}), and gently adjust the " f"lighting and colour tint on {who} to match the scene. Keep {who} exactly the same design, pose, size, " "position, armour, colours and weapons; do not add, remove or move anything else.") # ------------------------------------------------------------------ 角色框 / 贴回 ---- def char_box(m: dict, margin: float = 0.6, min_side: int = 160) -> tuple[int, int, int, int] | None: """所有可见角色的并集框(832×480 坐标 x0,y0,x1,y1),每边外扩 max(24, margin·边长),至少 min_side,夹在画面内。""" mk = m["chars"]["player"] | m["chars"]["boss"] if mk.sum() < MIN_PX: return None ys, xs = np.nonzero(mk) x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 bw, bh = x1 - x0, y1 - y0 px, py = max(24, margin * bw), max(24, margin * bh) cx, cy = (x0 + x1) / 2, (y0 + y1) / 2 hw, hh = max(bw / 2 + px, min_side / 2), max(bh / 2 + py, min_side / 2) x0, x1 = int(max(0, cx - hw)), int(min(W, cx + hw)) y0, y1 = int(max(0, cy - hh)), int(min(H, cy + hh)) return x0, y0, x1, y1 def paste_mask(m: dict, dilate: int = 14, feather: int = 6) -> np.ndarray: """角色并集掩码膨胀 dilate px + 高斯羽化 → float32 [H,W] ∈ [0,1](角色修补只换掩码内)。""" import cv2 mk = (m["chars"]["player"] | m["chars"]["boss"]).astype(np.uint8) k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * dilate + 1, 2 * dilate + 1)) a = cv2.dilate(mk, k).astype(np.float32) if feather: a = cv2.GaussianBlur(a, (0, 0), feather) a = np.maximum(a, mk.astype(np.float32)) return np.clip(a, 0, 1) def area_down(img: np.ndarray, w: int = W, h: int = H) -> np.ndarray: import cv2 img = np.asarray(img) if img.shape[:2] == (h, w): return img return cv2.resize(img, (w, h), interpolation=cv2.INTER_AREA if img.shape[1] > w else cv2.INTER_LANCZOS4) # ------------------------------------------------------------------ 人偶检测(规则) ---- def dummy_frac(img: np.ndarray, m: dict, tol: float = 40.0, smooth: float = 5.0) -> dict: """首帧(832×480)里每个可见角色掩码(腐蚀 1 px)内「人偶样」像素的比例:颜色离体块图同位置像素(浅蓝 / 浅鲑红人偶, 带明暗)、平涂色图(玩家蓝 / Boss 红 / 武器白)、剪影图(深蓝 / 深红)任一 RGB 距离 < tol,**且**局部平滑(5×5 灰度 标准差 < smooth;真角色有盔甲 / 皮毛的细节,照抄的人偶是光滑的)。≈ 0 = 真角色,高 = 照抄了人偶 / 火柴人 / 标记色小人。 → {"player": f, "boss": f}(不可见的角色不出现)。""" import cv2 img = area_down(np.asarray(img)).astype(np.float32) g = img.mean(-1) mu = cv2.blur(g, (5, 5)) sd = np.sqrt(np.maximum(cv2.blur(g * g, (5, 5)) - mu * mu, 0)) refs = [m["shaded"], m["flat"], m["sil"]] out = {} for k in ("player", "boss"): mk = cv2.erode(m["chars"][k].astype(np.uint8), np.ones((3, 3), np.uint8)).astype(bool) if mk.sum() < MIN_PX: continue x = img[mk] d = np.min([np.linalg.norm(x - r[mk].astype(np.float32), axis=-1) for r in refs], 0) out[k] = round(float(((d < tol) & (sd[mk] < smooth)).mean()), 4) return out DUMMY_FRAC = 0.35 # dummy_frac 任一角色 ≥ 这个 ⇒ 判为人偶(f0gen --detect rule) # ------------------------------------------------------------------ VLM 题(Qwen3-VL-8B,P(yes)) ---- QWEN8 = ("/opt/dlami/nvme/zhiyangdeng/PCE/weights/hf_hub/models--Qwen--Qwen3-VL-8B-Instruct/snapshots/" "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b") VLM_Q = { "scene": "Is this video game screenshot clearly set in {vlm}? Answer Yes or No.", "clean": "Is this image a clean game screenshot with no text, letters, numbers, logos, watermark, user interface, " "grid lines drawn over the ground, or frame border? Answer Yes or No.", "nodummy": "Look at the whole image. Is it free of any smooth featureless capsule-shaped mannequins, dummies or " "toy-like placeholder figures? Answer Yes or No.", "alive": "This is a close-up of a video game screenshot. Is there, at the centre of the image, a living video game " "character ({who}) rendered with detailed armour, skin, fur or clothing, and not a smooth mannequin, " "dummy, statue, rock, plant or object? Answer Yes or No.", } def zoom(img: Image.Image, mask: np.ndarray, min_side: int = 448) -> Image.Image: """mask(832×480)的外接框外扩 1/3 → 从 img(任意尺寸,同长宽比)裁出、放大到短边 ≥ min_side。""" ys, xs = np.nonzero(mask) x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 pw, ph = max(16, (x1 - x0) // 3), max(16, (y1 - y0) // 3) sx, sy = img.width / W, img.height / H box = (int(max(0, x0 - pw) * sx), int(max(0, y0 - ph) * sy), int(min(W, x1 + pw) * sx), int(min(H, y1 + ph) * sy)) c = img.crop(box) s = max(1.0, min_side / max(1, min(c.size))) return c.resize((int(c.width * s), int(c.height * s)), Image.LANCZOS) def vlm_items(img: Image.Image, m: dict, scene: str | None, keys=("scene", "clean", "nodummy", "alive")): """→ [(images, text)], [key]:alive 拆成 alive_p / alive_b(帧 0 里没有的角色不问)。""" batch, ks = [], [] for k in keys: if k == "alive": for kk, who, mk in (("alive_p", "the hero", m["chars"]["player"]), ("alive_b", "a monster", m["chars"]["boss"])): if mk.sum() >= MIN_PX: batch.append(([zoom(img, mk)], VLM_Q["alive"].format(who=who))) ks.append(kk) elif k == "scene": if scene: batch.append(([img], VLM_Q["scene"].format(vlm=SCENES[scene]["vlm"]))) ks.append(k) else: batch.append(([img], VLM_Q[k])) ks.append(k) return batch, ks # ------------------------------------------------------------------ 拼图 ---- def label(img: np.ndarray, text: str, scale: float = 0.5) -> np.ndarray: import cv2 out = np.ascontiguousarray(img.copy()) (tw, th), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, scale, 1) cv2.rectangle(out, (0, 0), (tw + 8, th + 10), (0, 0, 0), -1) cv2.putText(out, text, (4, th + 4), cv2.FONT_HERSHEY_SIMPLEX, scale, (255, 255, 255), 1, cv2.LINE_AA) return out def grid(tiles: list[np.ndarray], ncol: int, tile_w: int = 416) -> np.ndarray: import cv2 th = round(tile_w * H / W) ts = [cv2.resize(t, (tile_w, th), interpolation=cv2.INTER_AREA) for t in tiles] while len(ts) % ncol: ts.append(np.zeros_like(ts[0])) return np.concatenate([np.concatenate(ts[i:i + ncol], 1) for i in range(0, len(ts), ncol)], 0)