| """teaser · 首帧生成的共用件（纯 numpy / cv2 / PIL，img env 与 maliang venv 都能 import）：读地图、派生输入图、拼提示词、 |
| 角色框 / 贴回、对比拼图。生成后端在 f0gen.py，扫参在 f0sweep.py。 |
| |
| 地图目录（procmap 的输出，outputs/genbench/procmap/<case>/、outputs/teaser/maps/<case>/、outputs/teaser/f0_sweep/maps/<case>/）： |
| ids.npz（[81,480,832] parts raw id，取帧 0）、map0.png（色图）、map0_shaded.png（按法线打光的体块）、depth0.png（16 bit，近亮）。 |
| 派生（load_map 里一次算好，键见 load_map 的 docstring）： |
| * flat ：firstframe_flux.flux_map 的 "white" 色表（静态类 = geometry 色，玩家蓝 / Boss 红 / 武器白），**不画铺地网格** |
| （v4 的 16 px 网格提示会让首帧和视频里出现网格线）； |
| * shaded ：map0_shaded 原样（玩家浅蓝、Boss 浅鲑红的胶囊人体 = 「人偶」，qwen21 会照画成人偶 ⇒ 提示词说是占位符 + 角色修补）； |
| * sil ：shaded 但角色换成深蓝 / 深红实色（保留明暗）； |
| * depth8 ：depth0 → 8 bit 灰度 RGB；edge：语义类边界（角色部件合并）白线黑底，2 px。 |
| |
| 提示词（scene_prompt）：只写帧 0 里出现的类（像素 ≥ MIN_PX），「色 = 该场景里的这一类」（scenes.SCENES[s]["cls"]）；没出现的 |
| 建筑 / 石头 / 植被写「这一带没有、不许加」（genbench4 edit2 的经验）；植被 / 道具附形状提示（体块图里的圆球 = 俯视的树冠）； |
| 角色句只写帧 0 里可见的（Boss 不可见就只有玩家）；结尾禁文字 / UI / 网格 / 边框、禁沿画面边缘加墙。 |
| 画风：场景键可写 "<场景>@<画风>"（scenes.SCENES 按需派生，title / light 带画风句）；char_style / texture_phrase 让角色描述与 |
| 纹理句跟着画风走；restyle_prompt = f0gen 的整图画风重绘提示词（photoreal / 无画风 ⇒ None）。 |
| 其它：scene_blockout（体块图按场景色表重新上色，scb* 变体）、dummy_frac（人偶检测规则：角色掩码里「标记色且光滑」的比例）、 |
| charfix_prompt / refine_prompt、VLM_Q + vlm_items（Qwen3-VL-8B 题）、pixelate（可选像素化后处理）、label / grid（拼图）； |
| nochar_map / real_chars / soft_mask / mask_box / tint_match / harmonize_prompt 是实例 R（realchar）加的。 |
| """ |
| from __future__ import annotations |
|
|
| import glob |
| import json |
| import os |
| import sys |
|
|
| import numpy as np |
| from PIL import Image |
|
|
| M = "/opt/dlami/nvme/zhiyangdeng/maliang" |
| for _p in (f"{M}/code/gc25d", f"{M}/scripts/teaser"): |
| if _p not in sys.path: |
| sys.path.insert(0, _p) |
| import firstframe_flux as FF |
| from scenes import SCENES |
|
|
| W, H = 832, 480 |
| PLAYER, BOSS = (10, 12, 13, 14, 15, 16), (11, 17, 18, 19, 20, 21) |
| MIN_PX = FF.MIN_PX |
| MAP_ROOTS = (f"{M}/outputs/genbench/procmap", f"{M}/outputs/teaser/maps", f"{M}/outputs/teaser/f0_sweep/maps") |
|
|
| |
| COLOR_NAME = {0: "sky blue", 1: "brown", 2: "light grey", 3: "orange-red", 4: "dark grey", 5: "green", 6: "yellow"} |
| COLOR_NAME_SHP = {**COLOR_NAME, 6: "the small separate objects"} |
| SHAPE_HINT = {5: " (round green shapes are tree crowns and bushes seen from above, thin green posts are small trunks or " |
| "stems)", |
| 6: " (small separate objects)", 4: " (grey lumps and boulders)"} |
| SCB_SUBJ = {0: "the empty background", 1: "the flat open ground areas", 2: "the flat paved floors and paths", |
| 3: "the boxy buildings and walls", 4: "the lumpy grey boulders", 5: "the round crowns and posts (plants)", |
| 6: "the small separate objects"} |
| ABSENT = {3: "buildings, walls, towers, gates, huts or fences", 4: "rocks or boulders", 5: "trees or plants"} |
|
|
|
|
| |
| def find_map(case: str) -> str: |
| for r in MAP_ROOTS: |
| if os.path.isfile(f"{r}/{case}/ids.npz"): |
| return f"{r}/{case}" |
| raise FileNotFoundError(f"找不到地图 {case}（查过 {MAP_ROOTS}）") |
|
|
|
|
| def list_maps(roots=MAP_ROOTS) -> list[str]: |
| """所有齐全的地图目录（ids.npz + map0_shaded.png + depth0.png）。""" |
| out = [] |
| for r in roots: |
| for d in sorted(glob.glob(f"{r}/*/")): |
| d = d.rstrip("/") |
| if all(os.path.isfile(f"{d}/{f}") for f in ("ids.npz", "map0_shaded.png", "depth0.png")): |
| out.append(d) |
| return out |
|
|
|
|
| def load_ids0(path: str, frame: int = 0) -> np.ndarray: |
| """ids.npz（[F,H,W]）/ .npy / id png / 色图 png → parts raw id [H,W]（firstframe_flux.load_ids 的超集）。""" |
| if path.endswith(".npy"): |
| a = np.load(path) |
| return np.ascontiguousarray(a[frame] if a.ndim == 3 else a).astype(np.uint8) |
| return FF.load_ids(path, frame) |
|
|
|
|
| def char_masks(ids: np.ndarray) -> dict: |
| p, b = np.isin(ids, PLAYER), np.isin(ids, BOSS) |
| return {"player": p, "boss": b, "has_p": int(p.sum()) >= MIN_PX, "has_b": int(b.sum()) >= MIN_PX, |
| "player_px": int(p.sum()), "boss_px": int(b.sum()), "boss_weapon": int((ids == 21).sum()) >= 20, |
| "player_weapon": int((ids == 16).sum()) >= 10} |
|
|
|
|
| def sem_edges(ids: np.ndarray, width: int = 2) -> np.ndarray: |
| import cv2 |
| c = ids.copy() |
| c[np.isin(c, PLAYER)] = 10 |
| c[np.isin(c, BOSS)] = 11 |
| e = np.zeros(c.shape, np.uint8) |
| e[:, 1:] |= c[:, 1:] != c[:, :-1] |
| e[1:, :] |= c[1:, :] != c[:-1, :] |
| e = cv2.dilate(e * 255, np.ones((width, width), np.uint8)) |
| return np.repeat(e[..., None], 3, -1) |
|
|
|
|
| def recolor_chars(rgb: np.ndarray, ids: np.ndarray, player=(40, 70, 230), boss=(200, 30, 30)) -> np.ndarray: |
| """角色像素换成实色（乘原图亮度 / 200 保留明暗）。""" |
| out = rgb.copy() |
| lum = rgb.astype(np.float32).mean(-1, keepdims=True) / 200.0 |
| for m, c in ((np.isin(ids, PLAYER), player), (np.isin(ids, BOSS), boss)): |
| out[m] = np.clip(np.asarray(c, np.float32) * lum[m], 0, 255).astype(np.uint8) |
| return out |
|
|
|
|
| PALETTE_JSON = f"{M}/outputs/teaser/f0_sweep/palette.json" |
| _PAL: dict = {} |
| SCB_MIN_LUM = 70.0 |
|
|
|
|
| def prop_blockout(m: dict, scene: str) -> np.ndarray: |
| """体块图只把道具（类 6）按场景色表重新上色（t1：qwen21 把黄道具原样画成黄方块，其余类照抄颜色问题小）；场景没有色表 |
| ⇒ 原样。shp 变体用，提示词里道具那一项不写颜色名（COLOR_NAME_SHP）。""" |
| if not _PAL and os.path.exists(PALETTE_JSON): |
| _PAL.update(json.load(open(PALETTE_JSON))) |
| pal = _PAL.get(base_of(scene), {}) |
| if "6" not in pal: |
| return m["shaded"] |
| ids = m["ids"] |
| mk = ids == 6 |
| out = m["shaded"].copy() |
| if mk.any(): |
| f = np.clip(m["shaded"].astype(np.float32).mean(-1)[mk] / float(np.mean(FF.GEO_COLORS[6])), 0.3, 1.6)[:, None] |
| col = np.asarray(pal["6"], np.float32) + 4.0 |
| col = col * max(1.0, SCB_MIN_LUM / float(col.mean())) |
| out[mk] = np.clip(np.minimum(col, 255)[None] * f, 0, 255).astype(np.uint8) |
| return out |
|
|
|
|
| def scene_blockout(m: dict, scene: str, mix: float = 1.0) -> np.ndarray: |
| """体块图按场景色表重新上色：静态类像素 = 场景里该类的代表色 × 体块图的明暗（体块图亮度 / 该类 geometry 色亮度， |
| 夹在 [0.3, 1.6]）；角色保持原来的浅蓝 / 浅鲑红人偶。t1：qwen21 会把体块图的黄 / 绿原样画出来（墓碑成黄方块）， |
| 先换成场景色 ⇒ 照抄颜色也是对的颜色。""" |
| if not _PAL: |
| _PAL.update(json.load(open(PALETTE_JSON))) |
| pal = _PAL[base_of(scene)] |
| ids, sh = m["ids"], m["shaded"].astype(np.float32) |
| lum = sh.mean(-1) |
| out = m["shaded"].copy() |
| for c in range(7): |
| mk = ids == c |
| if not mk.any(): |
| continue |
| base = float(np.mean(FF.GEO_COLORS[c])) |
| f = np.clip(lum[mk] / base, 0.3, 1.6)[:, None] |
| col = np.asarray(pal[str(c)], np.float32) + 4.0 |
| col = col * max(1.0, SCB_MIN_LUM / float(col.mean())) |
| col = mix * col + (1 - mix) * np.asarray(FF.GEO_COLORS[c], np.float32) |
| out[mk] = np.clip(np.minimum(col, 255)[None] * f, 0, 255).astype(np.uint8) |
| return out |
|
|
|
|
| def load_map(map_dir: str) -> dict: |
| """→ {dir, case, ids [H,W], map0, shaded, flat, sil, depth8, edge（都是 uint8 [H,W,3]）, chars（char_masks）, present, |
| n（类像素计数 [256]）, source（source.json 或 {}）}。""" |
| import cv2 |
| ids = load_ids0(f"{map_dir}/ids.npz") |
| assert ids.shape == (H, W), ids.shape |
| rd = lambda f: np.asarray(Image.open(f"{map_dir}/{f}").convert("RGB")) |
| d16 = cv2.imread(f"{map_dir}/depth0.png", cv2.IMREAD_UNCHANGED) |
| d8 = (d16.astype(np.float32) / max(1, int(d16.max())) * 255).round().astype(np.uint8) if d16.dtype == np.uint16 \ |
| else d16.astype(np.uint8) |
| if d8.ndim == 3: |
| d8 = d8[..., 0] |
| shaded = rd("map0_shaded.png") |
| n = np.bincount(ids.reshape(-1), minlength=256) |
| src = {} |
| if os.path.isfile(f"{map_dir}/source.json"): |
| src = json.load(open(f"{map_dir}/source.json")) |
| return {"dir": map_dir, "case": os.path.basename(map_dir.rstrip("/")), "ids": ids, "map0": rd("map0.png"), |
| "shaded": shaded, "flat": FF.flux_map(ids, "white", tiles=False), "sil": recolor_chars(shaded, ids), |
| "depth8": np.repeat(d8[..., None], 3, -1), "edge": sem_edges(ids), "chars": char_masks(ids), |
| "present": [c for c in range(7) if n[c] >= MIN_PX], "n": n, "source": src} |
|
|
|
|
| def nochar_map(m: dict) -> dict: |
| """realchar 路线的第一阶段输入：把角色从地图里去掉 —— ids 的角色像素换成最近的静态类（distance_transform 最近邻）， |
| 体块图 / 色图在（膨胀 4 px 的）角色区域里 cv2.inpaint（Telea）补成周围的地面 ⇒ 生成器只画背景，角色之后从源 clip 真实帧 |
| 贴回（real_chars / f0gen.realchar）。返回新 dict（chars 全空，提示词自动写「没有角色」），原 dict 另存在 m['orig']。""" |
| import cv2 |
| from scipy import ndimage |
| ids = m["ids"] |
| ch = np.isin(ids, PLAYER + BOSS) |
| _, (iy, ix) = ndimage.distance_transform_edt(ch, return_indices=True) |
| ids2 = ids[iy, ix].astype(np.uint8) |
| k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9, 9)) |
| mk = cv2.dilate(ch.astype(np.uint8), k) |
| out = dict(m) |
| out.update(ids=ids2, orig=m, chars=char_masks(ids2), flat=FF.flux_map(ids2, "white", tiles=False), |
| shaded=cv2.inpaint(np.ascontiguousarray(m["shaded"]), mk, 5, cv2.INPAINT_TELEA), |
| sil=cv2.inpaint(np.ascontiguousarray(m["shaded"]), mk, 5, cv2.INPAINT_TELEA), |
| edge=sem_edges(ids2)) |
| n = np.bincount(ids2.reshape(-1), minlength=256) |
| out.update(n=n, present=[c for c in range(7) if n[c] >= MIN_PX]) |
| return out |
|
|
|
|
| def real_chars(m: dict) -> dict | None: |
| """源 clip 的真实帧 0 + 真实 ids 帧 0（procmap 借的是源 clip 的相机 + 胶囊 ⇒ 角色在同一像素位置，实测 10 张地图里 |
| 9 张 IoU = 1.00，plaza P9C 0.83 = 真实帧里玩家被场景挡了一块）→ {rgb, ids, mask（玩家 ∪ Boss）, clip_id}； |
| 地图没有 source.json / clip_id 时返回 None。""" |
| cid = (m.get("source") or {}).get("clip_id") |
| if not cid: |
| return None |
| rgb = FF.clip_frame(cid) |
| rid = FF.clip_ids(cid) |
| return {"rgb": rgb, "ids": rid, "mask": np.isin(rid, PLAYER + BOSS), "clip_id": cid} |
|
|
|
|
| def soft_mask(mask: np.ndarray, dilate: int, feather: float) -> np.ndarray: |
| """bool 掩码 → 膨胀 dilate px（≤ 0 = 腐蚀）+ 高斯羽化 → float32 ∈ [0,1]（羽化不往掩码内缩：max(羽化, 硬掩码)）。""" |
| import cv2 |
| mk = mask.astype(np.uint8) |
| if dilate > 0: |
| mk = cv2.dilate(mk, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * dilate + 1, 2 * dilate + 1))) |
| elif dilate < 0: |
| mk = cv2.erode(mk, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (-2 * dilate + 1, -2 * dilate + 1))) |
| a = mk.astype(np.float32) |
| if feather > 0: |
| a = np.maximum(cv2.GaussianBlur(a, (0, 0), feather), a) |
| return np.clip(a, 0, 1) |
|
|
|
|
| def mask_box(mask: np.ndarray, margin: float = 0.6, min_side: int = 160, pad: int = 24): |
| """bool 掩码的外接框（x0,y0,x1,y1），每边外扩 max(pad, margin·边长)、至少 min_side，夹在画面内。""" |
| if mask.sum() < MIN_PX: |
| return None |
| ys, xs = np.nonzero(mask) |
| x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 |
| bw, bh = x1 - x0, y1 - y0 |
| cx, cy = (x0 + x1) / 2, (y0 + y1) / 2 |
| hw = max(bw / 2 + max(pad, margin * bw), min_side / 2) |
| hh = max(bh / 2 + max(pad, margin * bh), min_side / 2) |
| return int(max(0, cx - hw)), int(max(0, cy - hh)), int(min(W, cx + hw)), int(min(H, cy + hh)) |
|
|
|
|
| def tint_match(fg: np.ndarray, bg_new: np.ndarray, bg_old: np.ndarray, ring: np.ndarray, alpha: float = 0.5): |
| """把真实角色的整体色调往新场景拉：增益 = 新背景环带均值 / 源背景环带均值（逐通道，夹在 [0.55, 1.3]），按 alpha 混合。""" |
| if ring.sum() < 50: |
| return fg |
| g = (bg_new[ring].astype(np.float32).mean(0) + 8) / (bg_old[ring].astype(np.float32).mean(0) + 8) |
| g = np.clip(g, 0.55, 1.3) |
| g = 1 + alpha * (g - 1) |
| return np.clip(fg.astype(np.float32) * g, 0, 255).astype(np.uint8) |
|
|
|
|
| |
| def look_of(scene: str) -> str | None: |
| """"<场景>@<画风>" → 画风键；纯场景名 → None（= 默认写实提示词）。""" |
| return SCENES[scene].get("look") |
|
|
|
|
| def base_of(scene: str) -> str: |
| return scene.split("@", 1)[0] |
|
|
|
|
| def with_look(scene: str, look: str | None) -> str: |
| """job 行的 look 键并进场景键（scenes.SCENES 认 "<场景>@<画风>"）；已带 @ 或 look 为空就原样。""" |
| return scene if (not look or "@" in scene) else f"{scene}@{look}" |
|
|
|
|
| def char_style(scene: str | None) -> str: |
| """角色的画风修饰（LOOKS[look]["char"]，如 "pixel-art" / "stylized cartoon"）；无画风 = 空。""" |
| lk = look_of(scene) if scene else None |
| if not lk: |
| return "" |
| import scenes as _SC |
| return _SC.LOOKS[lk]["char"] + " " |
|
|
|
|
| def texture_phrase(scene: str | None) -> str: |
| lk = look_of(scene) if scene else None |
| if not lk or lk == "photoreal": |
| return "detailed realistic textures" |
| import scenes as _SC |
| return f"detailed {_SC.LOOKS[lk]['short']} textures" |
|
|
|
|
| |
| def legend(m: dict, scene: str, color_name: dict = COLOR_NAME, hints: bool = True) -> str: |
| sc = SCENES[scene] |
| parts = [f"{color_name[c]} = {sc['cls'][c]}" + (SHAPE_HINT.get(c, "") if hints else "") for c in m["present"]] |
| return "; ".join(parts) |
|
|
|
|
| def absent_text(m: dict) -> str: |
| miss = [ABSENT[c] for c in ABSENT if c not in m["present"]] |
| return (" This part of the level contains no " + ", no ".join(miss) + "; do not add any.") if miss else "" |
|
|
|
|
| def char_text(m: dict, style: str, scene: str | None = None) -> str: |
| """style: placeholder（体块图里的胶囊人偶 = 占位符）| silhouette（深蓝 / 深红实色剪影）| marker（flat 色图的蓝 / 红人形）。 |
| scene 带画风时角色描述加 LOOKS[look]["char"]（像素 / 卡通画风下角色也是像素 / 卡通的）。""" |
| ch = m["chars"] |
| cs = char_style(scene) |
| hero = f"a fully detailed {cs}armored human warrior hero" + (" holding a steel sword" if ch["player_weapon"] else "") |
| boss = f"a huge, fully detailed {cs}monster boss" + (" wielding a big weapon" if ch["boss_weapon"] else "") |
| who = {"placeholder": ("the light-blue mannequin", "the salmon-pink mannequin"), |
| "silhouette": ("the dark blue silhouette", "the dark red silhouette"), |
| "marker": ("the blue humanoid figure", "the red humanoid figure")}[style] |
| items = ([f"{who[0]} becomes the player character, {hero}"] if ch["has_p"] else []) + \ |
| ([f"{who[1]} becomes the boss, {boss}"] if ch["has_b"] else []) |
| if not items: |
| return " There are no characters in this scene: do not draw any people, creatures, figures or statues." |
| n = len(items) |
| lead = {"placeholder": " The smooth capsule mannequins are only placeholders and must not appear in the final image: " |
| if n == 2 else " The smooth capsule mannequin is only a placeholder and must not appear in the final image: ", |
| "silhouette": " ", "marker": " "}[style] |
| txt = lead + "; ".join(items) + ". " |
| txt += ("Each character has exactly the position, pose, size, facing direction and limb directions of its " |
| "placeholder, and looks like a living video game character with natural colours and materials, not a " |
| "mannequin, statue or toy. ") |
| txt += "These are the only two characters; " if n == 2 else "This is the only character; " |
| txt += "do not add any other people or creatures." |
| return txt |
|
|
|
|
| QUALITY = (" {light} Bright, clear and readable game lighting: every object is easy to see. High-end isometric action RPG " |
| "graphics with {tex} and soft shadows; no text, " |
| "no letters, no UI, no grid lines, no outlines, no border. Do not frame the scene with new walls, cliffs, " |
| "trees or buildings along the image borders; the borders show the same kind of region as the blockout.") |
|
|
|
|
| NOCOPY = (" The colours of picture 1 are only a code that says what each region is: do not copy them; give every " |
| "region the natural colours and materials it has in {title}.") |
|
|
|
|
| def scene_prompt(m: dict, scene: str, kind: str = "blockout", chars: str = "placeholder", nocopy: bool = False) -> str: |
| """kind: blockout（体块图 / 剪影图：qwen21 / qie2511）| flatmap（平涂色图：klein / qie）。 |
| nocopy：加 NOCOPY 句（t1：qwen21 常把体块图的黄 / 绿原样画出来 —— 墓地的墓碑成黄方块、糖果的树成绿球）。""" |
| sc = SCENES[scene] |
| if kind == "scb": |
| head = (f"Picture 1 is a simple untextured 3D blockout of one level of an isometric action RPG video game, " |
| f"already painted in the colours of {sc['title']}. Render it as the finished, fully detailed in-game " |
| "screenshot, seen from exactly the same high top-down camera. Keep every region, object and figure " |
| "exactly where it is, with the same outline, size and shape, and add nothing that is not in picture " |
| "1: open ground and floors stay open and are only textured. In picture 1: " + |
| "; ".join(f"{SCB_SUBJ[c]} are {sc['cls'][c]}" for c in m["present"]) + ".") |
| return head + absent_text(m) + char_text(m, chars, scene) + \ |
| QUALITY.format(light=sc["light"], tex=texture_phrase(scene)) |
| if kind == "scb2": |
| head = (f"Picture 1 is a simple untextured 3D blockout of one level of an isometric action RPG video game, " |
| f"already painted in the colours of {sc['title']}; picture 2 is the same blockout colour-coded by " |
| f"region type ({legend(m, scene)}). Render picture 1 as the finished, fully detailed in-game screenshot " |
| "of " + sc["title"] + ", seen from exactly the same high top-down camera. Keep every region, object " |
| "and figure exactly where it is, with the same outline, size and shape, and add nothing that is not " |
| "in the blockout: open ground and floors stay open and are only textured.") |
| return head + absent_text(m) + char_text(m, chars, scene) + \ |
| QUALITY.format(light=sc["light"], tex=texture_phrase(scene)) |
| what = {"blockout": "a colour-coded 3D blockout", "shp": "a colour-coded 3D blockout", |
| "flatmap": "a flat colour-coded layout map"}[kind] |
| head = (f"Picture 1 is {what} of one level of an isometric action RPG video game. Render it as the finished in-game " |
| f"screenshot of {sc['title']}, seen from exactly the same high top-down camera. Keep every region, object " |
| f"and figure exactly where it is, with the same outline, size and shape, and add nothing that is not in " |
| f"picture 1: open ground and floors stay open and are only textured. Colour code: " |
| f"{legend(m, scene, COLOR_NAME_SHP if kind == 'shp' else COLOR_NAME)}.") |
| return head + (NOCOPY.format(title=sc["title"]) if nocopy else "") + absent_text(m) + char_text(m, chars, scene) + \ |
| QUALITY.format(light=sc["light"], tex=texture_phrase(scene)) |
|
|
|
|
| def klein_prompt(m: dict, scene: str) -> str: |
| """firstframe_flux.map2frame v3 模板（无网格）+ 场景色表 + 场景句（genbench4 make_scene 的 map 模式改进版）。""" |
| sc = SCENES[scene] |
| txt = ("Turn image 1, a flat color-coded layout map, into a finished screenshot of a 3D action RPG video game seen " |
| "from a high top-down isometric camera. Follow the layout of image 1 exactly: every region, object and " |
| "character keeps its exact position, shape and size, and nothing is added that is not in the map; open " |
| f"ground and floors stay open. Color legend of the layout map: {legend(m, scene)}.") |
| txt += absent_text(m) + char_text(m, "marker", scene) |
| txt += (" Replace every flat color with detailed textured materials, natural lighting and soft shadows; no flat " |
| f"colors, no outlines, no grid lines, no text, no UI, no border. Scene: the whole level is {sc['title']}. " |
| f"{sc['light']}") |
| return txt |
|
|
|
|
| def zimage_prompt(m: dict, scene: str) -> str: |
| """Z-Image（ControlNet 只给结构）：整段场景描述 + 按面积从大到小列出各类在该场景里是什么 + 角色。""" |
| sc = SCENES[scene] |
| order = sorted(m["present"], key=lambda c: -m["n"][c]) |
| ch = m["chars"] |
| chars = ("an armored human warrior hero with a sword fighting a huge monster boss" if ch["has_p"] and ch["has_b"] |
| else "a single armored human warrior hero with a sword" if ch["has_p"] else "no characters") |
| return (f"A screenshot of a high-end isometric action RPG video game seen from a high top-down camera: {sc['title']}. " |
| f"The ground is {sc['cls'][order[0]]}; the level also has " + ", ".join(sc["cls"][c] for c in order[1:]) + |
| f". In the scene: {chars}, exactly two figures at most. {sc['light']} Detailed realistic textures, soft " |
| "shadows, polished game graphics, no text, no UI, no grid lines, no border.") |
|
|
|
|
| def charfix_prompt(m: dict, scene: str) -> str: |
| ch = m["chars"] |
| items = [] |
| if ch["has_p"]: |
| items.append(f"the light-blue mannequin (or the blue figure) into a fully detailed {char_style(scene)}armored human " |
| "warrior hero" + (" holding a steel sword" if ch["player_weapon"] else "")) |
| if ch["has_b"]: |
| items.append("the salmon-pink mannequin (or the red figure)" + (" and its pink club" if ch["boss_weapon"] else "") + |
| f" into a huge, fully detailed {char_style(scene)}monster boss" + |
| (" wielding a big weapon" if ch["boss_weapon"] else "")) |
| return ("Picture 1 is a close-up crop of a video game screenshot of " + SCENES[scene]["title"] + ". Edit picture 1: " |
| "turn " + ", and turn ".join(items) + ", each in exactly the same position, pose, size and facing " |
| "direction. They must look like living video game characters with natural colours and materials that fit " |
| "the scene, not mannequins, statues or toys. If a character already looks fully detailed, keep it as it " |
| "is. Keep everything else in the image exactly unchanged; do not add any other characters or objects.") |
|
|
|
|
| def restyle_prompt(scene: str) -> str | None: |
| """画风重绘（f0gen 的 restyle 阶段，qwen21 整图编辑）：t4 冒烟 —— 场景提示词里写画风，qwen21 / klein 都还是画成 3D 渲染 |
| （体块图本身是 3D 渲染的样子）；先按写实生成再整图「换画风、布局不动」。look 为空 / photoreal ⇒ None（不重绘）。""" |
| lk = look_of(scene) |
| if not lk or lk == "photoreal": |
| return None |
| import scenes as _SC |
| L_ = _SC.LOOKS[lk] |
| return (f"Redraw picture 1 completely in this art style: {L_['prompt']} Every surface, object and character must be " |
| f"in the {L_['short']} style, nothing may stay photorealistic. Keep exactly the same composition, camera, " |
| "layout and every object, with the same positions, outlines and sizes; keep both characters with the same " |
| f"positions, poses, sizes and weapons, drawn as {L_['char']} game characters. Do not add or remove anything; " |
| "no text, no UI, no border.") |
|
|
|
|
| def pixelate(img: np.ndarray, cell: int = 4, colors: int = 48) -> np.ndarray: |
| """确定性的像素画后处理（look = pixel）：INTER_AREA 缩到 1/cell → PIL 中位切分量化到 colors 色（不抖动）→ 最近邻放回。 |
| t4c：qwen21 画风重绘的 pixel 只有一点点像素感，加这一步才是真正的方块像素。""" |
| import cv2 |
| h, w = img.shape[:2] |
| sm = cv2.resize(img, (w // cell, h // cell), interpolation=cv2.INTER_AREA) |
| q = Image.fromarray(sm).quantize(colors=colors, method=Image.Quantize.MEDIANCUT, dither=Image.Dither.NONE) |
| return cv2.resize(np.asarray(q.convert("RGB")), (w, h), interpolation=cv2.INTER_NEAREST) |
|
|
|
|
| def refine_prompt(scene: str) -> str: |
| """klein 低强度 SDEdit 补细节（f0gen 的 refine 阶段）。""" |
| sc = SCENES[scene] |
| return (f"Make image 1 a highly detailed, polished screenshot of a high-end isometric action RPG set in " |
| f"{sc['title']}: add rich realistic textures and fine material detail to the ground, floors, buildings, " |
| "rocks, plants and objects, and crisp detail to the characters. Keep exactly the same composition, layout, " |
| "objects, characters, poses, camera, colours and lighting; do not add or remove anything; no text, no UI, " |
| "no grid lines, no border.") |
|
|
|
|
| def harmonize_prompt(scene: str, n_chars: int) -> str: |
| """realchar 的融合编辑：真实角色已贴进生成背景，只修边缘 / 影子 / 光照，不改角色设计。""" |
| sc = SCENES[scene] |
| who = "the two characters" if n_chars == 2 else "the character" |
| return (f"Picture 1 is a close-up crop of a video game screenshot of {sc['title']}; {who} in the middle were pasted " |
| "in from another level. Blend them naturally into this scene: replace any leftover patches or halos of the " |
| "other level's ground around them with the surrounding ground of this scene, add soft natural contact " |
| f"shadows under them that match the scene lighting ({sc['light'].rstrip('.')}), and gently adjust the " |
| f"lighting and colour tint on {who} to match the scene. Keep {who} exactly the same design, pose, size, " |
| "position, armour, colours and weapons; do not add, remove or move anything else.") |
|
|
|
|
| |
| def char_box(m: dict, margin: float = 0.6, min_side: int = 160) -> tuple[int, int, int, int] | None: |
| """所有可见角色的并集框（832×480 坐标 x0,y0,x1,y1），每边外扩 max(24, margin·边长)，至少 min_side，夹在画面内。""" |
| mk = m["chars"]["player"] | m["chars"]["boss"] |
| if mk.sum() < MIN_PX: |
| return None |
| ys, xs = np.nonzero(mk) |
| x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 |
| bw, bh = x1 - x0, y1 - y0 |
| px, py = max(24, margin * bw), max(24, margin * bh) |
| cx, cy = (x0 + x1) / 2, (y0 + y1) / 2 |
| hw, hh = max(bw / 2 + px, min_side / 2), max(bh / 2 + py, min_side / 2) |
| x0, x1 = int(max(0, cx - hw)), int(min(W, cx + hw)) |
| y0, y1 = int(max(0, cy - hh)), int(min(H, cy + hh)) |
| return x0, y0, x1, y1 |
|
|
|
|
| def paste_mask(m: dict, dilate: int = 14, feather: int = 6) -> np.ndarray: |
| """角色并集掩码膨胀 dilate px + 高斯羽化 → float32 [H,W] ∈ [0,1]（角色修补只换掩码内）。""" |
| import cv2 |
| mk = (m["chars"]["player"] | m["chars"]["boss"]).astype(np.uint8) |
| k = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * dilate + 1, 2 * dilate + 1)) |
| a = cv2.dilate(mk, k).astype(np.float32) |
| if feather: |
| a = cv2.GaussianBlur(a, (0, 0), feather) |
| a = np.maximum(a, mk.astype(np.float32)) |
| return np.clip(a, 0, 1) |
|
|
|
|
| def area_down(img: np.ndarray, w: int = W, h: int = H) -> np.ndarray: |
| import cv2 |
| img = np.asarray(img) |
| if img.shape[:2] == (h, w): |
| return img |
| return cv2.resize(img, (w, h), interpolation=cv2.INTER_AREA if img.shape[1] > w else cv2.INTER_LANCZOS4) |
|
|
|
|
| |
| def dummy_frac(img: np.ndarray, m: dict, tol: float = 40.0, smooth: float = 5.0) -> dict: |
| """首帧（832×480）里每个可见角色掩码（腐蚀 1 px）内「人偶样」像素的比例：颜色离体块图同位置像素（浅蓝 / 浅鲑红人偶， |
| 带明暗）、平涂色图（玩家蓝 / Boss 红 / 武器白）、剪影图（深蓝 / 深红）任一 RGB 距离 < tol，**且**局部平滑（5×5 灰度 |
| 标准差 < smooth；真角色有盔甲 / 皮毛的细节，照抄的人偶是光滑的）。≈ 0 = 真角色，高 = 照抄了人偶 / 火柴人 / 标记色小人。 |
| → {"player": f, "boss": f}（不可见的角色不出现）。""" |
| import cv2 |
| img = area_down(np.asarray(img)).astype(np.float32) |
| g = img.mean(-1) |
| mu = cv2.blur(g, (5, 5)) |
| sd = np.sqrt(np.maximum(cv2.blur(g * g, (5, 5)) - mu * mu, 0)) |
| refs = [m["shaded"], m["flat"], m["sil"]] |
| out = {} |
| for k in ("player", "boss"): |
| mk = cv2.erode(m["chars"][k].astype(np.uint8), np.ones((3, 3), np.uint8)).astype(bool) |
| if mk.sum() < MIN_PX: |
| continue |
| x = img[mk] |
| d = np.min([np.linalg.norm(x - r[mk].astype(np.float32), axis=-1) for r in refs], 0) |
| out[k] = round(float(((d < tol) & (sd[mk] < smooth)).mean()), 4) |
| return out |
|
|
|
|
| DUMMY_FRAC = 0.35 |
|
|
|
|
| |
| QWEN8 = ("/opt/dlami/nvme/zhiyangdeng/PCE/weights/hf_hub/models--Qwen--Qwen3-VL-8B-Instruct/snapshots/" |
| "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b") |
| VLM_Q = { |
| "scene": "Is this video game screenshot clearly set in {vlm}? Answer Yes or No.", |
| "clean": "Is this image a clean game screenshot with no text, letters, numbers, logos, watermark, user interface, " |
| "grid lines drawn over the ground, or frame border? Answer Yes or No.", |
| "nodummy": "Look at the whole image. Is it free of any smooth featureless capsule-shaped mannequins, dummies or " |
| "toy-like placeholder figures? Answer Yes or No.", |
| "alive": "This is a close-up of a video game screenshot. Is there, at the centre of the image, a living video game " |
| "character ({who}) rendered with detailed armour, skin, fur or clothing, and not a smooth mannequin, " |
| "dummy, statue, rock, plant or object? Answer Yes or No.", |
| } |
|
|
|
|
| def zoom(img: Image.Image, mask: np.ndarray, min_side: int = 448) -> Image.Image: |
| """mask（832×480）的外接框外扩 1/3 → 从 img（任意尺寸，同长宽比）裁出、放大到短边 ≥ min_side。""" |
| ys, xs = np.nonzero(mask) |
| x0, x1, y0, y1 = xs.min(), xs.max() + 1, ys.min(), ys.max() + 1 |
| pw, ph = max(16, (x1 - x0) // 3), max(16, (y1 - y0) // 3) |
| sx, sy = img.width / W, img.height / H |
| box = (int(max(0, x0 - pw) * sx), int(max(0, y0 - ph) * sy), int(min(W, x1 + pw) * sx), int(min(H, y1 + ph) * sy)) |
| c = img.crop(box) |
| s = max(1.0, min_side / max(1, min(c.size))) |
| return c.resize((int(c.width * s), int(c.height * s)), Image.LANCZOS) |
|
|
|
|
| def vlm_items(img: Image.Image, m: dict, scene: str | None, keys=("scene", "clean", "nodummy", "alive")): |
| """→ [(images, text)], [key]：alive 拆成 alive_p / alive_b（帧 0 里没有的角色不问）。""" |
| batch, ks = [], [] |
| for k in keys: |
| if k == "alive": |
| for kk, who, mk in (("alive_p", "the hero", m["chars"]["player"]), ("alive_b", "a monster", m["chars"]["boss"])): |
| if mk.sum() >= MIN_PX: |
| batch.append(([zoom(img, mk)], VLM_Q["alive"].format(who=who))) |
| ks.append(kk) |
| elif k == "scene": |
| if scene: |
| batch.append(([img], VLM_Q["scene"].format(vlm=SCENES[scene]["vlm"]))) |
| ks.append(k) |
| else: |
| batch.append(([img], VLM_Q[k])) |
| ks.append(k) |
| return batch, ks |
|
|
|
|
| |
| def label(img: np.ndarray, text: str, scale: float = 0.5) -> np.ndarray: |
| import cv2 |
| out = np.ascontiguousarray(img.copy()) |
| (tw, th), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, scale, 1) |
| cv2.rectangle(out, (0, 0), (tw + 8, th + 10), (0, 0, 0), -1) |
| cv2.putText(out, text, (4, th + 4), cv2.FONT_HERSHEY_SIMPLEX, scale, (255, 255, 255), 1, cv2.LINE_AA) |
| return out |
|
|
|
|
| def grid(tiles: list[np.ndarray], ncol: int, tile_w: int = 416) -> np.ndarray: |
| import cv2 |
| th = round(tile_w * H / W) |
| ts = [cv2.resize(t, (tile_w, th), interpolation=cv2.INTER_AREA) for t in tiles] |
| while len(ts) % ncol: |
| ts.append(np.zeros_like(ts[0])) |
| return np.concatenate([np.concatenate(ts[i:i + ncol], 1) for i in range(0, len(ts), ncol)], 0) |
|
|
