File size: 16,371 Bytes
409fd9e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
#!/usr/bin/env python3
"""genbench3 · 首帧候选(PCE env pce_f3,FLUX.2 klein-9B;只读复用 firstframe_flux)。每个起点(sources.json)出:
  * style_<name>_s<seed>:整帧换画风(RESTYLE_V4:纯材质 / 渲染改写,不增删物体;6 个种子,门槛挑);
  * char_<role>_<look>_s<seed>:只换一个角色的外观 —— 按 ids0 的角色掩码取包围框(长宽比 = W:H,留边),裁出来放大到
    FLUX 网格编辑(CHAR_PROMPT:同位置 / 姿势 / 体型 / 轮廓 / 武器,只换外观),缩回原尺寸,再只在掩码内(外扩 2 px、
    羽化 1 px,扣掉另一个角色)贴回原帧 ⇒ 掩码外逐像素不变,语义图不会被改;
  * --combo picks.json:在已通过门槛的画风首帧上再换角色(char 编辑的底图换成那张画风帧,提示词带上画风)。
每张候选旁边写 <name>.json:设置 + 自动指标(layout_metrics vs ids0、struct_metrics vs 原帧、角色显著度 / 填充率 / 外观变化)。
可续跑(已有 png+json 跳过),按 --shard i --nshards n 分卡。
usage: CUDA_VISIBLE_DEVICES=0 $FPY scripts/genbench3/make_f0.py --shard 0 --nshards 8 [--combo outputs/genbench3/picks_pass1.json]"""
import argparse
import json
import os
import sys
import time

import numpy as np
from PIL import Image
from scipy import ndimage

M = "/opt/dlami/nvme/zhiyangdeng/maliang"
sys.path.insert(0, f"{M}/code/gc25d")
sys.path.insert(0, f"{M}/scripts")
import firstframe_flux as FF  # noqa: E402

G = f"{M}/outputs/genbench3"
STYLE_SEEDS, CHAR_SEEDS = (0, 1, 2, 3, 4, 5), (0, 1)
# genbench2 的 12 种(训练外观里都没有)+ 4 种常见游戏画风(pixel / paint 与 v9 的后处理画风相近 = 近分布对照)
# 主题画风写成「已有的东西换材质」:genbench2 的原描述(「巧克力墙、棒棒糖树」)等于让 FLUX 往空地上加东西(冒烟 candy / lego)
STYLES = {
    "ukiyoe": "a Japanese ukiyo-e woodblock print: bold black outlines, flat areas of Prussian blue, vermilion and ochre, "
              "visible wood-grain and washi paper texture, no shading gradients",
    "stainedglass": "a medieval stained-glass window: every existing surface, object and character rendered as pieces of "
                    "glowing translucent coloured glass separated by thick black lead lines, jewel-toned light",
    "lego": "a world made of plastic toy bricks, where the existing ground is re-textured as flat toy baseplates with "
            "studs, existing rocks, plants, walls and buildings become brick-built versions of themselves at the same "
            "places, characters become toy minifigures, glossy ABS plastic in bright primary colours",
    "voxel": "voxel art, where every existing surface, object and character is rebuilt from small uniform cubes at the "
             "same place and size, crisp flat-shaded cube faces, simple saturated colours",
    "knitted": "a knitted wool and felt craft diorama, where every existing surface, object and character is re-made "
               "from yarn and felt at the same place, visible stitches and fuzzy fibres, soft pastel textile colours",
    "porcelain": "Chinese blue-and-white porcelain painting: cobalt-blue brush patterns on glossy white glaze, everything "
                 "rendered only in shades of blue on white",
    "blueprint": "a technical blueprint drawing: thin white line-art on deep blue grid paper, hatching for shading, no "
                 "colour fills",
    "chalkboard": "a chalk drawing on a dark green school blackboard: dusty chalk strokes in white and pastel colours, "
                  "smudges and eraser marks",
    "candy": "a candy and dessert world, where the existing ground is re-textured as smooth pink frosting, existing rocks "
             "become glossy hard candies, existing plants become small lollipops and gummy shapes, existing walls and "
             "buildings become chocolate and wafer, with pastel sugary colours",
    "lava": "a volcanic hellscape, where the existing ground is re-textured as black obsidian and ash with glowing cracks "
            "of molten lava, existing objects become dark volcanic rock, floating embers, red-orange light from below",
    "mosaic": "an ancient Roman floor mosaic: everything made of small square stone and glass tesserae with visible grout "
              "lines, earthy terracotta, ochre and teal colours",
    "papyrus": "an ancient Egyptian tomb wall painting on papyrus: flat ochre, turquoise and lapis colours, thick dark "
               "outlines, cracked aged plaster texture",
    "winter": "a snowy winter night: thick fresh snow covering the existing ground, roofs and plants, cold blue moonlight "
              "and falling snowflakes",
    "watercolor": FF.STYLE_PRESETS["watercolor painting"],
    "pixelart": FF.STYLE_PRESETS["pixel-art retro game"],
    "neon": "a neon cyberpunk night: the existing surfaces become dark, wet and reflective, lit by glowing magenta and "
            "cyan neon light, existing objects and buildings get glowing neon edges, haze and rain",
}
NEAR_DIST = ("pixelart", "watercolor")          # 训练里有 pixel / paint 后处理:近分布
LOOKS = {
    "player": {
        "silverknight": "a knight in polished silver plate armour with a bright blue cape and a plumed silver helmet",
        "firemage": "a fire mage in flowing crimson-and-gold robes with a pointed hood and glowing orange runes",
        "ninja": "a ninja dressed entirely in black with a long red scarf and a black face mask",
        "ranger": "an elven ranger in a green hooded cloak and brown leather armour with long blond hair",
    },
    "boss": {
        "stonegolem": "a hulking golem made of grey granite boulders held together by glowing cyan crystals",
        "skeletonking": "a giant skeletal undead king in bone armour with a tattered purple cloak and a golden crown",
        "icetroll": "a giant ice troll with pale blue skin, white shaggy fur and frost crystals on its shoulders",
        "demonlord": "a demon warlord in jagged black-and-crimson armour with large curved horns and glowing yellow eyes",
    },
}
WHO = {"player": "the player hero (the smaller human character)", "boss": "the boss (the large enemy monster)"}
PARTS = {"player": FF.PLAYER_PARTS, "boss": FF.BOSS_PARTS}
CHAR_PROMPT = ("This is a close-up crop of a top-down isometric action RPG game screenshot. Redesign only {who} in the "
               "centre of the image as {look}. Keep exactly the same position, pose, body size, silhouette and facing "
               "direction, and keep the weapon in the same hand at the same angle. Keep the camera angle, lighting, shadows, "
               "the background and any other character exactly unchanged.{style}")
COMBO_STYLE = " The whole image is in the art style of {style}; draw the redesigned character in that same art style."
# 画风提示词:firstframe_flux 的 v1 会在空地上加东西(冒烟 candy:巧克力墙 / 棒棒糖 = 与语义图矛盾)⇒ 改成纯材质 / 渲染改写
# v4(冒烟:同种子下大轮廓多出的比 v3 少,浮世绘 coarse precision 0.58 → 0.74)
RESTYLE_V4 = ("Re-render this exact top-down isometric action RPG game screenshot in a new art style: {style}. This is a "
              "pure material and rendering change of the existing image: every region, object, building, wall, rock, "
              "plant, path and character stays exactly where it is with exactly the same outline, size and pose, and the "
              "two characters keep their weapons. Do not add anything new and do not remove anything; large empty areas "
              "of ground stay empty and are only re-textured. Same camera, same composition.")
PAD, MIN_CROP_W, FEATHER = 1.7, 176, 1.5
# 贴回区域 = 胶囊掩码外扩 CORE px ∪(掩码外 REACH·角色尺寸以内、FLUX 改动 ΔE > DIFF_DE 的连通区):语义图的角色是胶囊代理,
# 真实轮廓会伸出掩码(熔岩巨人的拳头)—— 只贴掩码内会留下旧角色的残块(冒烟 icetroll)
CORE, REACH, DIFF_DE = 2, 0.18, 14.0
AR = FF.W_OUT / FF.H_OUT


# ------------------------------------------------------------------ 指标 ----
def ring(mask, other, r0=3, r1=12):
    """角色掩码外 r0..r1 px 的一圈(扣掉另一个角色)= 角色周围的背景。"""
    out = ndimage.binary_dilation(mask, iterations=r1) & ~ndimage.binary_dilation(mask, iterations=r0)
    return out & ~other


def char_stats(rgb, ids):
    """每个角色:显著度 = 掩码内 Lab 均值与周围一圈的 ΔE;填充率 = 掩码内与周围背景(一圈的中位色)ΔE < 12 的比例
    (越高 ⇒ 掩码里画的是背景,角色没填满语义轮廓)。"""
    L = FF._lab(rgb)
    out = {}
    for role, parts in PARTS.items():
        m = np.isin(ids, parts)
        o = np.isin(ids, PARTS["boss" if role == "player" else "player"])
        rg = ring(m, o)
        if m.sum() < 50 or rg.sum() < 50:
            continue
        bg = np.median(L[rg], 0)
        out[role] = {"salience": round(float(np.linalg.norm(L[m].mean(0) - L[rg].mean(0))), 2),
                     "bg_fill": round(float((np.linalg.norm(L[m] - bg, axis=-1) < 12).mean()), 4), "px": int(m.sum())}
    return out


def coarse_struct(src, out):
    """粗尺度(½ 分辨率 + σ2 模糊,压掉画风纹理,只剩物体轮廓)的 struct_metrics:precision 低 ⇒ 多出了物体 / 轮廓。"""
    def half(x):
        return np.asarray(Image.fromarray(x).resize((x.shape[1] // 2, x.shape[0] // 2), Image.BILINEAR))
    a, b = (ndimage.gaussian_filter(half(x).astype(np.float32), (2, 2, 0)).clip(0, 255).astype(np.uint8) for x in (src, out))
    return {f"coarse_{k}": round(v, 4) for k, v in FF.struct_metrics(a, b, tol=2).items()}


def metrics(rgb, src, ids):
    m = {"layout": {k: (round(v, 4) if isinstance(v, float) else v) for k, v in FF.layout_metrics(rgb, ids).items()},
         "struct": {k: round(v, 4) for k, v in FF.struct_metrics(src, rgb).items()}, "chars": char_stats(rgb, ids)}
    m["struct"].update(coarse_struct(src, rgb))
    return m


# ------------------------------------------------------------------ 角色编辑 ----
def crop_box(mask):
    ys, xs = np.nonzero(mask)
    cy, cx = (ys.min() + ys.max()) / 2, (xs.min() + xs.max()) / 2
    bw, bh = xs.max() - xs.min() + 1, ys.max() - ys.min() + 1
    w = int(round(min(FF.W_OUT, max(MIN_CROP_W, PAD * max(bw, bh * AR)))))
    h = int(round(w / AR))
    if h > FF.H_OUT:
        h, w = FF.H_OUT, int(round(FF.H_OUT * AR))
    x0 = int(np.clip(round(cx - w / 2), 0, FF.W_OUT - w))
    y0 = int(np.clip(round(cy - h / 2), 0, FF.H_OUT - h))
    return y0, y0 + h, x0, x0 + w


def char_edit(ff, base, ids, role, look, seed, style=None):
    import torch
    m = np.isin(ids, PARTS[role])
    other = np.isin(ids, PARTS["boss" if role == "player" else "player"])
    y0, y1, x0, x1 = crop_box(m)
    crop = base[y0:y1, x0:x1]
    prompt = CHAR_PROMPT.format(who=WHO[role], look=look, style=COMBO_STYLE.format(style=style) if style else "")
    t0 = time.time()
    with torch.no_grad():
        big = ff.net(prompt, [FF.to_gen(crop)], FF.DEFAULT_STEPS["flux2"], seed)
    torch.cuda.synchronize()
    sec = time.time() - t0
    ed = FF.area_resize(big, x1 - x0, y1 - y0)
    mc, oc = m[y0:y1, x0:x1], other[y0:y1, x0:x1]
    ys, xs = np.nonzero(mc)
    reach = max(4, int(round(REACH * max(ys.max() - ys.min(), xs.max() - xs.min()))))
    diff = ndimage.gaussian_filter(np.linalg.norm(FF._lab(ed) - FF._lab(crop), axis=-1), 1.0) > DIFF_DE
    near = ndimage.binary_dilation(mc, iterations=reach)
    lab, _ = ndimage.label(diff & near)
    touch = np.unique(lab[ndimage.binary_dilation(mc, iterations=CORE) & (lab > 0)])
    grown = np.isin(lab, touch[touch > 0])                         # 与掩码相连的改动区(旧角色伸出掩码的部分)
    reg = ndimage.binary_fill_holes(ndimage.binary_dilation(mc, iterations=CORE) | grown)
    reg &= ~ndimage.binary_dilation(oc, iterations=1)
    a = np.clip(ndimage.gaussian_filter(reg.astype(np.float32), FEATHER), 0, 1)
    a[oc] = 0.0
    a[mc] = 1.0
    out = base.copy()
    out[y0:y1, x0:x1] = (a[..., None] * ed + (1 - a[..., None]) * crop).round().clip(0, 255).astype(np.uint8)
    Lb, Lo = FF._lab(base), FF._lab(out)
    info = {"prompt": prompt, "box": [y0, y1, x0, x1], "sec": round(sec, 2), "reach_px": reach,
            "grown_px": int((grown & ~mc).sum()), "region_px": int(reg.sum()), "mask_px": int(mc.sum()),
            "change_dE": round(float(np.linalg.norm(Lo[m] - Lb[m], axis=-1).mean()), 2)}
    return out, ed, info


# ------------------------------------------------------------------ 任务 ----
def jobs(sources, combo):
    out = []
    for s in sources:
        tag = s["tag"]
        if combo is None:
            for st in STYLES:
                for seed in STYLE_SEEDS:
                    out.append({"tag": tag, "kind": "style", "style": st, "seed": seed, "name": f"style_{st}_s{seed}"})
            for role, looks in LOOKS.items():
                for lk in looks:
                    for seed in CHAR_SEEDS:
                        out.append({"tag": tag, "kind": "char", "role": role, "look": lk, "seed": seed,
                                    "name": f"char_{role}_{lk}_s{seed}"})
        else:
            for c in combo.get(tag, []):                  # {"base": style 候选名, "style": st, "role":..., "look":...}
                for seed in CHAR_SEEDS:
                    out.append({"tag": tag, "kind": "combo", "base": c["base"], "style": c["style"], "role": c["role"],
                                "look": c["look"], "seed": seed,
                                "name": f"combo_{c['style']}+{c['role']}_{c['look']}_s{seed}"})
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--shard", type=int, default=0)
    ap.add_argument("--nshards", type=int, default=1)
    ap.add_argument("--combo", default=None, help="picks json:{tag: [{base, style, role, look}]}")
    ap.add_argument("--only", default=None, help="正则,只跑 tag/name 匹配的(冒烟用)")
    a = ap.parse_args()
    sources = json.load(open(f"{G}/sources.json"))
    combo = json.load(open(a.combo)) if a.combo else None
    todo = [j for i, j in enumerate(jobs(sources, combo)) if i % a.nshards == a.shard]
    if a.only:
        import re
        todo = [j for j in todo if re.search(a.only, f"{j['tag']}/{j['name']}")]
    todo = [j for j in todo if not os.path.exists(f"{G}/cands/{j['tag']}/{j['name']}.json")]
    print(f"[make_f0] shard {a.shard}/{a.nshards}: {len(todo)} 张待生成", flush=True)
    if not todo:
        return
    ff = FF.FirstFrameFlux("klein9b")
    print(f"[make_f0] FLUX 加载 {ff.load_sec:.0f}s", flush=True)
    cache = {}
    for j in todo:
        tag = j["tag"]
        if tag not in cache:
            d = f"{G}/sources/{tag}"
            src = np.asarray(Image.open(f"{d}/f0.png").convert("RGB"))
            ids = np.load(f"{d}/ids0.npy")
            cache[tag] = (src, ids)
            mpath = f"{d}/f0_metrics.json"
            if not os.path.exists(mpath):
                json.dump(metrics(src, src, ids), open(mpath, "w"), indent=1)
        src, ids = cache[tag]
        od = f"{G}/cands/{tag}"
        os.makedirs(od, exist_ok=True)
        info = dict(j)
        if j["kind"] == "style":
            out = ff.generate(RESTYLE_V4.format(style=STYLES[j["style"]]), [FF.to_gen(src)], seed=j["seed"])
            info.update(prompt=ff.last["prompt"], sec=ff.last["sec"], near_dist=j["style"] in NEAR_DIST)
        else:
            base = src if j["kind"] == "char" else np.asarray(Image.open(f"{od}/{j['base']}.png").convert("RGB"))
            out, ed, ci = char_edit(ff, base, ids, j["role"], LOOKS[j["role"]][j["look"]], j["seed"],
                                    STYLES[j["style"]] if j["kind"] == "combo" else None)
            info.update(ci)
            Image.fromarray(ed).save(f"{od}/{j['name']}_crop.png")
        Image.fromarray(out).save(f"{od}/{j['name']}.png")
        info["metrics"] = metrics(out, src, ids)
        json.dump(info, open(f"{od}/{j['name']}.json", "w"), indent=1)
        print(f"[make_f0] {tag} {j['name']} {info['sec']}s", flush=True)


if __name__ == "__main__":
    main()