xiponn's picture
Update app.py
e0b04b8 verified
Raw History Blame Contribute Delete
23.4 kB
# -*- coding: utf-8 -*-
from pathlib import Path
import shutil
import numpy as np
from PIL import Image
import gradio as gr
import random, json
import torch
# =========================
# 1) Classes
# =========================
CLASS_NAMES = [
"key", "bread", "carrot", "onion", "steak",
"sailboat", "bicycle", "cookie", "banana",
"ice_cream", "frying_pan", "sandwich", "hamburger",
]
# =========================
# 2) Story / Game
# - 支援每張圖片不同 title / prompt / status
# =========================
STORY = [
{
"id": 0,
"name": "故事開始",
"images": ["scene_01_kitchen.jpg", "scene_02_locked.jpg"],
"mode": "intro",
"titles": ["故事開始", "解鎖櫥櫃"],
"prompts": [
"### 小老鼠走進了廚房,準備做一份料理,但似乎遇到了一點困難……",
"### 小老鼠跳到了桌上,發現遠處的櫥櫃竟然被鎖住了!",
],
# intro 也可以給 statuses(可選)
# "statuses": ["", ""],
},
{
"id": 1,
"name": "尋找道具",
"images": ["scene_03_find_key.jpg"],
"mode": "single",
"targets": ["key"],
"prompt": "### 小老鼠來到了櫥櫃下方,好像有可以解鎖的道具……",
"status": "找到並畫出來給小老鼠吧!",
},
{
"id": 2,
"name": "尋找食材",
"images": ["scene_04_fridge.jpg"],
"mode": "count_any",
"targets": ["bread", "carrot", "onion", "steak"],
"need_count": 2,
"prompt": (
"### 小老鼠打開了冰箱,裡面有 ***麵包、胡蘿蔔、洋蔥、牛排***。\n\n"
"### 想想今天要準備 **哪2樣** 食材呢?"
),
"status": "成功解鎖🔓 畫出想準備的食材吧!",
},
{
"id": 3,
"name": "被發現了!",
"images": ["scene_05_cat.jpg"],
"mode": "intro",
"prompt": "### 貓咪來了!趕快先離開這裡……",
"status": "突發狀況!趕快先離開這裡(按 Next)",
},
{
"id": 4,
"name": "乘風破浪🌊",
"images": ["scene_06_river.jpg"],
"mode": "single",
"targets": ["sailboat"],
"prompt": "### 小老鼠不會游泳,貓咪緊追在後,需要你的幫忙!",
"status": "快幫小老鼠造一艘小船!",
},
{
"id": 5,
"name": "體力透支",
"images": ["scene_07_mouse.jpg"],
"mode": "collect_all",
"targets": ["cookie", "banana"],
"prompt": "### 要餓鼠了……怎麼辦?小老鼠想吃 **餅乾** 和 **香蕉**。",
"status": "成功渡河⛵ 現在需要你的餵食,幫小老鼠補充體力。",
},
{
"id": 6,
"name": "充電完畢,準備料理🧑🏻‍🍳",
"images": ["scene_08_cook.jpg"],
"mode": "any_one",
"targets": ["sandwich", "hamburger"],
"prompt": "### 試著用 **麵包、生菜、洋蔥、牛排** 做出一份美味的料理吧!",
"status": "備料完成,料理鼠王需要你的幫忙!",
},
{
"id": 7,
"name": "你就是料理鼠王",
"images": ["scene_09_finished.jpg"],
"mode": "ending",
"prompt": "### 謝謝你幫助小老鼠完成這次的冒險!",
"status": "🎉 恭喜完成料理!",
},
]
FINAL_SCENE_IDX = next(i for i, sc in enumerate(STORY) if sc.get("mode") == "ending")
def is_last_playable_scene(scene_idx: int) -> bool:
"""最後一個可玩關卡 = ending 前一個"""
return int(scene_idx) == FINAL_SCENE_IDX - 1
# =========================
# 3) Persistent storage
# =========================
BASE_DIR = Path("/app/my_drawings")
BASE_DIR.mkdir(parents=True, exist_ok=True)
print("資料會存到:", BASE_DIR.resolve())
# =========================
# 4) Preprocess (Sketch -> 28x28)
# =========================
def preprocess_to_28x28_gray(im):
if isinstance(im, dict):
if "composite" in im and im.get("composite") is not None:
im = im["composite"]
else:
im = im.get("image", None) or im.get("background", None)
if not isinstance(im, Image.Image):
im = Image.fromarray(np.array(im))
im = im.convert("L")
big = im.resize((112, 112), Image.Resampling.BILINEAR)
arr = np.array(big).astype("float32") / 255.0
# 反相:黑線白底 -> 黑底白線
arr = 1.0 - arr
ys, xs = np.where(arr > 0.15)
if len(xs) > 0:
x_min, x_max = xs.min(), xs.max()
y_min, y_max = ys.min(), ys.max()
pad = 6
x_min = max(x_min - pad, 0)
y_min = max(y_min - pad, 0)
x_max = min(x_max + pad, arr.shape[1] - 1)
y_max = min(y_max + pad, arr.shape[0] - 1)
arr = arr[y_min:y_max + 1, x_min:x_max + 1]
small = Image.fromarray((arr * 255).astype("uint8"))
small = small.resize((28, 28), Image.Resampling.LANCZOS)
arr28 = np.array(small).astype("float32") / 255.0
# 3x3 最大池化(膨脹)
padded = np.pad(arr28, 1, mode="constant", constant_values=0.0)
dilated = np.zeros_like(arr28)
for i in range(28):
for j in range(28):
dilated[i, j] = padded[i:i + 3, j:j + 3].max()
return dilated
# =========================
# 5) Tab1: save drawings
# =========================
def save_drawing(label, im):
if label not in CLASS_NAMES:
return "⚠️ 請先從上方選擇一個類別(例如 key)", None
arr28 = preprocess_to_28x28_gray(im)
class_dir = BASE_DIR / label
class_dir.mkdir(parents=True, exist_ok=True)
existing = list(class_dir.glob("*.png"))
next_idx = 0
if existing:
indices = []
for p in existing:
parts = p.stem.split("_")
if len(parts) >= 2:
try:
indices.append(int(parts[-1]))
except ValueError:
pass
next_idx = (max(indices) + 1) if indices else 0
fname = class_dir / f"{label}_{next_idx:04d}.png"
img = Image.fromarray((arr28 * 255).astype("uint8"))
img.save(fname)
preview = img.resize((224, 224), Image.Resampling.NEAREST)
total = len(list(class_dir.glob("*.png")))
return f"已儲存:{fname.name}({label} 類別目前共 {total} 張)", preview
def clear_canvas_tab1():
return gr.update(value=None), "", None
def list_images_for_class(label):
class_dir = BASE_DIR / label
if not class_dir.exists():
return []
paths = sorted(class_dir.glob("*.png"))
return [[str(p), p.name] for p in paths]
def list_filenames_for_class(label):
class_dir = BASE_DIR / label
if not class_dir.exists():
return []
paths = sorted(class_dir.glob("*.png"))
return [p.name for p in paths]
def load_class_images(label):
return list_images_for_class(label), gr.update(choices=list_filenames_for_class(label), value=None)
def delete_one_image(label, filename):
if isinstance(filename, list):
filename = filename[0] if filename else None
if not filename:
return (
"⚠️ 請先在右側選要刪除的檔名",
list_images_for_class(label),
gr.update(choices=list_filenames_for_class(label), value=None),
)
path = BASE_DIR / label / filename
if path.exists():
path.unlink()
msg = f"已刪除:{filename}(類別:{label})"
else:
msg = f"❌ 找不到檔案:{filename}"
return msg, list_images_for_class(label), gr.update(choices=list_filenames_for_class(label), value=None)
def make_zip():
if not BASE_DIR.exists():
return "目前還沒有任何資料可以打包", None
if not any(BASE_DIR.rglob("*.png")):
return "目前還沒有任何圖像可以打包", None
zip_path = Path("my_drawings_export.zip")
if zip_path.exists():
zip_path.unlink()
shutil.make_archive("my_drawings_export", "zip", BASE_DIR)
return "已建立 ZIP 檔,可以下載囉", str(zip_path)
# =========================
# 6) Model inference
# =========================
MODEL_PATH = Path("quickdraw_cnn.pt")
LABEL_PATH = Path("label_map.json")
device = "cpu"
model = None
idx_to_class = None
class SmallCNN(torch.nn.Module):
def __init__(self, num_classes):
super().__init__()
self.net = torch.nn.Sequential(
torch.nn.Conv2d(1, 32, 3, padding=1), torch.nn.ReLU(),
torch.nn.MaxPool2d(2),
torch.nn.Conv2d(32, 64, 3, padding=1), torch.nn.ReLU(),
torch.nn.MaxPool2d(2),
torch.nn.Conv2d(64, 128, 3, padding=1), torch.nn.ReLU(),
torch.nn.AdaptiveAvgPool2d((1, 1)),
torch.nn.Flatten(),
torch.nn.Dropout(0.2),
torch.nn.Linear(128, num_classes),
)
def forward(self, x):
return self.net(x)
def load_model_once():
global model, idx_to_class
if model is not None:
return
if not MODEL_PATH.exists() or not LABEL_PATH.exists():
raise FileNotFoundError("找不到 quickdraw_cnn.pt 或 label_map.json(請先上傳到 Space 根目錄)")
with open(LABEL_PATH, "r", encoding="utf-8") as f:
m = json.load(f)
raw = m["idx_to_class"]
idx_to_class = {int(k): v for k, v in raw.items()}
model_ = SmallCNN(len(idx_to_class))
state = torch.load(MODEL_PATH, map_location=device)
model_.load_state_dict(state)
model_.eval()
model = model_
def predict_probs_dict(im):
load_model_once()
arr28 = preprocess_to_28x28_gray(im)
x = torch.tensor(arr28, dtype=torch.float32).unsqueeze(0).unsqueeze(0)
with torch.no_grad():
logits = model(x)
probs = torch.softmax(logits, dim=1).squeeze(0)
return {idx_to_class[i]: float(probs[i].item()) for i in range(probs.numel())}
# =========================
# 7) Story helpers
# =========================
def current_scene(scene_idx):
return STORY[int(scene_idx)]
def scene_image_path(scene_idx, img_idx):
sc = current_scene(scene_idx)
imgs = sc.get("images", [])
if not imgs:
return None
i = max(0, min(int(img_idx), len(imgs) - 1))
return imgs[i]
def in_draw_phase(scene_idx, img_idx):
sc = current_scene(scene_idx)
mode = sc.get("mode")
if mode in ("intro", "ending"):
return False
imgs = sc.get("images", [])
return (len(imgs) <= 1) or (int(img_idx) == len(imgs) - 1)
def next_enabled(scene_idx, img_idx):
sc = current_scene(scene_idx)
imgs = sc.get("images", [])
mode = sc.get("mode")
if mode == "intro":
return True
if mode == "ending":
return False
return (len(imgs) > 1) and (int(img_idx) < len(imgs) - 1)
def get_prompt_for_image(scene, img_idx):
prompts = scene.get("prompts")
if isinstance(prompts, list):
i = int(img_idx)
return prompts[min(i, len(prompts) - 1)]
return scene.get("prompt", "")
def get_status_for_image(scene, img_idx):
statuses = scene.get("statuses")
if isinstance(statuses, list):
i = int(img_idx)
return statuses[min(i, len(statuses) - 1)]
return scene.get("status", "")
def get_title_for_image(scene, img_idx):
titles = scene.get("titles")
if isinstance(titles, list):
i = int(img_idx)
return titles[min(i, len(titles) - 1)]
return scene.get("name", "")
def render_ui(scene_idx, img_idx, count, collected):
sc = current_scene(scene_idx)
img_idx = int(img_idx)
title = get_title_for_image(sc, img_idx)
img_path = scene_image_path(scene_idx, img_idx)
mode = sc.get("mode", "")
base_prompt = get_prompt_for_image(sc, img_idx)
base_status = get_status_for_image(sc, img_idx)
extra = ""
if mode == "count_any":
need = int(sc.get("need_count", 1))
extra = f"\n\n食材:**{int(count)}/{need}**"
elif mode == "collect_all":
targets = sc.get("targets", [])
got = set(collected or [])
checklist = " / ".join([f"{'✅' if t in got else '⬜'} {t}" for t in targets])
extra = f"\n\n收集:{checklist}"
prompt_md = f"# **{title}**\n\n{base_prompt}{extra}"
can_draw = in_draw_phase(scene_idx, img_idx)
can_next = next_enabled(scene_idx, img_idx)
guess_upd = gr.update(interactive=bool(can_draw))
next_upd = gr.update(interactive=bool(can_next))
return img_path, base_status, prompt_md, guess_upd, next_upd
# =========================
# 8) Actions (Tab3)
# =========================
def start_game():
scene_idx, img_idx = 0, 0
count, collected = 0, []
live = False
img_path, status, prompt, guess_upd, next_upd = render_ui(scene_idx, img_idx, count, collected)
return (
scene_idx, img_idx, count, collected, live,
status if status else "遊戲開始!按 Next 繼續故事。",
img_path,
prompt,
{}, # top5 清空
gr.update(value=None), # 畫布清空
guess_upd,
next_upd
)
def go_next(scene_idx, img_idx, count, collected, live):
scene_idx = int(scene_idx)
img_idx = int(img_idx)
count = int(count)
collected = list(collected or [])
sc = current_scene(scene_idx)
mode = sc.get("mode")
imgs = sc.get("images", [])
if mode == "ending":
img_path, status2, prompt, guess_upd, next_upd = render_ui(scene_idx, img_idx, count, collected)
return (
scene_idx, img_idx, count, collected, live,
"🎉 遊戲已結束~",
img_path, prompt,
{}, gr.update(value=None),
guess_upd, next_upd
)
# 同 scene 多張圖:先換圖
if len(imgs) > 1 and img_idx < len(imgs) - 1:
img_idx += 1
else:
# 進下一個 scene
scene_idx = min(scene_idx + 1, len(STORY) - 1)
img_idx = 0
count = 0
collected = []
live = False
img_path, status2, prompt, guess_upd, next_upd = render_ui(scene_idx, img_idx, count, collected)
status_show = (
status2 if status2 else
("劇情推進中…➡️(按 Next)" if not in_draw_phase(scene_idx, img_idx)
else "✏️ 請開始畫圖,按 Send 讓模型猜!")
)
return (
scene_idx, img_idx, count, collected, live,
status_show,
img_path,
prompt,
{}, # top5 清空
gr.update(value=None), # 畫布清空
guess_upd,
next_upd
)
def guess_story(scene_idx, img_idx, count, collected, live, im):
scene_idx = int(scene_idx)
img_idx = int(img_idx)
count = int(count)
collected = list(collected or [])
sc = current_scene(scene_idx)
mode = sc.get("mode")
# 先拿目前 UI(避免分支漏變數)
img_path, status_now, prompt_now, guess_upd, next_upd = render_ui(
scene_idx, img_idx, count, collected
)
# intro / ending / 非可畫狀態:不能猜
if mode in ("intro", "ending") or (not in_draw_phase(scene_idx, img_idx)):
msg = "⚠️ 這段是故事過場,請按 Next。" if mode != "ending" else "🎉 已完成遊戲~"
return (
scene_idx, img_idx, count, collected, live,
msg, # ✅ game_status
{}, gr.update(value=None),
img_path, prompt_now,
guess_upd, next_upd
)
# --- 模型推論 ---
probs = predict_probs_dict(im)
top5 = sorted(probs.items(), key=lambda x: x[1], reverse=True)[:5]
top5_labels = [lab for lab, _ in top5]
targets = sc.get("targets", [])
hit = [t for t in targets if t in top5_labels]
# ✅ 命中顯示(第一行)
if hit:
hit_text = f"✅ 命中:{', '.join(hit)}(Top-5)"
else:
hit_text = f"✏️ 還沒猜到(目標:{', '.join(targets)})"
progressed = False
if mode in ("single", "any_one"):
progressed = bool(hit)
elif mode == "count_any":
if hit:
count += 1
progressed = count >= int(sc.get("need_count", 1))
elif mode == "collect_all":
if hit:
for t in hit:
if t not in collected:
collected.append(t)
progressed = all(t in collected for t in targets)
# --- 未完成:留在本關(第二行 = 本關狀態 / 進度)---
if not progressed:
img_path, status_now, prompt_now, guess_upd, next_upd = render_ui(
scene_idx, img_idx, count, collected
)
hint_text = hit_text + ("\n" + status_now if status_now else "")
return (
scene_idx, img_idx, count, collected, live,
hint_text, # ✅ game_status:第一行命中,第二行提示
probs, gr.update(), # top5 更新、畫布不清
img_path, prompt_now,
guess_upd, next_upd
)
# --- 完成:推進下一段(第二行 = 下一段狀態)---
if is_last_playable_scene(scene_idx):
next_scene_idx = FINAL_SCENE_IDX
else:
next_scene_idx = min(scene_idx + 1, len(STORY) - 1)
scene_idx = next_scene_idx
img_idx = 0
count = 0
collected = []
live = False
img_path, status_next, prompt_next, guess_upd, next_upd = render_ui(
scene_idx, img_idx, count, collected
)
hint_text = hit_text + ("\n" + status_next if status_next else "")
return (
scene_idx, img_idx, count, collected, live,
hint_text, # ✅ game_status:第一行命中,第二行下一段提示
probs, gr.update(value=None), # top5 顯示、畫布清空
img_path, prompt_next,
guess_upd, next_upd
)
# =========================
# 9) UI
# =========================
CSS = """
#hint_box{
height: 32px; /* ✅ 只佔一行高度 */
overflow: hidden; /* ✅ 第二行被裁掉,但內容仍保留 \\n 分行 */
white-space: pre-line; /* ✅ \\n 保留,不會變同一行 */
padding: 6px 10px;
border-radius: 8px;
border: 1px solid rgba(0,0,0,0.12);
}
"""
with gr.Blocks(
title="Cook or Caught",
css="""
#hint_box textarea{
height: 32px !important;
overflow: hidden !important;
white-space: pre-line;
}
"""
) as demo:
gr.Markdown("# 🍳 **Cook or Caught**")
...
# ---- Tab 1 ----
with gr.Tab("圖形蒐集"):
gr.Markdown("### 選一個類別,畫圖並點 Submit(會存成 28×28 PNG)")
with gr.Row():
with gr.Column(scale=2):
label_input = gr.Dropdown(choices=CLASS_NAMES, value="key", label="類別")
sketch_input = gr.Sketchpad(label="畫圖區(建議筆刷調細)")
with gr.Row():
clear_btn = gr.Button("Clear", variant="secondary")
submit_btn = gr.Button("Submit", variant="primary")
with gr.Column(scale=1):
status_output = gr.Textbox(label="狀態", interactive=False)
preview_output = gr.Image(label="預覽(28×28 放大)", image_mode="L")
submit_btn.click(fn=save_drawing, inputs=[label_input, sketch_input], outputs=[status_output, preview_output])
clear_btn.click(fn=clear_canvas_tab1, inputs=None, outputs=[sketch_input, status_output, preview_output])
# ---- Tab 2 ----
with gr.Tab("瀏覽資料庫"):
gr.Markdown("### 選擇類別 → 載入 → 可刪單張 / 打包下載")
gallery_label = gr.Dropdown(choices=CLASS_NAMES, value="key", label="類別")
load_btn = gr.Button("載入該類別所有圖像", variant="primary")
with gr.Row():
with gr.Column(scale=3):
gallery = gr.Gallery(
label="已蒐集圖像(28×28)",
columns=10,
height=400,
allow_preview=False,
)
with gr.Column(scale=1):
file_radio = gr.Radio(choices=[], label="選擇要刪除的檔名")
delete_btn = gr.Button("刪除選取圖片", variant="stop")
delete_status = gr.Textbox(label="刪除狀態", interactive=False)
gr.Markdown("---")
zip_status = gr.Textbox(label="打包狀態", interactive=False)
zip_file = gr.File(label="下載 ZIP(my_drawings_export.zip)")
zip_btn = gr.Button("Download")
load_btn.click(fn=load_class_images, inputs=gallery_label, outputs=[gallery, file_radio])
delete_btn.click(fn=delete_one_image, inputs=[gallery_label, file_radio], outputs=[delete_status, gallery, file_radio])
zip_btn.click(fn=make_zip, inputs=None, outputs=[zip_status, zip_file])
# ---- Tab 3 ----
with gr.Tab("遊戲開始!"):
gr.Markdown(
"按 Next 推進故事;到需要你幫忙的時候,再畫圖按 **Send🙌🏻**。\n\n"
"只要正解進到 Top-5,即可成功推進劇情!"
)
st_scene_idx = gr.State(0)
st_img_idx = gr.State(0)
st_count = gr.State(0)
st_collected = gr.State([])
st_live = gr.State(False)
with gr.Column():
scene_image = gr.Image(
label="情境畫面",
type="filepath",
show_download_button=False,
show_share_button=False,
)
prompt_box = gr.Markdown("**提示:** Start!")
# ✅ Tab3 畫布尺寸 = Tab1
game_sketch = gr.Sketchpad(label="畫圖區")
with gr.Row():
start_btn = gr.Button("Start!", variant="primary")
guess_btn = gr.Button("Send🙌🏻", variant="secondary")
next_btn = gr.Button("Next", variant="secondary")
# ✅ Hint 改成 Markdown(保留換行),高度由 CSS 控制
game_status = gr.Textbox(label="Hint💡", interactive=False, lines=2, max_lines=2)
top5_bar = gr.Label(label="模型猜測(Top-5)", num_top_classes=5)
start_btn.click(
fn=start_game,
inputs=None,
outputs=[
st_scene_idx, st_img_idx, st_count, st_collected, st_live,
game_status, scene_image, prompt_box, top5_bar, game_sketch,
guess_btn, next_btn
],
)
next_btn.click(
fn=go_next,
inputs=[st_scene_idx, st_img_idx, st_count, st_collected, st_live],
outputs=[
st_scene_idx, st_img_idx, st_count, st_collected, st_live,
game_status, scene_image, prompt_box, top5_bar, game_sketch,
guess_btn, next_btn
],
)
guess_btn.click(
fn=guess_story,
inputs=[st_scene_idx, st_img_idx, st_count, st_collected, st_live, game_sketch],
outputs=[
st_scene_idx, st_img_idx, st_count, st_collected, st_live,
game_status, top5_bar, game_sketch,
scene_image, prompt_box,
guess_btn, next_btn
],
)
if __name__ == "__main__":
demo.launch()