# -*- coding: utf-8 -*- from pathlib import Path import shutil import numpy as np from PIL import Image import gradio as gr import random, json import torch # ========================= # 1) Classes # ========================= CLASS_NAMES = [ "key", "bread", "carrot", "onion", "steak", "sailboat", "bicycle", "cookie", "banana", "ice_cream", "frying_pan", "sandwich", "hamburger", ] # ========================= # 2) Story / Game # - 支援每張圖片不同 title / prompt / status # ========================= STORY = [ { "id": 0, "name": "故事開始", "images": ["scene_01_kitchen.jpg", "scene_02_locked.jpg"], "mode": "intro", "titles": ["故事開始", "解鎖櫥櫃"], "prompts": [ "### 小老鼠走進了廚房,準備做一份料理,但似乎遇到了一點困難……", "### 小老鼠跳到了桌上,發現遠處的櫥櫃竟然被鎖住了!", ], # intro 也可以給 statuses(可選) # "statuses": ["", ""], }, { "id": 1, "name": "尋找道具", "images": ["scene_03_find_key.jpg"], "mode": "single", "targets": ["key"], "prompt": "### 小老鼠來到了櫥櫃下方,好像有可以解鎖的道具……", "status": "找到並畫出來給小老鼠吧!", }, { "id": 2, "name": "尋找食材", "images": ["scene_04_fridge.jpg"], "mode": "count_any", "targets": ["bread", "carrot", "onion", "steak"], "need_count": 2, "prompt": ( "### 小老鼠打開了冰箱,裡面有 ***麵包、胡蘿蔔、洋蔥、牛排***。\n\n" "### 想想今天要準備 **哪2樣** 食材呢?" ), "status": "成功解鎖🔓 畫出想準備的食材吧!", }, { "id": 3, "name": "被發現了!", "images": ["scene_05_cat.jpg"], "mode": "intro", "prompt": "### 貓咪來了!趕快先離開這裡……", "status": "突發狀況!趕快先離開這裡(按 Next)", }, { "id": 4, "name": "乘風破浪🌊", "images": ["scene_06_river.jpg"], "mode": "single", "targets": ["sailboat"], "prompt": "### 小老鼠不會游泳,貓咪緊追在後,需要你的幫忙!", "status": "快幫小老鼠造一艘小船!", }, { "id": 5, "name": "體力透支", "images": ["scene_07_mouse.jpg"], "mode": "collect_all", "targets": ["cookie", "banana"], "prompt": "### 要餓鼠了……怎麼辦?小老鼠想吃 **餅乾** 和 **香蕉**。", "status": "成功渡河⛵ 現在需要你的餵食,幫小老鼠補充體力。", }, { "id": 6, "name": "充電完畢,準備料理🧑🏻‍🍳", "images": ["scene_08_cook.jpg"], "mode": "any_one", "targets": ["sandwich", "hamburger"], "prompt": "### 試著用 **麵包、生菜、洋蔥、牛排** 做出一份美味的料理吧!", "status": "備料完成,料理鼠王需要你的幫忙!", }, { "id": 7, "name": "你就是料理鼠王", "images": ["scene_09_finished.jpg"], "mode": "ending", "prompt": "### 謝謝你幫助小老鼠完成這次的冒險!", "status": "🎉 恭喜完成料理!", }, ] FINAL_SCENE_IDX = next(i for i, sc in enumerate(STORY) if sc.get("mode") == "ending") def is_last_playable_scene(scene_idx: int) -> bool: """最後一個可玩關卡 = ending 前一個""" return int(scene_idx) == FINAL_SCENE_IDX - 1 # ========================= # 3) Persistent storage # ========================= BASE_DIR = Path("/app/my_drawings") BASE_DIR.mkdir(parents=True, exist_ok=True) print("資料會存到:", BASE_DIR.resolve()) # ========================= # 4) Preprocess (Sketch -> 28x28) # ========================= def preprocess_to_28x28_gray(im): if isinstance(im, dict): if "composite" in im and im.get("composite") is not None: im = im["composite"] else: im = im.get("image", None) or im.get("background", None) if not isinstance(im, Image.Image): im = Image.fromarray(np.array(im)) im = im.convert("L") big = im.resize((112, 112), Image.Resampling.BILINEAR) arr = np.array(big).astype("float32") / 255.0 # 反相:黑線白底 -> 黑底白線 arr = 1.0 - arr ys, xs = np.where(arr > 0.15) if len(xs) > 0: x_min, x_max = xs.min(), xs.max() y_min, y_max = ys.min(), ys.max() pad = 6 x_min = max(x_min - pad, 0) y_min = max(y_min - pad, 0) x_max = min(x_max + pad, arr.shape[1] - 1) y_max = min(y_max + pad, arr.shape[0] - 1) arr = arr[y_min:y_max + 1, x_min:x_max + 1] small = Image.fromarray((arr * 255).astype("uint8")) small = small.resize((28, 28), Image.Resampling.LANCZOS) arr28 = np.array(small).astype("float32") / 255.0 # 3x3 最大池化(膨脹) padded = np.pad(arr28, 1, mode="constant", constant_values=0.0) dilated = np.zeros_like(arr28) for i in range(28): for j in range(28): dilated[i, j] = padded[i:i + 3, j:j + 3].max() return dilated # ========================= # 5) Tab1: save drawings # ========================= def save_drawing(label, im): if label not in CLASS_NAMES: return "⚠️ 請先從上方選擇一個類別(例如 key)", None arr28 = preprocess_to_28x28_gray(im) class_dir = BASE_DIR / label class_dir.mkdir(parents=True, exist_ok=True) existing = list(class_dir.glob("*.png")) next_idx = 0 if existing: indices = [] for p in existing: parts = p.stem.split("_") if len(parts) >= 2: try: indices.append(int(parts[-1])) except ValueError: pass next_idx = (max(indices) + 1) if indices else 0 fname = class_dir / f"{label}_{next_idx:04d}.png" img = Image.fromarray((arr28 * 255).astype("uint8")) img.save(fname) preview = img.resize((224, 224), Image.Resampling.NEAREST) total = len(list(class_dir.glob("*.png"))) return f"已儲存:{fname.name}({label} 類別目前共 {total} 張)", preview def clear_canvas_tab1(): return gr.update(value=None), "", None def list_images_for_class(label): class_dir = BASE_DIR / label if not class_dir.exists(): return [] paths = sorted(class_dir.glob("*.png")) return [[str(p), p.name] for p in paths] def list_filenames_for_class(label): class_dir = BASE_DIR / label if not class_dir.exists(): return [] paths = sorted(class_dir.glob("*.png")) return [p.name for p in paths] def load_class_images(label): return list_images_for_class(label), gr.update(choices=list_filenames_for_class(label), value=None) def delete_one_image(label, filename): if isinstance(filename, list): filename = filename[0] if filename else None if not filename: return ( "⚠️ 請先在右側選要刪除的檔名", list_images_for_class(label), gr.update(choices=list_filenames_for_class(label), value=None), ) path = BASE_DIR / label / filename if path.exists(): path.unlink() msg = f"已刪除:{filename}(類別:{label})" else: msg = f"❌ 找不到檔案:{filename}" return msg, list_images_for_class(label), gr.update(choices=list_filenames_for_class(label), value=None) def make_zip(): if not BASE_DIR.exists(): return "目前還沒有任何資料可以打包", None if not any(BASE_DIR.rglob("*.png")): return "目前還沒有任何圖像可以打包", None zip_path = Path("my_drawings_export.zip") if zip_path.exists(): zip_path.unlink() shutil.make_archive("my_drawings_export", "zip", BASE_DIR) return "已建立 ZIP 檔,可以下載囉", str(zip_path) # ========================= # 6) Model inference # ========================= MODEL_PATH = Path("quickdraw_cnn.pt") LABEL_PATH = Path("label_map.json") device = "cpu" model = None idx_to_class = None class SmallCNN(torch.nn.Module): def __init__(self, num_classes): super().__init__() self.net = torch.nn.Sequential( torch.nn.Conv2d(1, 32, 3, padding=1), torch.nn.ReLU(), torch.nn.MaxPool2d(2), torch.nn.Conv2d(32, 64, 3, padding=1), torch.nn.ReLU(), torch.nn.MaxPool2d(2), torch.nn.Conv2d(64, 128, 3, padding=1), torch.nn.ReLU(), torch.nn.AdaptiveAvgPool2d((1, 1)), torch.nn.Flatten(), torch.nn.Dropout(0.2), torch.nn.Linear(128, num_classes), ) def forward(self, x): return self.net(x) def load_model_once(): global model, idx_to_class if model is not None: return if not MODEL_PATH.exists() or not LABEL_PATH.exists(): raise FileNotFoundError("找不到 quickdraw_cnn.pt 或 label_map.json(請先上傳到 Space 根目錄)") with open(LABEL_PATH, "r", encoding="utf-8") as f: m = json.load(f) raw = m["idx_to_class"] idx_to_class = {int(k): v for k, v in raw.items()} model_ = SmallCNN(len(idx_to_class)) state = torch.load(MODEL_PATH, map_location=device) model_.load_state_dict(state) model_.eval() model = model_ def predict_probs_dict(im): load_model_once() arr28 = preprocess_to_28x28_gray(im) x = torch.tensor(arr28, dtype=torch.float32).unsqueeze(0).unsqueeze(0) with torch.no_grad(): logits = model(x) probs = torch.softmax(logits, dim=1).squeeze(0) return {idx_to_class[i]: float(probs[i].item()) for i in range(probs.numel())} # ========================= # 7) Story helpers # ========================= def current_scene(scene_idx): return STORY[int(scene_idx)] def scene_image_path(scene_idx, img_idx): sc = current_scene(scene_idx) imgs = sc.get("images", []) if not imgs: return None i = max(0, min(int(img_idx), len(imgs) - 1)) return imgs[i] def in_draw_phase(scene_idx, img_idx): sc = current_scene(scene_idx) mode = sc.get("mode") if mode in ("intro", "ending"): return False imgs = sc.get("images", []) return (len(imgs) <= 1) or (int(img_idx) == len(imgs) - 1) def next_enabled(scene_idx, img_idx): sc = current_scene(scene_idx) imgs = sc.get("images", []) mode = sc.get("mode") if mode == "intro": return True if mode == "ending": return False return (len(imgs) > 1) and (int(img_idx) < len(imgs) - 1) def get_prompt_for_image(scene, img_idx): prompts = scene.get("prompts") if isinstance(prompts, list): i = int(img_idx) return prompts[min(i, len(prompts) - 1)] return scene.get("prompt", "") def get_status_for_image(scene, img_idx): statuses = scene.get("statuses") if isinstance(statuses, list): i = int(img_idx) return statuses[min(i, len(statuses) - 1)] return scene.get("status", "") def get_title_for_image(scene, img_idx): titles = scene.get("titles") if isinstance(titles, list): i = int(img_idx) return titles[min(i, len(titles) - 1)] return scene.get("name", "") def render_ui(scene_idx, img_idx, count, collected): sc = current_scene(scene_idx) img_idx = int(img_idx) title = get_title_for_image(sc, img_idx) img_path = scene_image_path(scene_idx, img_idx) mode = sc.get("mode", "") base_prompt = get_prompt_for_image(sc, img_idx) base_status = get_status_for_image(sc, img_idx) extra = "" if mode == "count_any": need = int(sc.get("need_count", 1)) extra = f"\n\n食材:**{int(count)}/{need}**" elif mode == "collect_all": targets = sc.get("targets", []) got = set(collected or []) checklist = " / ".join([f"{'✅' if t in got else '⬜'} {t}" for t in targets]) extra = f"\n\n收集:{checklist}" prompt_md = f"# **{title}**\n\n{base_prompt}{extra}" can_draw = in_draw_phase(scene_idx, img_idx) can_next = next_enabled(scene_idx, img_idx) guess_upd = gr.update(interactive=bool(can_draw)) next_upd = gr.update(interactive=bool(can_next)) return img_path, base_status, prompt_md, guess_upd, next_upd # ========================= # 8) Actions (Tab3) # ========================= def start_game(): scene_idx, img_idx = 0, 0 count, collected = 0, [] live = False img_path, status, prompt, guess_upd, next_upd = render_ui(scene_idx, img_idx, count, collected) return ( scene_idx, img_idx, count, collected, live, status if status else "遊戲開始!按 Next 繼續故事。", img_path, prompt, {}, # top5 清空 gr.update(value=None), # 畫布清空 guess_upd, next_upd ) def go_next(scene_idx, img_idx, count, collected, live): scene_idx = int(scene_idx) img_idx = int(img_idx) count = int(count) collected = list(collected or []) sc = current_scene(scene_idx) mode = sc.get("mode") imgs = sc.get("images", []) if mode == "ending": img_path, status2, prompt, guess_upd, next_upd = render_ui(scene_idx, img_idx, count, collected) return ( scene_idx, img_idx, count, collected, live, "🎉 遊戲已結束~", img_path, prompt, {}, gr.update(value=None), guess_upd, next_upd ) # 同 scene 多張圖:先換圖 if len(imgs) > 1 and img_idx < len(imgs) - 1: img_idx += 1 else: # 進下一個 scene scene_idx = min(scene_idx + 1, len(STORY) - 1) img_idx = 0 count = 0 collected = [] live = False img_path, status2, prompt, guess_upd, next_upd = render_ui(scene_idx, img_idx, count, collected) status_show = ( status2 if status2 else ("劇情推進中…➡️(按 Next)" if not in_draw_phase(scene_idx, img_idx) else "✏️ 請開始畫圖,按 Send 讓模型猜!") ) return ( scene_idx, img_idx, count, collected, live, status_show, img_path, prompt, {}, # top5 清空 gr.update(value=None), # 畫布清空 guess_upd, next_upd ) def guess_story(scene_idx, img_idx, count, collected, live, im): scene_idx = int(scene_idx) img_idx = int(img_idx) count = int(count) collected = list(collected or []) sc = current_scene(scene_idx) mode = sc.get("mode") # 先拿目前 UI(避免分支漏變數) img_path, status_now, prompt_now, guess_upd, next_upd = render_ui( scene_idx, img_idx, count, collected ) # intro / ending / 非可畫狀態:不能猜 if mode in ("intro", "ending") or (not in_draw_phase(scene_idx, img_idx)): msg = "⚠️ 這段是故事過場,請按 Next。" if mode != "ending" else "🎉 已完成遊戲~" return ( scene_idx, img_idx, count, collected, live, msg, # ✅ game_status {}, gr.update(value=None), img_path, prompt_now, guess_upd, next_upd ) # --- 模型推論 --- probs = predict_probs_dict(im) top5 = sorted(probs.items(), key=lambda x: x[1], reverse=True)[:5] top5_labels = [lab for lab, _ in top5] targets = sc.get("targets", []) hit = [t for t in targets if t in top5_labels] # ✅ 命中顯示(第一行) if hit: hit_text = f"✅ 命中:{', '.join(hit)}(Top-5)" else: hit_text = f"✏️ 還沒猜到(目標:{', '.join(targets)})" progressed = False if mode in ("single", "any_one"): progressed = bool(hit) elif mode == "count_any": if hit: count += 1 progressed = count >= int(sc.get("need_count", 1)) elif mode == "collect_all": if hit: for t in hit: if t not in collected: collected.append(t) progressed = all(t in collected for t in targets) # --- 未完成:留在本關(第二行 = 本關狀態 / 進度)--- if not progressed: img_path, status_now, prompt_now, guess_upd, next_upd = render_ui( scene_idx, img_idx, count, collected ) hint_text = hit_text + ("\n" + status_now if status_now else "") return ( scene_idx, img_idx, count, collected, live, hint_text, # ✅ game_status:第一行命中,第二行提示 probs, gr.update(), # top5 更新、畫布不清 img_path, prompt_now, guess_upd, next_upd ) # --- 完成:推進下一段(第二行 = 下一段狀態)--- if is_last_playable_scene(scene_idx): next_scene_idx = FINAL_SCENE_IDX else: next_scene_idx = min(scene_idx + 1, len(STORY) - 1) scene_idx = next_scene_idx img_idx = 0 count = 0 collected = [] live = False img_path, status_next, prompt_next, guess_upd, next_upd = render_ui( scene_idx, img_idx, count, collected ) hint_text = hit_text + ("\n" + status_next if status_next else "") return ( scene_idx, img_idx, count, collected, live, hint_text, # ✅ game_status:第一行命中,第二行下一段提示 probs, gr.update(value=None), # top5 顯示、畫布清空 img_path, prompt_next, guess_upd, next_upd ) # ========================= # 9) UI # ========================= CSS = """ #hint_box{ height: 32px; /* ✅ 只佔一行高度 */ overflow: hidden; /* ✅ 第二行被裁掉,但內容仍保留 \\n 分行 */ white-space: pre-line; /* ✅ \\n 保留,不會變同一行 */ padding: 6px 10px; border-radius: 8px; border: 1px solid rgba(0,0,0,0.12); } """ with gr.Blocks( title="Cook or Caught", css=""" #hint_box textarea{ height: 32px !important; overflow: hidden !important; white-space: pre-line; } """ ) as demo: gr.Markdown("# 🍳 **Cook or Caught**") ... # ---- Tab 1 ---- with gr.Tab("圖形蒐集"): gr.Markdown("### 選一個類別,畫圖並點 Submit(會存成 28×28 PNG)") with gr.Row(): with gr.Column(scale=2): label_input = gr.Dropdown(choices=CLASS_NAMES, value="key", label="類別") sketch_input = gr.Sketchpad(label="畫圖區(建議筆刷調細)") with gr.Row(): clear_btn = gr.Button("Clear", variant="secondary") submit_btn = gr.Button("Submit", variant="primary") with gr.Column(scale=1): status_output = gr.Textbox(label="狀態", interactive=False) preview_output = gr.Image(label="預覽(28×28 放大)", image_mode="L") submit_btn.click(fn=save_drawing, inputs=[label_input, sketch_input], outputs=[status_output, preview_output]) clear_btn.click(fn=clear_canvas_tab1, inputs=None, outputs=[sketch_input, status_output, preview_output]) # ---- Tab 2 ---- with gr.Tab("瀏覽資料庫"): gr.Markdown("### 選擇類別 → 載入 → 可刪單張 / 打包下載") gallery_label = gr.Dropdown(choices=CLASS_NAMES, value="key", label="類別") load_btn = gr.Button("載入該類別所有圖像", variant="primary") with gr.Row(): with gr.Column(scale=3): gallery = gr.Gallery( label="已蒐集圖像(28×28)", columns=10, height=400, allow_preview=False, ) with gr.Column(scale=1): file_radio = gr.Radio(choices=[], label="選擇要刪除的檔名") delete_btn = gr.Button("刪除選取圖片", variant="stop") delete_status = gr.Textbox(label="刪除狀態", interactive=False) gr.Markdown("---") zip_status = gr.Textbox(label="打包狀態", interactive=False) zip_file = gr.File(label="下載 ZIP(my_drawings_export.zip)") zip_btn = gr.Button("Download") load_btn.click(fn=load_class_images, inputs=gallery_label, outputs=[gallery, file_radio]) delete_btn.click(fn=delete_one_image, inputs=[gallery_label, file_radio], outputs=[delete_status, gallery, file_radio]) zip_btn.click(fn=make_zip, inputs=None, outputs=[zip_status, zip_file]) # ---- Tab 3 ---- with gr.Tab("遊戲開始!"): gr.Markdown( "按 Next 推進故事;到需要你幫忙的時候,再畫圖按 **Send🙌🏻**。\n\n" "只要正解進到 Top-5,即可成功推進劇情!" ) st_scene_idx = gr.State(0) st_img_idx = gr.State(0) st_count = gr.State(0) st_collected = gr.State([]) st_live = gr.State(False) with gr.Column(): scene_image = gr.Image( label="情境畫面", type="filepath", show_download_button=False, show_share_button=False, ) prompt_box = gr.Markdown("**提示:** Start!") # ✅ Tab3 畫布尺寸 = Tab1 game_sketch = gr.Sketchpad(label="畫圖區") with gr.Row(): start_btn = gr.Button("Start!", variant="primary") guess_btn = gr.Button("Send🙌🏻", variant="secondary") next_btn = gr.Button("Next", variant="secondary") # ✅ Hint 改成 Markdown(保留換行),高度由 CSS 控制 game_status = gr.Textbox(label="Hint💡", interactive=False, lines=2, max_lines=2) top5_bar = gr.Label(label="模型猜測(Top-5)", num_top_classes=5) start_btn.click( fn=start_game, inputs=None, outputs=[ st_scene_idx, st_img_idx, st_count, st_collected, st_live, game_status, scene_image, prompt_box, top5_bar, game_sketch, guess_btn, next_btn ], ) next_btn.click( fn=go_next, inputs=[st_scene_idx, st_img_idx, st_count, st_collected, st_live], outputs=[ st_scene_idx, st_img_idx, st_count, st_collected, st_live, game_status, scene_image, prompt_box, top5_bar, game_sketch, guess_btn, next_btn ], ) guess_btn.click( fn=guess_story, inputs=[st_scene_idx, st_img_idx, st_count, st_collected, st_live, game_sketch], outputs=[ st_scene_idx, st_img_idx, st_count, st_collected, st_live, game_status, top5_bar, game_sketch, scene_image, prompt_box, guess_btn, next_btn ], ) if __name__ == "__main__": demo.launch()