"""Laya document classifier: archive batches, resumable exports and individual checks.""" import spaces # Must precede torch, laya and all CUDA-touching imports. import hashlib import io import json from pathlib import Path, PurePosixPath import tempfile import time import zipfile import gradio as gr from PIL import Image from classifier import EXTENSIONS, MODEL_ID, POLICY_VERSION, REVISION, classify, load_agent from batch import FIELDS, export_csv # This small model is intentionally CPU-bound. Free ZeroGPU hosts require a # registered decorated function; the normal batch handlers never allocate GPU. agent = load_agent("cpu") @spaces.GPU(duration=1) def runtime_probe() -> bool: """Check that the optional ZeroGPU runtime is reachable.""" return True def check_one(image: Image.Image, threshold: float, review_confidence: float) -> tuple: """Classify a photo as a document or non-document and return its probability.""" if image is None: raise gr.Error("請先上傳圖片。") result = classify(image, threshold, review_confidence, agent) name = "文件" if result["label"] == "document" else "非文件" review = "建議人工覆核" if result["needs_review"] else "未觸發覆核門檻" text = f"## {name}\n文件機率 **{result['document_probability']:.1%}** · {review}" return text, result def batch_archive(archive: str, resume: str | None, threshold: float, review_confidence: float): """Classify all images in a ZIP, optionally resuming a matching JSONL checkpoint.""" if not archive: raise gr.Error("請上傳包含相片的 ZIP。") settings = {"model": MODEL_ID, "revision": REVISION, "policy": POLICY_VERSION, "threshold": threshold, "review_confidence": review_confidence} previous = {} if resume: try: with open(resume, encoding="utf-8") as checkpoint: for line in checkpoint: record = json.loads(line) if any(record.get(k) != v for k, v in settings.items()): raise ValueError("checkpoint settings differ") if not record.get("error") and record.get("label") in {"document", "non_document"}: previous[record["filename"]] = record except (ValueError, KeyError, TypeError) as exc: raise gr.Error("續跑檔案不完整或分類設定不同;請選擇正確的 JSONL,或移除續跑檔案。") from exc output = Path(tempfile.mkdtemp(prefix="document-batch-")) csv_path, journal_path = output / "results.csv", output / "predictions.jsonl" rows, start = [], time.monotonic() try: z = zipfile.ZipFile(archive) except zipfile.BadZipFile as exc: raise gr.Error("無法讀取 ZIP,請重新壓縮後上傳。") from exc with z, journal_path.open("w", encoding="utf-8") as journal: entries = [entry for entry in z.infolist() if not entry.is_dir() and Path(entry.filename).suffix.lower() in EXTENSIONS and not any(part.startswith((".", "__MACOSX")) for part in PurePosixPath(entry.filename).parts)] if not entries: raise gr.Error("ZIP 內沒有可處理的相片。") if len(entries) > 20000 or sum(e.file_size for e in entries) > 8 * 1024**3: raise gr.Error("每個 ZIP 最多 20,000 張、解壓後合共 8GB;請分批上傳。") yield f"準備處理 **{len(entries):,}** 張圖片…", [], None, None for index, entry in enumerate(entries, 1): fingerprint = "" try: if entry.file_size > 50 * 1024**2: raise ValueError("單張圖片超過 50MB,請縮小後重試。") data = z.read(entry) fingerprint = hashlib.sha256(data).hexdigest() old = previous.get(entry.filename) if old and old.get("sha256") == fingerprint: row = old else: with Image.open(io.BytesIO(data)) as image: result = classify(image, threshold, review_confidence, agent) row = {**settings, **result, "filename": entry.filename, "sha256": fingerprint, "error": ""} except Exception as exc: row = {**settings, "filename": entry.filename, "sha256": fingerprint, "label": "error", "needs_review": True, "error": f"{type(exc).__name__}: {exc}"} rows.append(row) journal.write(json.dumps(row, ensure_ascii=False) + "\n") journal.flush() if index == 1 or index % 25 == 0 or index == len(entries): export_csv(rows, csv_path) docs = sum(r["label"] == "document" for r in rows) errors = sum(r["label"] == "error" for r in rows) review = sum(bool(r.get("needs_review")) for r in rows) elapsed = time.monotonic() - start text = (f"已處理 **{index:,} / {len(entries):,}** 張 · 文件 {docs:,} · " f"非文件 {index-docs-errors:,} · 錯誤 {errors:,} · 建議覆核 {review:,}\n\n" f"已用 {elapsed/60:.1f} 分鐘。下方可下載目前結果及續跑檔。") if index == len(entries): text += "\n\n**本批次處理完成。**" preview = [[r.get(field, "") for field in FIELDS] for r in rows[-100:]] yield text, preview, str(csv_path), str(journal_path) with gr.Blocks(title="相片文件分類") as demo: gr.Markdown("# 相片文件分類\n批次辨識表格、收據、報告及文件截圖,保留每張圖片的文件機率。") gr.Markdown( "**5,500+ 張圖片建議使用本機批次程式**,直接讀取資料夾並自動續跑;" "這個 Space 也支援 ZIP 批次上傳。原始圖片不會加入公開程式碼倉庫。" "上傳到 Space 的圖片會在 Hugging Face 伺服器處理。" ) with gr.Accordion("分類規則與判定門檻", open=False): gr.Markdown("文件:表格、收據、報告、掃描頁面及文件截圖。非文件:人物、物件、工地相片,即使有招牌或少量文字。") threshold = gr.Slider(0.05, 0.95, value=0.5, step=0.05, label="文件判定門檻", info="文件機率高於或等於此值時,分類為文件。") review = gr.Slider(0.5, 0.99, value=0.8, step=0.01, label="覆核門檻", info="所選類別的機率低於此值時,標記為建議覆核。") with gr.Tab("批次處理"): archive = gr.File(label="相片 ZIP(可包含子資料夾)", file_types=[".zip"], type="filepath") resume = gr.File(label="續跑檔案(選填,上次下載的 predictions.jsonl)", file_types=[".jsonl"], type="filepath") with gr.Row(): batch_button = gr.Button("開始批次分類", variant="primary") stop_button = gr.Button("停止處理") status = gr.Markdown("上傳 ZIP 後開始。每 25 張更新結果,預覽顯示最近 100 張。") preview = gr.Dataframe(headers=FIELDS, datatype=["str", "str", "number", "number", "bool", "number", "str"], interactive=False, label="最近處理的圖片") with gr.Row(): csv_output = gr.File(label="分類結果 CSV") journal_output = gr.File(label="續跑檔案 JSONL") event = batch_button.click(batch_archive, [archive, resume, threshold, review], [status, preview, csv_output, journal_output], api_name="classify_batch", concurrency_limit=1, concurrency_id="model") stop_button.click(fn=None, cancels=[event]) with gr.Tab("單張檢查"): with gr.Row(): image = gr.Image(type="pil", label="圖片") with gr.Column(): single_button = gr.Button("分類這張圖片", variant="primary") single_result = gr.Markdown("上傳圖片以查看結果。") raw = gr.JSON(label="詳細結果", open=False) single_button.click(check_one, [image, threshold, review], [single_result, raw], api_name="classify", concurrency_limit=1, concurrency_id="model") gr.Examples([[str(Path(__file__).parent / "examples" / "synthetic-document.png")]], inputs=[image], label="合成文件範例(不含個人資料)", cache_examples=False) # Register the optional runtime probe without invoking it for CPU requests. probe = gr.Button(visible=False) probe.click(runtime_probe, outputs=None, api_name=False) gr.Markdown( "實驗性模型,這批相片的準確率及機率尚未經標註驗證。請優先檢查覆核項目。\n\n" "模型:[thaitea/laya-vision-201m](https://huggingface.co/thaitea/laya-vision-201m) · " "權重授權:CC BY-NC-SA 4.0(非商業) · " "[本機批次程式及說明](https://huggingface.co/spaces/seromy/laya-document-batch/tree/main)" ) if __name__ == "__main__": demo.queue(max_size=8).launch(mcp_server=True, theme=gr.themes.Default(primary_hue="teal"))