Karl Qiu
Pre-download model weights at startup and increase GPU duration
a8d5f96
Raw History Blame Contribute Delete
17.5 kB
"""
app.py β€” HuggingFace Spaces entry point for Hybrid Comment Moderation.
`import spaces` MUST be the first import so that ZeroGPU can patch
torch before any other module loads CUDA. All subsequent imports
(config, ml_classifier, llm_reviewer) inherit the patched torch.
Three tabs:
πŸ” Live Demo β€” interactive classification (ML / LLM / Hybrid)
πŸ“Š Evaluation β€” pre-computed metrics and charts from Colab
πŸ—οΈ Architecture β€” pipeline diagram and design rationale
"""
# ── ZeroGPU patching (must precede every other import) ────────
try:
import spaces # noqa: F401
except ImportError:
pass
import time
import traceback
import gradio as gr
from config import (
CONFIDENCE_LOW,
CONFIDENCE_HIGH,
DECISION_BLOCK,
DECISION_ALLOW,
LLM_MODEL_NAME,
CLASSIFIER_MODEL_NAME,
)
from ml_classifier import MLClassifier
from llm_reviewer import gpu_classify
from precomputed import (
SYSTEM_CONFIG_MD,
RESULTS_TABLE_MD,
KEY_FINDINGS_MD,
)
# ── Constants ─────────────────────────────────────────────────
_LLM_SHORT = LLM_MODEL_NAME.split("/")[-1]
# ── Load ML classifier (CPU β€” instant, always available) ──────
ml = MLClassifier()
# ══════════════════════════════════════════════════════════════
# CLASSIFICATION LOGIC
# ══════════════════════════════════════════════════════════════
def classify_comment(comment: str, strategy: str):
"""Route a comment through the selected strategy."""
if not comment.strip():
return "⚠️ *Please enter a comment first.*", ""
try:
# ── ML-only (CPU, no GPU needed) ──────────────────────
if strategy == "ML-only (CPU, no login needed)":
t0 = time.perf_counter()
decision, confidence, _ = ml.classify(comment)
latency = (time.perf_counter() - t0) * 1000
used_llm = False
stage = "ml_classifier"
reason = "ML-only baseline"
# ── LLM-only (GPU via ZeroGPU) ────────────────────────
elif strategy == "LLM-only (GPU required)":
t0 = time.perf_counter()
decision, confidence, reason = gpu_classify(comment)
latency = (time.perf_counter() - t0) * 1000
used_llm = True
stage = "llm"
# ── Hybrid (ML first, LLM only if uncertain) ─────────
else:
t0 = time.perf_counter()
decision, confidence, is_uncertain = ml.classify(comment)
if not is_uncertain:
latency = (time.perf_counter() - t0) * 1000
used_llm = False
stage = "ml_classifier"
reason = "ML classifier is confident"
else:
decision, confidence, reason = gpu_classify(comment)
latency = (time.perf_counter() - t0) * 1000
used_llm = True
stage = "llm"
# ── Format output ─────────────────────────────────────
is_blocked = (decision == DECISION_BLOCK)
verdict = "## πŸ”΄ BLOCK" if is_blocked else "## 🟒 ALLOW"
details = (
"| Field | Value |\n"
"|:------|:------|\n"
f"| **Decision** | {decision} |\n"
f"| **Confidence** | {confidence:.2%} |\n"
f"| **Stage** | {stage} |\n"
f"| **LLM Used** | {'Yes ✨' if used_llm else 'No'} |\n"
f"| **Latency** | {latency:.0f} ms |\n"
f"| **Reason** | {reason} |\n"
)
return verdict, details
except Exception as e:
tb = traceback.format_exc()
err_type = type(e).__name__
err_msg = str(e)
# ── ZeroGPU wraps the real error β†’ guide user to logs ─
gpu_hints = [
"gpu", "quota", "zero", "login", "sign",
"unauthorized", "ZeroGPUTask",
]
is_gpu_error = (
any(k.lower() in (err_type + err_msg).lower() for k in gpu_hints)
or isinstance(e, RuntimeError)
)
if is_gpu_error:
return (
"## ⚠️ GPU Unavailable",
"This can happen if:\n"
"- You are not signed in to HuggingFace "
"([sign in here](https://huggingface.co/login) β€” free)\n"
"- Your daily ZeroGPU quota is exhausted\n"
"- The model is still loading on first call "
"(try again in ~30 s)\n"
"- The GPU timed out during model loading\n\n"
"**ML-only works without GPU** β€” select "
"**ML-only (CPU, no login needed)** above.\n\n"
f"<details><summary>Error: {err_type}: {err_msg}"
f"</summary>\n\n```\n{tb}\n```\n\n"
"Check full logs: Space Settings β†’ Logs"
"\n</details>",
)
return "## ❌ Error", f"```\n{tb}\n```"
# ══════════════════════════════════════════════════════════════
# BUILD GRADIO APP
# ══════════════════════════════════════════════════════════════
with gr.Blocks(
title="Hybrid Comment Moderation",
theme=gr.themes.Soft(),
) as demo:
gr.Markdown(
"# πŸ’¬ Hybrid Comment Moderation\n"
"**CS541 AI Final Project β€” Stevens Institute of Technology**\n\n"
f"DistilBERT (fast, cheap) β†’ {_LLM_SHORT} (smart, expensive) β€” "
"only when needed."
)
with gr.Tabs():
# ══════════════════════════════════════════════════════
# TAB 1: LIVE DEMO
# ══════════════════════════════════════════════════════
with gr.Tab("πŸ” Live Demo"):
gr.Markdown(
"> πŸ’‘ **ML-only** works for everyone (CPU). \n"
"> **Hybrid** and **LLM-only** need a free HuggingFace "
"account for GPU access. \n"
f"> First LLM call loads the 14B model (~30–40 s); "
"subsequent calls take ~1–2 s.\n"
)
with gr.Row():
# ── Left: input ───────────────────────────────
with gr.Column(scale=3):
comment_box = gr.Textbox(
label="Comment",
placeholder=(
"Type or paste a comment to classify …\n"
"e.g. 'This article is a great read, "
"thanks for sharing!'"
),
lines=4,
)
strategy_radio = gr.Radio(
choices=[
"Hybrid (recommended)",
"ML-only (CPU, no login needed)",
"LLM-only (GPU required)",
],
value="Hybrid (recommended)",
label="Strategy",
info=(
"Hybrid: ML handles most, LLM reviews "
"uncertain. ML-only: DistilBERT on every "
"comment (CPU). LLM-only: "
f"{_LLM_SHORT} on every comment (GPU)."
),
)
classify_btn = gr.Button(
"πŸ” Classify",
variant="primary",
size="lg",
)
# ── Right: output ─────────────────────────────
with gr.Column(scale=2):
verdict_md = gr.Markdown(
value="*Verdict will appear here …*"
)
details_md = gr.Markdown(value="")
# ── Examples ──────────────────────────────────────
gr.Examples(
examples=[
[
"Thank you for this insightful article!",
"Hybrid (recommended)",
],
[
"You are an absolute idiot and should die.",
"Hybrid (recommended)",
],
[
"I'm not sure I agree with the second point "
"about taxes.",
"Hybrid (recommended)",
],
[
"Kill yourself you worthless piece of garbage.",
"ML-only (CPU, no login needed)",
],
[
"The data seems to support a moderate position "
"on this issue.",
"Hybrid (recommended)",
],
[
"This is the dumbest thing I've ever read, "
"the author is clueless.",
"LLM-only (GPU required)",
],
[
"Great post!",
"ML-only (CPU, no login needed)",
],
],
inputs=[comment_box, strategy_radio],
)
# ── Wire events ───────────────────────────────────
classify_btn.click(
fn=classify_comment,
inputs=[comment_box, strategy_radio],
outputs=[verdict_md, details_md],
)
comment_box.submit(
fn=classify_comment,
inputs=[comment_box, strategy_radio],
outputs=[verdict_md, details_md],
)
# ══════════════════════════════════════════════════════
# TAB 2: EVALUATION RESULTS (pre-computed in Colab)
# ══════════════════════════════════════════════════════
with gr.Tab("πŸ“Š Evaluation Results"):
gr.Markdown(
"### Pre-computed on Google Colab (T4 GPU)\n"
"All three strategies evaluated on the **same** 100 "
"balanced samples from Civil Comments."
)
gr.Markdown(SYSTEM_CONFIG_MD)
gr.Markdown("---\n### Strategy Comparison")
gr.Markdown(RESULTS_TABLE_MD)
gr.Markdown("---\n### Visual Comparison")
gr.Image(
value="assets/strategy_comparison.png",
label="Strategy Comparison (6-panel bar chart)",
show_download_button=False,
)
gr.Markdown("---\n### Hybrid Routing Distribution")
gr.Image(
value="assets/stage_distribution.png",
label="Where decisions are made in the Hybrid pipeline",
show_download_button=False,
)
gr.Markdown("---\n### Key Findings")
gr.Markdown(KEY_FINDINGS_MD)
# ══════════════════════════════════════════════════════
# TAB 3: ARCHITECTURE
# ══════════════════════════════════════════════════════
with gr.Tab("πŸ—οΈ Architecture"):
gr.Markdown(
"### Two-Stage Pipeline\n"
"\n"
"```\n"
"User Comment\n"
" |\n"
" v\n"
"+----------------------------------------+\n"
"| Stage 1: ML Classifier |\n"
"| DistilBERT (CPU, ~7 ms per comment) |\n"
"| Fine-tuned on Jigsaw toxic comments |\n"
"+-------------------+--------------------+\n"
" |\n"
" Confident?\n"
" / \\\n"
" Yes No (grey zone)\n"
" | |\n"
" v v\n"
" DONE +----------------------------------+\n"
f" | Stage 2: LLM Reviewer |\n"
f" | {_LLM_SHORT} (GPU, ~1-2 s) |\n"
" | 14B params, NF4 quantized |\n"
" +-----------------+----------------+\n"
" |\n"
" v\n"
" DONE\n"
"```\n"
)
gr.Markdown(
"---\n"
"### Confidence Routing Logic\n"
"\n"
"The ML classifier outputs a toxic probability "
"in [0, 1]. Two thresholds define three zones:\n"
"\n"
"| Toxic Probability | Zone | Action |\n"
"|:------------------|:-----|:-------|\n"
f"| **>= {CONFIDENCE_HIGH}** | High confidence "
"toxic | BLOCK (ML decides) |\n"
f"| **<= {CONFIDENCE_LOW}** | High confidence "
"safe | ALLOW (ML decides) |\n"
f"| **{CONFIDENCE_LOW} - {CONFIDENCE_HIGH}** | "
"Grey zone | Escalate to LLM |\n"
"\n"
"Only grey-zone comments cost GPU time. On the "
"evaluation set, **88%** of comments were handled "
"by the ML classifier alone.\n"
)
gr.Markdown(
"---\n"
"### Why Hybrid?\n"
"\n"
"| Property | ML-Only | LLM-Only | Hybrid |\n"
"|:---------|:--------|:---------|:-------|\n"
"| **Speed** | ~7 ms | ~1-2 s | ~7 ms (most) |\n"
"| **Cost** | $0 (CPU) | High (every comment) "
"| Low (~12% reach LLM) |\n"
"| **Nuance** | Training dist. only "
"| Broad world knowledge | Best of both |\n"
"| **Scalability** | Excellent | Poor | Good |\n"
"\n"
"> The hybrid gives **LLM-quality decisions where "
"they matter** at **ML-classifier cost for the "
"easy cases**.\n"
)
gr.Markdown(
"---\n"
"### Models\n"
"\n"
"| Component | Model | Parameters | Quantization | Serving |\n"
"|:----------|:------|:-----------|:-------------|:--------|\n"
f"| ML Classifier | `{CLASSIFIER_MODEL_NAME}` "
"| ~67 M | None (full precision) | CPU |\n"
f"| LLM Reviewer | `{LLM_MODEL_NAME}` "
"| ~14 B | NF4 (~8 GB) | ZeroGPU A100 |\n"
)
gr.Markdown(
"---\n"
"### Reference\n"
"\n"
"- **Course:** CS541 Artificial Intelligence, "
"Stevens Institute of Technology\n"
"- **Textbook:** *Understanding Deep Learning* "
"by Simon J.D. Prince\n"
"- **Dataset:** "
"[google/civil_comments]"
"(https://huggingface.co/datasets/"
"google/civil_comments)\n"
)
# ── Footer ────────────────────────────────────────────────
gr.Markdown(
"---\n"
"*CS541 AI Final Project Β· Stevens Institute of Technology Β· "
f"DistilBERT + {_LLM_SHORT} Β· ZeroGPU*"
)
# ── Launch ────────────────────────────────────────────────────
demo.launch()