File size: 17,470 Bytes
fb62a11
 
 
7912365
 
 
 
fb62a11
7912365
 
 
fb62a11
 
7912365
 
 
 
 
 
fb62a11
7912365
fb62a11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7912365
a8d5f96
 
 
 
 
 
 
 
 
 
 
 
7912365
a8d5f96
fb62a11
 
7912365
 
 
 
a8d5f96
 
 
7912365
 
a8d5f96
 
 
 
fb62a11
7912365
 
fb62a11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7912365
 
 
fb62a11
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7912365
fb62a11
 
 
 
 
7912365
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fb62a11
 
 
 
 
 
 
7912365
 
fb62a11
 
 
7912365
 
 
 
 
 
fb62a11
 
7912365
 
fb62a11
 
 
 
 
 
 
 
7912365
fb62a11
7912365
 
fb62a11
 
 
 
 
 
 
 
 
 
 
 
7912365
 
fb62a11
7912365
fb62a11
7912365
fb62a11
 
 
 
 
 
 
 
7912365
 
fb62a11
 
7912365
 
fb62a11
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
"""
app.py β€” HuggingFace Spaces entry point for Hybrid Comment Moderation.

`import spaces` MUST be the first import so that ZeroGPU can patch
torch before any other module loads CUDA. All subsequent imports
(config, ml_classifier, llm_reviewer) inherit the patched torch.

Three tabs:
  πŸ” Live Demo       β€” interactive classification (ML / LLM / Hybrid)
  πŸ“Š Evaluation       β€” pre-computed metrics and charts from Colab
  πŸ—οΈ Architecture    β€” pipeline diagram and design rationale
"""

# ── ZeroGPU patching (must precede every other import) ────────
try:
    import spaces                         # noqa: F401
except ImportError:
    pass

import time
import traceback
import gradio as gr
from config import (
    CONFIDENCE_LOW,
    CONFIDENCE_HIGH,
    DECISION_BLOCK,
    DECISION_ALLOW,
    LLM_MODEL_NAME,
    CLASSIFIER_MODEL_NAME,
)
from ml_classifier import MLClassifier
from llm_reviewer import gpu_classify
from precomputed import (
    SYSTEM_CONFIG_MD,
    RESULTS_TABLE_MD,
    KEY_FINDINGS_MD,
)

# ── Constants ─────────────────────────────────────────────────
_LLM_SHORT = LLM_MODEL_NAME.split("/")[-1]

# ── Load ML classifier (CPU β€” instant, always available) ──────
ml = MLClassifier()


# ══════════════════════════════════════════════════════════════
# CLASSIFICATION LOGIC
# ══════════════════════════════════════════════════════════════

def classify_comment(comment: str, strategy: str):
    """Route a comment through the selected strategy."""
    if not comment.strip():
        return "⚠️ *Please enter a comment first.*", ""

    try:
        # ── ML-only (CPU, no GPU needed) ──────────────────────
        if strategy == "ML-only (CPU, no login needed)":
            t0 = time.perf_counter()
            decision, confidence, _ = ml.classify(comment)
            latency = (time.perf_counter() - t0) * 1000
            used_llm = False
            stage = "ml_classifier"
            reason = "ML-only baseline"

        # ── LLM-only (GPU via ZeroGPU) ────────────────────────
        elif strategy == "LLM-only (GPU required)":
            t0 = time.perf_counter()
            decision, confidence, reason = gpu_classify(comment)
            latency = (time.perf_counter() - t0) * 1000
            used_llm = True
            stage = "llm"

        # ── Hybrid (ML first, LLM only if uncertain) ─────────
        else:
            t0 = time.perf_counter()
            decision, confidence, is_uncertain = ml.classify(comment)

            if not is_uncertain:
                latency = (time.perf_counter() - t0) * 1000
                used_llm = False
                stage = "ml_classifier"
                reason = "ML classifier is confident"
            else:
                decision, confidence, reason = gpu_classify(comment)
                latency = (time.perf_counter() - t0) * 1000
                used_llm = True
                stage = "llm"

        # ── Format output ─────────────────────────────────────
        is_blocked = (decision == DECISION_BLOCK)
        verdict = "## πŸ”΄ BLOCK" if is_blocked else "## 🟒 ALLOW"

        details = (
            "| Field | Value |\n"
            "|:------|:------|\n"
            f"| **Decision** | {decision} |\n"
            f"| **Confidence** | {confidence:.2%} |\n"
            f"| **Stage** | {stage} |\n"
            f"| **LLM Used** | {'Yes ✨' if used_llm else 'No'} |\n"
            f"| **Latency** | {latency:.0f} ms |\n"
            f"| **Reason** | {reason} |\n"
        )

        return verdict, details

    except Exception as e:
        tb = traceback.format_exc()
        err_type = type(e).__name__
        err_msg = str(e)

        # ── ZeroGPU wraps the real error β†’ guide user to logs ─
        gpu_hints = [
            "gpu", "quota", "zero", "login", "sign",
            "unauthorized", "ZeroGPUTask",
        ]
        is_gpu_error = (
            any(k.lower() in (err_type + err_msg).lower() for k in gpu_hints)
            or isinstance(e, RuntimeError)
        )

        if is_gpu_error:
            return (
                "## ⚠️ GPU Unavailable",
                "This can happen if:\n"
                "- You are not signed in to HuggingFace "
                "([sign in here](https://huggingface.co/login) β€” free)\n"
                "- Your daily ZeroGPU quota is exhausted\n"
                "- The model is still loading on first call "
                "(try again in ~30 s)\n"
                "- The GPU timed out during model loading\n\n"
                "**ML-only works without GPU** β€” select "
                "**ML-only (CPU, no login needed)** above.\n\n"
                f"<details><summary>Error: {err_type}: {err_msg}"
                f"</summary>\n\n```\n{tb}\n```\n\n"
                "Check full logs: Space Settings β†’ Logs"
                "\n</details>",
            )

        return "## ❌ Error", f"```\n{tb}\n```"


# ══════════════════════════════════════════════════════════════
# BUILD GRADIO APP
# ══════════════════════════════════════════════════════════════

with gr.Blocks(
    title="Hybrid Comment Moderation",
    theme=gr.themes.Soft(),
) as demo:

    gr.Markdown(
        "# πŸ’¬ Hybrid Comment Moderation\n"
        "**CS541 AI Final Project β€” Stevens Institute of Technology**\n\n"
        f"DistilBERT (fast, cheap) β†’ {_LLM_SHORT} (smart, expensive) β€” "
        "only when needed."
    )

    with gr.Tabs():
        # ══════════════════════════════════════════════════════
        # TAB 1: LIVE DEMO
        # ══════════════════════════════════════════════════════
        with gr.Tab("πŸ” Live Demo"):
            gr.Markdown(
                "> πŸ’‘ **ML-only** works for everyone (CPU).  \n"
                "> **Hybrid** and **LLM-only** need a free HuggingFace "
                "account for GPU access.  \n"
                f"> First LLM call loads the 14B model (~30–40 s); "
                "subsequent calls take ~1–2 s.\n"
            )

            with gr.Row():
                # ── Left: input ───────────────────────────────
                with gr.Column(scale=3):
                    comment_box = gr.Textbox(
                        label="Comment",
                        placeholder=(
                            "Type or paste a comment to classify …\n"
                            "e.g. 'This article is a great read, "
                            "thanks for sharing!'"
                        ),
                        lines=4,
                    )
                    strategy_radio = gr.Radio(
                        choices=[
                            "Hybrid (recommended)",
                            "ML-only (CPU, no login needed)",
                            "LLM-only (GPU required)",
                        ],
                        value="Hybrid (recommended)",
                        label="Strategy",
                        info=(
                            "Hybrid: ML handles most, LLM reviews "
                            "uncertain. ML-only: DistilBERT on every "
                            "comment (CPU). LLM-only: "
                            f"{_LLM_SHORT} on every comment (GPU)."
                        ),
                    )
                    classify_btn = gr.Button(
                        "πŸ” Classify",
                        variant="primary",
                        size="lg",
                    )

                # ── Right: output ─────────────────────────────
                with gr.Column(scale=2):
                    verdict_md = gr.Markdown(
                        value="*Verdict will appear here …*"
                    )
                    details_md = gr.Markdown(value="")

            # ── Examples ──────────────────────────────────────
            gr.Examples(
                examples=[
                    [
                        "Thank you for this insightful article!",
                        "Hybrid (recommended)",
                    ],
                    [
                        "You are an absolute idiot and should die.",
                        "Hybrid (recommended)",
                    ],
                    [
                        "I'm not sure I agree with the second point "
                        "about taxes.",
                        "Hybrid (recommended)",
                    ],
                    [
                        "Kill yourself you worthless piece of garbage.",
                        "ML-only (CPU, no login needed)",
                    ],
                    [
                        "The data seems to support a moderate position "
                        "on this issue.",
                        "Hybrid (recommended)",
                    ],
                    [
                        "This is the dumbest thing I've ever read, "
                        "the author is clueless.",
                        "LLM-only (GPU required)",
                    ],
                    [
                        "Great post!",
                        "ML-only (CPU, no login needed)",
                    ],
                ],
                inputs=[comment_box, strategy_radio],
            )

            # ── Wire events ───────────────────────────────────
            classify_btn.click(
                fn=classify_comment,
                inputs=[comment_box, strategy_radio],
                outputs=[verdict_md, details_md],
            )
            comment_box.submit(
                fn=classify_comment,
                inputs=[comment_box, strategy_radio],
                outputs=[verdict_md, details_md],
            )

        # ══════════════════════════════════════════════════════
        # TAB 2: EVALUATION RESULTS (pre-computed in Colab)
        # ══════════════════════════════════════════════════════
        with gr.Tab("πŸ“Š Evaluation Results"):
            gr.Markdown(
                "### Pre-computed on Google Colab (T4 GPU)\n"
                "All three strategies evaluated on the **same** 100 "
                "balanced samples from Civil Comments."
            )
            gr.Markdown(SYSTEM_CONFIG_MD)

            gr.Markdown("---\n### Strategy Comparison")
            gr.Markdown(RESULTS_TABLE_MD)

            gr.Markdown("---\n### Visual Comparison")
            gr.Image(
                value="assets/strategy_comparison.png",
                label="Strategy Comparison (6-panel bar chart)",
                show_download_button=False,
            )

            gr.Markdown("---\n### Hybrid Routing Distribution")
            gr.Image(
                value="assets/stage_distribution.png",
                label="Where decisions are made in the Hybrid pipeline",
                show_download_button=False,
            )

            gr.Markdown("---\n### Key Findings")
            gr.Markdown(KEY_FINDINGS_MD)

        # ══════════════════════════════════════════════════════
        # TAB 3: ARCHITECTURE
        # ══════════════════════════════════════════════════════
        with gr.Tab("πŸ—οΈ Architecture"):

            gr.Markdown(
                "### Two-Stage Pipeline\n"
                "\n"
                "```\n"
                "User Comment\n"
                "     |\n"
                "     v\n"
                "+----------------------------------------+\n"
                "|  Stage 1: ML Classifier                |\n"
                "|  DistilBERT  (CPU, ~7 ms per comment)  |\n"
                "|  Fine-tuned on Jigsaw toxic comments   |\n"
                "+-------------------+--------------------+\n"
                "                    |\n"
                "              Confident?\n"
                "              /         \\\n"
                "           Yes           No (grey zone)\n"
                "            |             |\n"
                "            v             v\n"
                "          DONE   +----------------------------------+\n"
                f"                 |  Stage 2: LLM Reviewer           |\n"
                f"                 |  {_LLM_SHORT} (GPU, ~1-2 s)  |\n"
                "                 |  14B params, NF4 quantized       |\n"
                "                 +-----------------+----------------+\n"
                "                                   |\n"
                "                                   v\n"
                "                                 DONE\n"
                "```\n"
            )

            gr.Markdown(
                "---\n"
                "### Confidence Routing Logic\n"
                "\n"
                "The ML classifier outputs a toxic probability "
                "in [0, 1]. Two thresholds define three zones:\n"
                "\n"
                "| Toxic Probability | Zone | Action |\n"
                "|:------------------|:-----|:-------|\n"
                f"| **>= {CONFIDENCE_HIGH}** | High confidence "
                "toxic | BLOCK (ML decides) |\n"
                f"| **<= {CONFIDENCE_LOW}** | High confidence "
                "safe | ALLOW (ML decides) |\n"
                f"| **{CONFIDENCE_LOW} - {CONFIDENCE_HIGH}** | "
                "Grey zone | Escalate to LLM |\n"
                "\n"
                "Only grey-zone comments cost GPU time. On the "
                "evaluation set, **88%** of comments were handled "
                "by the ML classifier alone.\n"
            )

            gr.Markdown(
                "---\n"
                "### Why Hybrid?\n"
                "\n"
                "| Property | ML-Only | LLM-Only | Hybrid |\n"
                "|:---------|:--------|:---------|:-------|\n"
                "| **Speed** | ~7 ms | ~1-2 s | ~7 ms (most) |\n"
                "| **Cost** | $0 (CPU) | High (every comment) "
                "| Low (~12% reach LLM) |\n"
                "| **Nuance** | Training dist. only "
                "| Broad world knowledge | Best of both |\n"
                "| **Scalability** | Excellent | Poor | Good |\n"
                "\n"
                "> The hybrid gives **LLM-quality decisions where "
                "they matter** at **ML-classifier cost for the "
                "easy cases**.\n"
            )

            gr.Markdown(
                "---\n"
                "### Models\n"
                "\n"
                "| Component | Model | Parameters | Quantization | Serving |\n"
                "|:----------|:------|:-----------|:-------------|:--------|\n"
                f"| ML Classifier | `{CLASSIFIER_MODEL_NAME}` "
                "| ~67 M | None (full precision) | CPU |\n"
                f"| LLM Reviewer | `{LLM_MODEL_NAME}` "
                "| ~14 B | NF4 (~8 GB) | ZeroGPU A100 |\n"
            )

            gr.Markdown(
                "---\n"
                "### Reference\n"
                "\n"
                "- **Course:** CS541 Artificial Intelligence, "
                "Stevens Institute of Technology\n"
                "- **Textbook:** *Understanding Deep Learning* "
                "by Simon J.D. Prince\n"
                "- **Dataset:** "
                "[google/civil_comments]"
                "(https://huggingface.co/datasets/"
                "google/civil_comments)\n"
            )

    # ── Footer ────────────────────────────────────────────────
    gr.Markdown(
        "---\n"
        "*CS541 AI Final Project Β· Stevens Institute of Technology Β· "
        f"DistilBERT + {_LLM_SHORT} Β· ZeroGPU*"
    )

# ── Launch ────────────────────────────────────────────────────
demo.launch()