Create app.py

app.py

@@ -0,0 +1,207 @@
+import os, json, html, traceback
+import gradio as gr
+import pandas as pd
+import spaces
+from google import genai
+from google.genai import types
+
+import judge_prompts as J
+import trust_analysis as ta
+
+# ------------------------------------------------------------------ config
+GEMINI_KEY   = os.environ.get("GEMINI_API_KEY")
+APP_PASSWORD = os.environ.get("APP_PASSWORD", "")
+MODEL        = os.environ.get("JUDGE_MODEL", "gemini-3.1-flash-lite")
+
+MAX_SOURCE_CHARS  = 50000
+MAX_SUMMARY_CHARS = 6000
+SESSION_LIMIT     = 15
+
+client = genai.Client(api_key=GEMINI_KEY) if GEMINI_KEY else None
+
+DIMENSIONS = {
+    "Faithfulness":              ("faithfulness",   "faithfulness_score",        "unsupported_claims", ("span","type","explanation")),
+    "Overclaiming":              ("overclaiming",   "overclaiming_score",        "overclaims",         ("span","type","explanation")),
+    "Uncertainty preservation":  ("uncertainty",    "uncertainty_score",         "dropped_hedges",     ("rendered_as","source_hedge",None)),
+    "Methods & limitations":     ("methods_limits", "methods_limitations_score", None,                 None),
+}
+ALL_DIMS = list(DIMENSIONS)
+
+
+# ------------------------------------------------------------------ judge
+@spaces.GPU
+def _zeroGPU_dummy():
+    return True
+
+def parse_json(text):
+    obj, _ = json.JSONDecoder().raw_decode(text.strip())
+    return obj
+
+def judge(dimension, task, source, output):
+    prompt = J.build_prompt(dimension, task, source, output)
+    resp = client.models.generate_content(
+        model=MODEL, contents=prompt,
+        config=types.GenerateContentConfig(
+            system_instruction=J.SYSTEM_PROMPT, temperature=0,
+            response_mime_type="application/json"))
+    return parse_json(resp.text)
+
+def screen(output):
+    df = ta.screen_validity(pd.DataFrame([{"generated_output": output,
+                                           "has_output": bool(str(output).strip())}]))
+    row = df.iloc[0]
+    return bool(row["is_valid"]), str(row["degeneracy_type"])
+
+# ------------------------------------------------------------------ rendering
+def esc(s): return html.escape(str(s))
+
+def score_colour(s):
+    try: s = float(s)
+    except: return "#888"
+    return "#b5523f" if s <= 2 else "#c9a227" if s < 4 else "#2e8f6a"
+
+def highlight(summary, spans):
+    out = esc(summary)
+    for sp in sorted({s for s in spans if s and len(str(s)) > 4}, key=len, reverse=True):
+        out = out.replace(esc(sp), f"<mark style='background:#ffe0b3;color:{TEXT}'>{esc(sp)}</mark>")
+    return out.replace("\n", "<br>")
+
+TEXT = "#1a1a1a"
+MUTED = "#4b5563"
+
+def render_scores(results, valid, degen):
+    """Middle column: validity banner + one box per scored dimension."""
+    parts = []
+    if not valid:
+        parts.append(f"<div style='padding:10px;border-left:4px solid #b5523f;background:#fbeae6;"
+                     f"color:{TEXT}'>"
+                     f"<b>Validity screen: DEGENERATE</b> — flagged as <code>{esc(degen)}</code>. "
+                     f"Fault-absence dimensions (overclaiming, uncertainty) are not scored for "
+                     f"non-functional output; faithfulness is reported as minimal.</div>")
+    else:
+        parts.append(f"<div style='padding:8px;border-left:4px solid #2e8f6a;background:#eaf6f0;"
+                     f"color:{TEXT}'><b>Validity screen: functional output</b></div>")
+
+    all_spans = []
+    for label, res in results.items():
+        key, sfield, ifield, ikeys = DIMENSIONS[label]
+        if "error" in res:
+            parts.append(f"<div style='margin-top:12px;color:{TEXT}'><b>{esc(label)}</b>: "
+                         f"<span style='color:#b5523f'>could not score ({esc(res['error'])})</span></div>")
+            continue
+        score = res.get(sfield, "–")
+        col = score_colour(score)
+        head = (f"<div style='margin-top:14px;padding:8px;border:1px solid #e3e7ee;border-radius:8px;"
+                f"background:#ffffff;color:{TEXT}'>"
+                f"<span style='font-size:15px;font-weight:700'>{esc(label)}</span> "
+                f"<span style='float:right;font-weight:700;color:{col}'>{esc(score)} / 5</span>")
+        body = ""
+        if label == "Methods & limitations":
+            flags = [("Methods", res.get("methods_present")),
+                     ("Limitations", res.get("limitations_present")),
+                     ("Population", res.get("population_present"))]
+            body += f"<div style='margin-top:6px;font-size:13px;color:{TEXT}'>" + " · ".join(
+                f"{n}: {'✓' if v else '✗'}" for n, v in flags) + "</div>"
+        elif ifield:
+            issues = res.get(ifield) or []
+            if issues:
+                body += f"<ul style='margin:6px 0 0 0;font-size:13px;color:{TEXT}'>"
+                for it in issues:
+                    if not isinstance(it, dict): continue
+                    span = it.get(ikeys[0], "")
+                    all_spans.append(span)
+                    typ = it.get(ikeys[1], "") if ikeys[1] else ""
+                    exp = it.get(ikeys[2], "") if ikeys[2] else ""
+                    line = f"<li><i>“{esc(span)}”</i>"
+                    if typ: line += f" — <code>{esc(typ)}</code>"
+                    if exp: line += f": {esc(exp)}"
+                    body += line + "</li>"
+                body += "</ul>"
+            else:
+                body += f"<div style='margin-top:6px;font-size:13px;color:#2e8f6a'>No issues flagged.</div>"
+        parts.append(head + body + "</div>")
+
+    html_out = "<div style='font-family:system-ui;color:" + TEXT + "'>" + "".join(parts) + "</div>"
+    return html_out, all_spans
+
+
+def render_summary(summary, all_spans):
+    hi = highlight(summary, all_spans)
+    return (f"<div style='font-family:system-ui;color:{TEXT}'>"
+           f"<div style='padding:10px;background:#fbfaf7;border:1px solid #ece7dc;border-radius:8px;"
+           f"font-size:14px;line-height:1.6;color:{TEXT}'>{hi}</div></div>")
+
+# ------------------------------------------------------------------ main callback
+def _err(msg, remaining):
+    return f"<b style='color:#b5523f'>{esc(msg)}</b>", "", remaining
+
+def evaluate(source, summary, task, dims, remaining):
+    if client is None:
+        return _err("Server not configured: GEMINI_API_KEY secret is missing.", remaining)
+    if remaining <= 0:
+        return _err("Session evaluation limit reached. Reload the page to reset.", remaining)
+    source, summary = (source or "").strip(), (summary or "").strip()
+    if not source or not summary:
+        return _err("Please provide both a source text and an AI summary.", remaining)
+    if len(source) > MAX_SOURCE_CHARS:
+        return _err(f"Source too long ({len(source)} chars; max {MAX_SOURCE_CHARS}). Please trim.", remaining)
+    if len(summary) > MAX_SUMMARY_CHARS:
+        return _err(f"Summary too long ({len(summary)} chars; max {MAX_SUMMARY_CHARS}). Please trim.", remaining)
+    if not dims:
+        return _err("Select at least one dimension to run.", remaining)
+
+    task_key = "structured" if task.startswith("Structured") else "lay"
+    valid, degen = screen(summary)
+
+    results = {}
+    for label in dims:
+        key, sfield, *_ = DIMENSIONS[label]
+        # gate fault-absence dims on validity
+        if not valid and key in ("overclaiming", "uncertainty"):
+            continue
+        if not valid and key in ("faithfulness", "methods_limits"):
+            results[label] = {sfield: 1}
+            continue
+        try:
+            results[label] = judge(key, task_key, source, summary)
+        except Exception as e:
+            results[label] = {"error": str(e)[:120]}
+
+    scores_html, all_spans = render_scores(
+        {k: v for k, v in results.items() if k in DIMENSIONS}, valid, degen)
+    summary_html = render_summary(summary, all_spans)
+    return scores_html, summary_html, remaining - 1
+
+# ------------------------------------------------------------------ UI
+CSS = ".gradio-container{max-width:1450px!important}"
+with gr.Blocks(title="Trust Evaluation Toolkit") as demo:
+    gr.Markdown("## Biomedical Summary Trustworthiness Toolkit\n"
+                "Scores an AI-generated summary against a validated multi-dimensional trust rubric. "
+                "Paste the source article and the AI summary, choose the register and dimensions, and run.")
+    remaining = gr.State(SESSION_LIMIT)
+    with gr.Row():
+        with gr.Column(scale=4):
+            source = gr.Textbox(label="Source text (the article)", lines=12, max_lines=20)
+            summary = gr.Textbox(label="AI-generated summary", lines=8, max_lines=14)
+            task = gr.Radio(["Structured summary", "Lay summary"], value="Structured summary",
+                            label="Register")
+            dims = gr.CheckboxGroup(ALL_DIMS, value=ALL_DIMS, label="Trust dimensions (LLM-judge)")
+            btn = gr.Button("Evaluate", variant="primary")
+        with gr.Column(scale=3):
+            gr.Markdown("**Scores**")
+            scores_out = gr.HTML()
+        with gr.Column(scale=3):
+            gr.Markdown("**Summary with flagged spans**")
+            summary_out = gr.HTML()
+    btn.click(evaluate, [source, summary, task, dims, remaining], [scores_out, summary_out, remaining])
+    gr.Markdown("<small>Scores are produced by an LLM judge validated against human ratings; they are "
+                "comparative, not absolute ground truth. Access is rate-limited per session.</small>")
+
+def check_auth(username, password):
+    return bool(APP_PASSWORD) and password == APP_PASSWORD
+
+demo.launch(
+    auth=check_auth,
+    auth_message="Enter the shared access password.",
+    css=CSS,
+)