Fix column headers and stale cache via demo.load()

app.py

@@ -4,6 +4,8 @@ Reads EvalEval instance-level records from a HuggingFace dataset and
 renders a sortable comparison table across models and skills.
 """
 import json
+import shutil
+import tempfile
 from pathlib import Path
 
 import gradio as gr
@@ -14,53 +16,55 @@ DATASET_REPO = "webchick/eval-results-poc"
 
 
 def load_results() -> pd.DataFrame:
-    """Download the dataset and flatten instance records into a DataFrame."""
-    local_dir = snapshot_download(repo_id=DATASET_REPO, repo_type="dataset")
-    rows = []
-
-    # Instance-level data lives in *_samples.jsonl files, one JSON object per line.
-    # The aggregate *.json files have a different schema and are intentionally skipped.
-    for jsonl_file in sorted(Path(local_dir).rglob("*_samples.jsonl")):
-        with open(jsonl_file) as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    rec = json.loads(line)
-                except json.JSONDecodeError:
-                    continue
-                if not rec.get("schema_version", "").startswith("instance_level_eval"):
-                    continue
-
-                meta = rec.get("metadata") or {}
-                perf = rec.get("performance") or {}
-                usage = rec.get("token_usage") or {}
-                ev = rec.get("evaluation") or {}
-
-                raw_prompt = (rec.get("input") or {}).get("raw", "")
-                task = (raw_prompt[:97] + "…") if len(raw_prompt) > 100 else raw_prompt
-
-                rows.append({
-                    "Namespace":      meta.get("eval_namespace", "?"),
-                    "Skill":          meta.get("skill_name", "?"),
-                    "Case":           meta.get("case_id", "?"),
-                    "Task":           task,
-                    "Model":          rec.get("model_id", "?"),
-                    "Passed":         "✅" if ev.get("is_correct") else "❌",
-                    "Latency (ms)":   round(perf.get("latency_ms") or 0, 0),
-                    "Input tokens":   usage.get("input_tokens", 0),
-                    "Output tokens":  usage.get("output_tokens", 0),
-                    "Cost (USD)":     round(float(meta.get("cost_usd") or 0), 6),
-                    "Run timestamp":  meta.get("timestamp_run", ""),
-                })
-
-    return pd.DataFrame(rows)
+    """Download the dataset fresh and flatten instance records into a DataFrame."""
+    tmp = tempfile.mkdtemp()
+    try:
+        local_dir = snapshot_download(
+            repo_id=DATASET_REPO,
+            repo_type="dataset",
+            local_dir=tmp,
+        )
+        rows = []
+        for jsonl_file in sorted(Path(local_dir).rglob("*_samples.jsonl")):
+            with open(jsonl_file) as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        rec = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+                    if not rec.get("schema_version", "").startswith("instance_level_eval"):
+                        continue
+
+                    meta = rec.get("metadata") or {}
+                    perf = rec.get("performance") or {}
+                    usage = rec.get("token_usage") or {}
+                    ev = rec.get("evaluation") or {}
+
+                    raw_prompt = (rec.get("input") or {}).get("raw", "")
+                    task = (raw_prompt[:97] + "…") if len(raw_prompt) > 100 else raw_prompt
+
+                    rows.append({
+                        "Namespace":      meta.get("eval_namespace", "?"),
+                        "Skill":          meta.get("skill_name", "?"),
+                        "Case":           meta.get("case_id", "?"),
+                        "Task":           task,
+                        "Model":          rec.get("model_id", "?"),
+                        "Passed":         "✅" if ev.get("is_correct") else "❌",
+                        "Latency (ms)":   round(perf.get("latency_ms") or 0, 0),
+                        "Input tokens":   usage.get("input_tokens", 0),
+                        "Output tokens":  usage.get("output_tokens", 0),
+                        "Cost (USD)":     round(float(meta.get("cost_usd") or 0), 6),
+                        "Run timestamp":  meta.get("timestamp_run", ""),
+                    })
+        return pd.DataFrame(rows)
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
 
 
 def build_ui():
-    df = load_results()
-
     with gr.Blocks(title="Drupal AI Eval Dashboard (POC)") as demo:
         gr.Markdown(
             "# Drupal AI Eval Dashboard (POC)\n\n"
@@ -71,48 +75,83 @@ def build_ui():
             "(https://github.com/evaleval/every_eval_ever)."
         )
 
-        if df.empty:
-            gr.Markdown("_No results found in the dataset yet._")
-        else:
-            namespaces = sorted(df["Namespace"].unique().tolist())
-            namespace_filter = gr.Dropdown(
-                choices=["(all)"] + namespaces,
-                value="(all)",
-                label="Filter by namespace",
+        namespace_filter = gr.Dropdown(
+            choices=["(all)"],
+            value="(all)",
+            label="Filter by namespace",
+        )
+        summary = gr.Markdown("_Loading results…_")
+        results_table = gr.Dataframe(interactive=False, wrap=True)
+        gr.Markdown("## Pass rate by model")
+        pass_rate_table = gr.Dataframe(interactive=False)
+
+        def filter_data(ns, df_json):
+            df = pd.read_json(df_json, orient="split")
+            filtered = df if ns == "(all)" else df[df["Namespace"] == ns]
+            if filtered.empty:
+                return "_No results found._", pd.DataFrame(), pd.DataFrame()
+            summary_text = (
+                f"**{len(filtered)} result(s)** across "
+                f"{filtered['Model'].nunique()} model(s) and "
+                f"{filtered['Case'].nunique()} case(s)."
             )
-            summary = gr.Markdown()
-            results_table = gr.Dataframe(interactive=False, wrap=True)
-            pass_rate_table = gr.Dataframe(interactive=False)
-
-            def update(ns):
-                filtered = df if ns == "(all)" else df[df["Namespace"] == ns]
-                summary_text = (
-                    f"**{len(filtered)} result(s)** across "
-                    f"{filtered['Model'].nunique()} model(s) and "
-                    f"{filtered['Case'].nunique()} case(s)."
-                )
-                filtered["_passed_int"] = (filtered["Passed"] == "✅").astype(int)
-                agg = (
-                    filtered.groupby("Model")["_passed_int"]
-                    .agg(["mean", "count"])
-                    .reset_index()
-                    .rename(columns={"mean": "Pass rate", "count": "Cases run"})
+            filtered = filtered.copy()
+            filtered["_passed_int"] = (filtered["Passed"] == "✅").astype(int)
+            agg = (
+                filtered.groupby("Model")["_passed_int"]
+                .agg(["mean", "count"])
+                .reset_index()
+                .rename(columns={"mean": "Pass rate", "count": "Cases run"})
+            )
+            agg["Pass rate"] = agg["Pass rate"].round(2)
+            return summary_text, filtered.drop(columns=["_passed_int"]), agg
+
+        # Hidden state holds the full DataFrame so filter_data is pure
+        data_store = gr.State()
+
+        def on_load():
+            df = load_results()
+            if df.empty:
+                return (
+                    gr.Dropdown(choices=["(all)"], value="(all)"),
+                    df.to_json(orient="split"),
+                    "_No results found in the dataset yet._",
+                    pd.DataFrame(),
+                    pd.DataFrame(),
                 )
-                agg["Pass rate"] = agg["Pass rate"].round(2)
-                return summary_text, filtered.drop(columns=["_passed_int"], errors="ignore"), agg
-
-            namespace_filter.change(
-                fn=update,
-                inputs=namespace_filter,
-                outputs=[summary, results_table, pass_rate_table],
+            namespaces = sorted(df["Namespace"].unique().tolist())
+            summary_text = (
+                f"**{len(df)} result(s)** across "
+                f"{df['Model'].nunique()} model(s) and "
+                f"{df['Case'].nunique()} case(s)."
+            )
+            df_copy = df.copy()
+            df_copy["_passed_int"] = (df_copy["Passed"] == "✅").astype(int)
+            agg = (
+                df_copy.groupby("Model")["_passed_int"]
+                .agg(["mean", "count"])
+                .reset_index()
+                .rename(columns={"mean": "Pass rate", "count": "Cases run"})
+            )
+            agg["Pass rate"] = agg["Pass rate"].round(2)
+            return (
+                gr.Dropdown(choices=["(all)"] + namespaces, value="(all)"),
+                df.to_json(orient="split"),
+                summary_text,
+                df,
+                agg,
             )
-            # populate on load
-            init_summary, init_table, init_agg = update("(all)")
-            summary.value = init_summary
-            results_table.value = init_table
 
-            gr.Markdown("## Pass rate by model")
-            pass_rate_table.value = init_agg
+        demo.load(
+            fn=on_load,
+            outputs=[namespace_filter, data_store, summary, results_table, pass_rate_table],
+        )
+
+        namespace_filter.change(
+            fn=filter_data,
+            inputs=[namespace_filter, data_store],
+            outputs=[summary, results_table, pass_rate_table],
+        )
 
     return demo