Upload 6 files

app.py

@@ -0,0 +1,356 @@
+"""Multicultural Riddles Benchmark — human evaluation Space.
+
+Self-contained Streamlit app for the Cohere Labs lead team. Recomputes the
+coverage tracker and the floor-vs-ceiling leaderboard directly from Henry's
+flat file on every load, so dropping a newer
+`data/model_riddle_results_flat.csv` (and pushing it to the Space) refreshes
+every number and chart with no code change.
+
+Single source of truth for scores = the flat file. Every non-pending row
+carries a score (human 0-3, or auto: exact_match=3, empty/placeholder=0,
+idk=1). The leaderboard is computed on COMPLETE communities only, using those
+scores for every row, with pending rows DROPPED (never judge-filled) — so
+every value traces to the flat file with zero assumptions. Provisional until
+coverage completes; every view is labelled N/44.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+import pandas as pd
+import plotly.graph_objects as go
+import streamlit as st
+
+DATA = Path(__file__).parent / "data"
+FLAT = DATA / "model_riddle_results_flat.csv"
+AUTO = DATA / "results_tidy.parquet"          # our heuristic-judge output (static)
+
+TOTAL_COMMUNITIES = 44
+EXPECTED_MODELS = 24
+RIDDLES_PER_COMMUNITY = 100
+THIN_THRESHOLD = 0.20
+STRICT_SCORE = 3.0
+LENIENT_SCORE = 2.0
+EXCLUDED_RIDDLES = {"spanish_cuba_history_016"}  # Cuba: wrong v0 text, redone in v2
+
+# Henry's model_name -> our display name (verified 24/24).
+MODEL_NAME_MAP = {
+    "Amazon-Nova-Lite": "Amazon Nova Lite v1",
+    "Amazon-Nova-Micro": "Amazon Nova Micro v1",
+    "Deepseek-v4-flash": "DeepSeek V4 Flash",
+    "EuroLLM-Instruct": "EuroLLM-22B-Instruct-2512",
+    "GPT5.5-high": "GPT5.5 [high]",
+    "GPT5.5-none": "GPT5.5 [none]",
+    "Gemini-2.5-Flash": "Gemini 2.5 Flash",
+    "Gemini-2.5-Flash-Lite": "Gemini 2.5 Flash Lite",
+    "Mistral-Large-3": "Mistral Large 3",
+    "Olmo-3-32B-Think": "OLMo-3-32B-Think",
+    "Qwen-3.5-0.8B": "Qwen/Qwen3.5-0.8B",
+    "Qwen-SEA-LION-v4.5-27B-IT": "Qwen-SEA-LION-v4.5-27B-IT",
+    "Qwen3.5-27B": "Qwen3.5 27B",
+    "claude-opus-4-7": "claude-opus-4-7 [none]",
+    "claude-opus-4-7-thinking": "claude-opus-4-7 [thinking]",
+    "command-a-plus-05-2026": "Cohere Command A Plus",
+    "deepseek-v4-pro": "deepseek-v4-pro",
+    "deepseek-v4-pro-thinking": "deepseek-v4-pro-thinking",
+    "gemini-3.1-pro-preview": "gemini-3.1-pro-preview",
+    "gemma-4-31b": "Gemma 4 31B IT",
+    "tiny-aya-earth": "Tiny-Aya earth",
+    "tiny-aya-fire": "Tiny-Aya fire",
+    "tiny-aya-global": "Tiny-Aya global",
+    "tiny-aya-water": "Tiny-Aya water",
+}
+
+FLOOR_COLOR = "#9aa0a6"
+CEIL_COLOR = "#3b6fb6"
+
+
+@st.cache_data(show_spinner="Loading Henry's flat file…")
+def load():
+    """Read + join once per file version; cache keyed on file mtime."""
+    h = pd.read_csv(FLAT, low_memory=False)
+    h = h[~h["riddle_id"].isin(EXCLUDED_RIDDLES)].copy()
+    h["model_display"] = h["model_name"].map(MODEL_NAME_MAP)
+    unmapped = sorted(h.loc[h["model_display"].isna(), "model_name"].dropna().unique())
+    if unmapped:
+        st.error(f"Unmapped model_name(s) in flat file: {unmapped}. "
+                 "Numbers withheld until the name map is updated.")
+        st.stop()
+
+    auto = pd.read_parquet(AUTO)[["riddle_id", "model_display", "judge_verdict"]]
+    h["is_human"] = h["scoring_method"].eq("human_eval")
+    h["is_pending"] = h["scoring_method"].eq("pending_human_eval")
+    h["is_correct"] = h["score"].eq(STRICT_SCORE)
+    h["is_correct_lenient"] = h["score"].ge(LENIENT_SCORE)
+    m = h.merge(auto, on=["riddle_id", "model_display"], how="left")
+    mtime = datetime.fromtimestamp(FLAT.stat().st_mtime).strftime("%Y-%m-%d %H:%M")
+    return m, mtime
+
+
+def coverage(h: pd.DataFrame) -> pd.DataFrame:
+    g = h.groupby("community")
+    cov = pd.DataFrame({
+        "human_done": g["is_human"].sum().astype(int),
+        "pending": g["is_pending"].sum().astype(int),
+    })
+    cov["human_eligible"] = cov["human_done"] + cov["pending"]
+    cov["pct_complete"] = (
+        cov["human_done"] / cov["human_eligible"].where(cov["human_eligible"] > 0) * 100
+    ).round(1)
+
+    def status(r):
+        if r["human_done"] == 0:
+            return "not_started"
+        if r["human_eligible"] == 0 or r["pct_complete"] >= 99:
+            return "complete"
+        if r["pct_complete"] < THIN_THRESHOLD * 100:
+            return "thin"
+        return "partial"
+
+    cov["status"] = cov.apply(status, axis=1)
+    return cov.sort_values(["status", "pct_complete"])
+
+
+def leaderboard(m: pd.DataFrame, communities: set, strict: bool) -> pd.DataFrame:
+    """Floor (heuristic) vs ceiling (human/Henry) on complete communities.
+
+    Ceiling uses Henry's score for every row, pending dropped (never filled).
+    Floor is the heuristic judge on the very same rows.
+    """
+    col = "is_correct" if strict else "is_correct_lenient"
+    sub = m[m["community"].isin(communities) & ~m["is_pending"]]
+    ceil = sub.groupby("model_display")[col].mean().mul(100)
+    floor = (
+        sub.assign(auto=sub["judge_verdict"].eq("correct"))
+        .groupby("model_display")["auto"].mean().mul(100)
+    )
+    n = sub.groupby("model_display").size()
+    human_share = sub.groupby("model_display")["is_human"].mean().mul(100)
+    lb = pd.DataFrame({
+        "Human (ceiling) %": ceil, "Heuristic (floor) %": floor,
+        "n": n, "human-scored %": human_share.round(0),
+    }).sort_values("Human (ceiling) %", ascending=False)
+    lb.insert(0, "rank", range(1, len(lb) + 1))
+    return lb.round(1)
+
+
+def community_difficulty(m: pd.DataFrame, communities: set, strict: bool) -> pd.DataFrame:
+    """Mean accuracy across all models per community (how hard each culture is)."""
+    col = "is_correct" if strict else "is_correct_lenient"
+    sub = m[m["community"].isin(communities) & ~m["is_pending"]]
+    human = sub.groupby("community")[col].mean().mul(100)
+    floor = (
+        sub.assign(a=sub["judge_verdict"].eq("correct"))
+        .groupby("community")["a"].mean().mul(100)
+    )
+    d = pd.DataFrame({"Human %": human, "Heuristic %": floor})
+    return d.sort_values("Human %").round(1)
+
+
+def community_kappa(m: pd.DataFrame, communities: set) -> tuple:
+    """Cohen's kappa between heuristic judge and human, per community.
+
+    Computed on human-scored rows only (where both labels exist). Measures how
+    well strict string-matching agrees with human judgement; low kappa flags
+    communities where the auto-judge is unreliable (usually non-Latin scripts).
+    """
+    r = m[m["is_human"] & m["community"].isin(communities) & m["judge_verdict"].notna()].copy()
+    r["h"] = r["score"].eq(STRICT_SCORE)
+    r["j"] = r["judge_verdict"].eq("correct")
+
+    def kappa(x):
+        po = (x["h"] == x["j"]).mean()
+        ph, pj = x["h"].mean(), x["j"].mean()
+        pe = ph * pj + (1 - ph) * (1 - pj)
+        return float("nan") if pe == 1 else (po - pe) / (1 - pe)
+
+    per = r.groupby("community").apply(kappa).rename("kappa").sort_values()
+    overall = kappa(r)
+    misses = int((~r["j"] & r["h"]).sum())        # human-correct the judge missed
+    overcredit = int((r["j"] & ~r["h"]).sum())     # judge-correct humans rejected
+    return per.round(3), overall, misses, overcredit, len(r)
+
+
+def hbar(series: pd.Series, xlabel: str, color: str, fmt: str = ".1f") -> go.Figure:
+    fig = go.Figure(go.Bar(
+        x=series.values, y=series.index, orientation="h",
+        marker_color=color,
+        hovertemplate="%{y}<br>%{x:" + fmt + "}<extra></extra>"))
+    fig.update_layout(
+        height=max(420, 24 * len(series)), margin=dict(l=10, r=10, t=10, b=40),
+        xaxis_title=xlabel, plot_bgcolor="white", font=dict(size=12))
+    fig.update_xaxes(showgrid=True, gridcolor="#eef0f2", zeroline=False)
+    return fig
+
+
+def difficulty_fig(d: pd.DataFrame) -> go.Figure:
+    fig = go.Figure()
+    for i, (c, r) in enumerate(d.iterrows()):
+        fig.add_trace(go.Scatter(
+            x=[r["Heuristic %"], r["Human %"]], y=[c, c], mode="lines",
+            line=dict(color="#d0d3d8", width=3), showlegend=False, hoverinfo="skip"))
+    fig.add_trace(go.Scatter(
+        x=d["Heuristic %"], y=d.index, mode="markers", name="Heuristic (floor)",
+        marker=dict(color=FLOOR_COLOR, size=10),
+        hovertemplate="%{y}<br>heuristic %{x:.1f}%<extra></extra>"))
+    fig.add_trace(go.Scatter(
+        x=d["Human %"], y=d.index, mode="markers", name="Human (ceiling)",
+        marker=dict(color=CEIL_COLOR, size=10),
+        hovertemplate="%{y}<br>human %{x:.1f}%<extra></extra>"))
+    fig.update_layout(
+        height=max(420, 26 * len(d)), margin=dict(l=10, r=10, t=10, b=40),
+        xaxis_title="Mean accuracy across models (%)",
+        legend=dict(orientation="h", y=1.05), plot_bgcolor="white", font=dict(size=12))
+    fig.update_xaxes(showgrid=True, gridcolor="#eef0f2", zeroline=False)
+    return fig
+
+
+def dumbbell(lb: pd.DataFrame, strict: bool) -> go.Figure:
+    d = lb.sort_values("Human (ceiling) %")
+    fig = go.Figure()
+    for i, (model, r) in enumerate(d.iterrows()):
+        fig.add_trace(go.Scatter(
+            x=[r["Heuristic (floor) %"], r["Human (ceiling) %"]], y=[model, model],
+            mode="lines", line=dict(color="#d0d3d8", width=3),
+            showlegend=False, hoverinfo="skip"))
+    fig.add_trace(go.Scatter(
+        x=d["Heuristic (floor) %"], y=d.index, mode="markers", name="Heuristic (floor)",
+        marker=dict(color=FLOOR_COLOR, size=11),
+        hovertemplate="%{y}<br>heuristic %{x:.1f}%<extra></extra>"))
+    fig.add_trace(go.Scatter(
+        x=d["Human (ceiling) %"], y=d.index, mode="markers", name="Human (ceiling)",
+        marker=dict(color=CEIL_COLOR, size=11),
+        hovertemplate="%{y}<br>human %{x:.1f}%<extra></extra>"))
+    lab = "strict (top score)" if strict else "lenient (score ≥ 2)"
+    fig.update_layout(
+        height=max(420, 26 * len(d)), margin=dict(l=10, r=10, t=10, b=40),
+        xaxis_title=f"Accuracy (%), {lab}", legend=dict(orientation="h", y=1.05),
+        plot_bgcolor="white", font=dict(size=13))
+    fig.update_xaxes(showgrid=True, gridcolor="#eef0f2", zeroline=False)
+    return fig
+
+
+# ------------------------------- UI -------------------------------
+st.set_page_config(page_title="Multicultural Riddles — Human Eval",
+                   page_icon="🧩", layout="wide")
+
+if not FLAT.exists():
+    st.error("data/model_riddle_results_flat.csv not found in the Space.")
+    st.stop()
+
+m, mtime = load()
+cov = coverage(m.drop_duplicates(["community", "riddle_id", "model_display"]))
+complete = set(cov[cov.status == "complete"].index)
+counts = cov.status.value_counts().to_dict()
+
+st.title("🧩 Multicultural Riddles Benchmark — Human Evaluation")
+st.caption(
+    f"Provisional leaderboard, live from Henry's flat file (updated {mtime}). "
+    "Every score traces to the flat file; pending rows are dropped, never "
+    "guessed. Numbers can still shift as human coverage completes.")
+
+c1, c2, c3, c4 = st.columns(4)
+c1.metric("Communities complete", f"{len(complete)} / {TOTAL_COMMUNITIES}")
+c2.metric("Models", m["model_display"].nunique())
+c3.metric("Human-scored rows", f"{int(m['is_human'].sum()):,}")
+c4.metric("Pending", f"{int(m['is_pending'].sum()):,}")
+
+strict = st.radio("Scoring", ["Strict (top score = 3)", "Lenient (score ≥ 2)"],
+                  horizontal=True) == "Strict (top score = 3)"
+
+if not complete:
+    st.warning("No community has complete human evaluation yet.")
+    st.stop()
+
+lb = leaderboard(m, complete, strict)
+
+tab_lb, tab_diff, tab_kappa, tab_cov, tab_map = st.tabs(
+    ["🏆 Leaderboard", "🌍 Community difficulty", "🤝 Judge vs human",
+     "📊 Coverage", "🗺️ Coverage map"])
+
+with tab_lb:
+    st.subheader(f"Floor → ceiling  (N={len(complete)}/{TOTAL_COMMUNITIES} "
+                 "complete communities, provisional)")
+    st.plotly_chart(dumbbell(lb, strict), use_container_width=True)
+    st.caption("Grey = heuristic auto-judge (lower bound). Blue = human "
+               "judgement (upper bound). The gap is the lift from strict "
+               "string-matching to human scoring.")
+    st.dataframe(
+        lb[["rank", "Human (ceiling) %", "Heuristic (floor) %",
+            "human-scored %", "n"]],
+        use_container_width=True)
+    st.download_button("Download leaderboard (CSV)",
+                       lb.to_csv().encode(), "human_leaderboard.csv")
+
+with tab_diff:
+    st.subheader(f"How hard is each culture?  (N={len(complete)}/"
+                 f"{TOTAL_COMMUNITIES} complete communities)")
+    d = community_difficulty(m, complete, strict)
+    st.plotly_chart(difficulty_fig(d), use_container_width=True)
+    hardest = ", ".join(d.head(3).index)
+    easiest = ", ".join(d.tail(3).index)
+    st.caption(f"Mean accuracy across all {m['model_display'].nunique()} models "
+               f"per community. Hardest: {hardest}. Easiest: {easiest}. "
+               "Grey = heuristic floor, blue = human ceiling.")
+    st.dataframe(d, use_container_width=True)
+    st.download_button("Download community difficulty (CSV)",
+                       d.to_csv().encode(), "community_difficulty.csv")
+
+with tab_kappa:
+    st.subheader("Does the auto-judge agree with humans?")
+    per, overall, misses, overcredit, n = community_kappa(m, complete)
+    k1, k2, k3 = st.columns(3)
+    k1.metric("Overall Cohen's κ", f"{overall:.3f}",
+              help="Agreement between heuristic judge and human, on human-scored rows.")
+    k2.metric("Human-correct the judge missed", f"{misses:,}",
+              help="Answers humans scored correct but strict string-matching rejected.")
+    k3.metric("Judge over-credits", f"{overcredit:,}",
+              help="Answers the judge marked correct but humans rejected.")
+    st.plotly_chart(hbar(per, "Cohen's κ (judge vs human)", CEIL_COLOR, ".3f"),
+                    use_container_width=True)
+    st.caption(
+        f"On {n:,} human-scored rows in complete communities. Low κ flags "
+        "communities where strict string-matching is unreliable (typically "
+        "non-Latin scripts and rich morphology). The judge misses far more "
+        f"correct answers ({misses:,}) than it over-credits ({overcredit:,}), "
+        "which is why the human ceiling sits well above the heuristic floor.")
+    st.download_button("Download per-community κ (CSV)",
+                       per.to_csv().encode(), "community_kappa.csv")
+
+with tab_map:
+    st.subheader("Coverage map")
+    map_img = DATA / "coverage_map.png"
+    if map_img.exists():
+        st.image(str(map_img), use_container_width=True)
+        st.caption("Coverage map — visualization by Pouya Sadeghi (Cohere Labs). "
+                   "Drop an updated image at data/coverage_map.png to replace.")
+    else:
+        st.info(
+            "No map image yet. Add Pouya Sadeghi's coverage map as "
+            "`data/coverage_map.png` and it will render here, credited. "
+            "We can later augment it by shading each community by human "
+            "accuracy from the difficulty tab.")
+
+with tab_cov:
+    st.subheader("Human-eval coverage by community")
+    order = {"complete": 0, "partial": 1, "thin": 2, "not_started": 3}
+    view = cov.reset_index().rename(columns={"index": "community"})
+    view["_o"] = view["status"].map(order)
+    view = view.sort_values(["_o", "pct_complete"], ascending=[True, False])
+    st.dataframe(
+        view[["community", "status", "pct_complete", "human_done",
+              "pending", "human_eligible"]],
+        use_container_width=True, hide_index=True)
+    chase = view[view.status.isin(["not_started", "thin"])]["community"].tolist()
+    if chase:
+        st.info("Chase list (not started / thin): " + ", ".join(chase))
+
+st.divider()
+st.caption(
+    "Methodology: single human annotator per riddle (no inter-annotator "
+    "agreement). Leaderboard restricted to communities with complete human "
+    "coverage; other communities excluded until done. Source of truth: "
+    "model_riddle_results_flat.csv (Henry). Cuba riddle "
+    "spanish_cuba_history_016 excluded (redone in v2).")

build_static.py

@@ -0,0 +1,297 @@
+"""Build a self-contained static dashboard from Henry's flat file.
+
+Free HuggingFace Static Spaces cannot run Streamlit, so this precomputes every
+metric (reusing app.py's exact functions) and bakes the numbers into a single
+self-contained index.html with Plotly from CDN.
+
+This is the human-evaluation layer of the benchmark: the *ceiling* (what models
+actually get right when a person judges) that complements the team's full-corpus
+heuristic *floor*. Views: floor->ceiling leaderboard, community difficulty,
+how-models-fail error types, judge-vs-human agreement, coverage.
+
+Refresh: drop a newer data/model_riddle_results_flat.csv, rerun this script,
+push index.html. Numbers are identical to human_eval.py / app.py.
+
+    python build_static.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import types
+from pathlib import Path
+
+# stub streamlit and exec only app.py's compute half (before the UI marker),
+# so we reuse the exact functions without running the Streamlit UI.
+_st = types.ModuleType("streamlit")
+_st.cache_data = lambda *a, **k: (lambda f: f)
+sys.modules["streamlit"] = _st
+
+_APP = Path(__file__).parent / "app.py"
+_src = _APP.read_text().split("# ------------------------------- UI")[0]
+app = types.ModuleType("app")
+app.__file__ = str(_APP)
+exec(compile(_src, str(_APP), "exec"), app.__dict__)
+
+OUT = Path(__file__).parent / "index.html"  # repo root: HF Static Space entry
+
+# ordered failure/outcome buckets + colours (how models fail, from human eval)
+FAIL_ORDER = ["Correct", "Partially correct", "Explicit uncertainty",
+              "Wrong entity", "Too broad", "Hallucinated", "Other incorrect"]
+FAIL_COLORS = {
+    "Correct": "#1a9e76", "Partially correct": "#8fcfb8",
+    "Explicit uncertainty": "#4c9be8", "Wrong entity": "#e79a3c",
+    "Too broad": "#efc84a", "Hallucinated": "#d1552f",
+    "Other incorrect": "#a7adb5",
+}
+
+
+def records(df, index_name):
+    df = df.reset_index().rename(columns={df.index.name or "index": index_name})
+    return json.loads(df.to_json(orient="records"))
+
+
+def fail_modes(m, communities: set):
+    """Per-model outcome/error-type breakdown (%), human-scored rows only."""
+    r = m[m["is_human"] & m["community"].isin(communities)].copy()
+
+    def bucket(row):
+        s, lab = row["score"], str(row.get("error_type_label") or "")
+        if s == 3:
+            return "Correct"
+        if s == 2:
+            return "Partially correct"
+        if s == 1:
+            return "Explicit uncertainty"
+        if lab.startswith("0.3"):
+            return "Hallucinated"
+        if lab.startswith("0.1"):
+            return "Too broad"
+        if lab.startswith("0.2"):
+            return "Wrong entity"
+        return "Other incorrect"
+
+    r["bucket"] = r.apply(bucket, axis=1)
+    import pandas as pd
+    piv = (pd.crosstab(r["model_display"], r["bucket"], normalize="index")
+             .mul(100).round(1))
+    for c in FAIL_ORDER:
+        if c not in piv.columns:
+            piv[c] = 0.0
+    piv = piv[FAIL_ORDER].sort_values("Correct", ascending=False)
+    return records(piv, "model")
+
+
+def main() -> None:
+    m, mtime = app.load()
+    cov = app.coverage(m.drop_duplicates(["community", "riddle_id", "model_display"]))
+    complete = set(cov[cov.status == "complete"].index)
+
+    data = {
+        "meta": {
+            "mtime": mtime, "complete": len(complete), "total": app.TOTAL_COMMUNITIES,
+            "models": int(m["model_display"].nunique()),
+            "human_rows": int(m["is_human"].sum()), "pending": int(m["is_pending"].sum()),
+        },
+        "leaderboard": {}, "difficulty": {},
+        "fail_order": FAIL_ORDER, "fail_colors": FAIL_COLORS,
+    }
+    for tag, strict in [("strict", True), ("lenient", False)]:
+        data["leaderboard"][tag] = records(app.leaderboard(m, complete, strict), "model")
+        data["difficulty"][tag] = records(app.community_difficulty(m, complete, strict), "community")
+
+    per, overall, misses, overcredit, n = app.community_kappa(m, complete)
+    data["kappa"] = {"overall": round(overall, 3), "misses": misses,
+                     "overcredit": overcredit, "n": n,
+                     "per": records(per.to_frame("kappa"), "community")}
+    data["failmodes"] = fail_modes(m, complete)
+    data["coverage"] = records(
+        cov[["status", "pct_complete", "human_done", "pending", "human_eligible"]],
+        "community")
+
+    OUT.write_text(HTML.replace("__DATA__", json.dumps(data)), encoding="utf-8")
+    print(f"Wrote {OUT}  (complete {len(complete)}/{app.TOTAL_COMMUNITIES}, "
+          f"kappa {overall:.3f})")
+
+
+HTML = r"""<!doctype html>
+<html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Multicultural Riddles — Human Evaluation</title>
+<script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
+<style>
+ :root{
+   --ceil:#2f6f9f; --floor:#b8bfc7; --ink:#14181f; --mut:#5b6673; --line:#e7eaee;
+   --bg:#fbfcfd; --card:#ffffff; --accent:#2f6f9f;
+   --font:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
+ }
+ *{box-sizing:border-box}
+ body{font-family:var(--font);color:var(--ink);background:var(--bg);margin:0;
+   line-height:1.5;-webkit-font-smoothing:antialiased}
+ .wrap{max-width:1080px;margin:0 auto;padding:32px 24px 56px}
+ header{border-bottom:1px solid var(--line);padding-bottom:18px;margin-bottom:22px}
+ h1{font-size:24px;font-weight:700;letter-spacing:-.01em;margin:0 0 6px}
+ .sub{color:var(--mut);font-size:13.5px;max-width:760px}
+ .pill{display:inline-block;background:#eaf2f8;color:#245b82;font-size:12px;
+   font-weight:600;padding:3px 10px;border-radius:999px;margin-bottom:10px}
+ .cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:12px;margin:20px 0}
+ .card{background:var(--card);border:1px solid var(--line);border-radius:12px;
+   padding:14px 16px;box-shadow:0 1px 2px rgba(20,24,31,.03)}
+ .card .v{font-size:24px;font-weight:700;letter-spacing:-.02em}
+ .card .l{color:var(--mut);font-size:12px;margin-top:2px}
+ .tabs{display:flex;gap:4px;flex-wrap:wrap;border-bottom:1px solid var(--line);margin:8px 0 6px}
+ .tab{padding:9px 14px;cursor:pointer;border:none;background:none;font-size:14px;
+   font-weight:500;color:var(--mut);border-bottom:2px solid transparent;margin-bottom:-1px}
+ .tab:hover{color:var(--ink)}
+ .tab.on{color:var(--accent);border-bottom-color:var(--accent);font-weight:600}
+ .toggle{display:flex;gap:6px;margin:16px 0 4px}
+ .toggle button{border:1px solid var(--line);background:var(--card);color:var(--mut);
+   font-size:13px;padding:6px 12px;border-radius:8px;cursor:pointer;font-family:var(--font)}
+ .toggle button.on{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
+ .pane{display:none;padding-top:10px} .pane.on{display:block}
+ .h2{font-size:16px;font-weight:600;margin:6px 0 2px}
+ .cap{color:var(--mut);font-size:12.5px;margin:8px 0 4px}
+ .dl{display:inline-block;margin-top:10px;font-size:12.5px;color:var(--accent);
+   border:1px solid var(--line);border-radius:8px;padding:5px 11px;text-decoration:none;cursor:pointer}
+ table{border-collapse:collapse;width:100%;font-size:13px;margin-top:12px}
+ th,td{border-bottom:1px solid var(--line);padding:7px 9px;text-align:left}
+ th{color:var(--mut);font-weight:600;font-size:12px;text-transform:uppercase;letter-spacing:.03em}
+ tr:hover td{background:#f7f9fb}
+ .badge{padding:2px 9px;border-radius:999px;font-size:11.5px;font-weight:600}
+ .complete{background:#e3f4e8;color:#1c7a3f}.partial{background:#fdf3e0;color:#9a6a12}
+ .thin{background:#fdeaea;color:#b23}.not_started{background:#eef0f2;color:#697}
+ .note{background:#f4f8fb;border:1px solid #dde8f1;border-radius:10px;padding:12px 14px;
+   font-size:13px;color:#2c4a63;margin-top:12px}
+ .foot{color:var(--mut);font-size:12px;margin-top:26px;padding-top:16px;
+   border-top:1px solid var(--line);line-height:1.6}
+</style></head><body><div class="wrap">
+<header>
+ <div class="pill">Human evaluation · provisional</div>
+ <h1>🧩 Multicultural Riddles Benchmark — Human Evaluation</h1>
+ <div class="sub" id="sub"></div>
+</header>
+<div class="cards" id="cards"></div>
+<div class="tabs" id="tabs"></div>
+
+<div id="lb" class="pane on">
+ <div class="h2">Heuristic floor → human ceiling</div>
+ <div class="toggle" id="toggle">
+  <button data-sc="strict" class="on">Strict (correct = top score)</button>
+  <button data-sc="lenient">Lenient (score ≥ 2)</button>
+ </div>
+ <div id="lbchart"></div>
+ <div class="cap">Grey = automatic heuristic judge (lower bound). Blue = human judgement (upper bound). The gap is the lift from strict string-matching to human scoring.</div>
+ <div id="lbtable"></div>
+ <a class="dl" id="lbdl">Download leaderboard (CSV)</a>
+</div>
+
+<div id="fail" class="pane">
+ <div class="h2">How models fail (human error types)</div>
+ <div id="failchart"></div>
+ <div class="cap" id="failcap"></div>
+</div>
+
+<div id="diff" class="pane">
+ <div class="h2">How hard is each culture?</div>
+ <div id="diffchart"></div>
+ <div class="cap" id="diffcap"></div>
+</div>
+
+<div id="kappa" class="pane">
+ <div class="h2">Does the auto-judge agree with humans?</div>
+ <div class="cards" id="kcards"></div>
+ <div id="kchart"></div>
+ <div class="cap" id="kcap"></div>
+</div>
+
+<div id="cov" class="pane">
+ <div class="h2">Human-eval coverage by community</div>
+ <div id="covtable"></div>
+ <div class="note" id="covnote"></div>
+</div>
+
+<div class="foot" id="foot"></div>
+</div>
+<script>
+const D=__DATA__;
+const CEIL="#2f6f9f",FLOOR="#b8bfc7";
+let SC="strict";
+
+document.getElementById('sub').textContent=`Live from the human-eval flat file (updated ${D.meta.mtime}). Every score traces to the flat file — pending rows are dropped, never guessed. This is the human ceiling that complements the team's full-corpus heuristic floor. Provisional; numbers shift as coverage completes.`;
+const cards=[['Communities complete',`${D.meta.complete} / ${D.meta.total}`],['Models',D.meta.models],['Human-scored rows',D.meta.human_rows.toLocaleString()],['Pending',D.meta.pending.toLocaleString()]];
+document.getElementById('cards').innerHTML=cards.map(c=>`<div class="card"><div class="v">${c[1]}</div><div class="l">${c[0]}</div></div>`).join('');
+
+const TABS=[['lb','🏆 Leaderboard'],['fail','🔍 How models fail'],['diff','🌍 Community difficulty'],['kappa','🤝 Judge vs human'],['cov','📊 Coverage']];
+document.getElementById('tabs').innerHTML=TABS.map((t,i)=>`<button class="tab${i==0?' on':''}" data-p="${t[0]}">${t[1]}</button>`).join('');
+document.querySelectorAll('.tab').forEach(b=>b.onclick=()=>{
+ document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));b.classList.add('on');
+ document.querySelectorAll('.pane').forEach(p=>p.classList.remove('on'));
+ document.getElementById(b.dataset.p).classList.add('on');window.dispatchEvent(new Event('resize'));});
+
+const L={margin:{l:10,r:16,t:8,b:44},plot_bgcolor:'#fff',paper_bgcolor:'#fff',
+ font:{family:'-apple-system,Segoe UI,Roboto,sans-serif',size:12,color:'#14181f'},
+ xaxis:{gridcolor:'#eef0f2',zeroline:false},yaxis:{automargin:true}};
+
+function dumbbell(el,rows,keyF,keyC,label,xlab){
+ const y=rows.map(label);
+ const segs=rows.map((r,i)=>({x:[r[keyF],r[keyC]],y:[y[i],y[i]],mode:'lines',line:{color:'#dfe3e8',width:3},showlegend:false,hoverinfo:'skip',type:'scatter'}));
+ const f={x:rows.map(r=>r[keyF]),y,mode:'markers',type:'scatter',name:'Heuristic (floor)',marker:{color:FLOOR,size:10,line:{color:'#9aa0a6',width:1}}};
+ const c={x:rows.map(r=>r[keyC]),y,mode:'markers',type:'scatter',name:'Human (ceiling)',marker:{color:CEIL,size:10}};
+ Plotly.newPlot(el,[...segs,f,c],{...L,height:Math.max(400,27*rows.length),xaxis:{...L.xaxis,title:xlab,range:[0,100]},legend:{orientation:'h',y:1.07}},{displayModeBar:false,responsive:true});
+}
+function renderLB(){
+ const lb=D.leaderboard[SC];
+ dumbbell('lbchart',[...lb].sort((a,b)=>a['Human (ceiling) %']-b['Human (ceiling) %']),'Heuristic (floor) %','Human (ceiling) %',r=>r.model,`Accuracy (%), ${SC}`);
+ document.getElementById('lbtable').innerHTML='<table><tr><th>#</th><th>Model</th><th>Human %</th><th>Heuristic %</th><th>human-judged %</th><th>n</th></tr>'+lb.map(r=>`<tr><td>${r.rank}</td><td>${r.model}</td><td><b>${r['Human (ceiling) %']}</b></td><td>${r['Heuristic (floor) %']}</td><td>${r['human-scored %']}</td><td>${r.n}</td></tr>`).join('')+'</table>';
+ const d=D.difficulty[SC];
+ dumbbell('diffchart',d,'Heuristic %','Human %',r=>r.community,`Mean accuracy across models (%), ${SC}`);
+ document.getElementById('diffcap').textContent=`Mean accuracy across all ${D.meta.models} models per community. Hardest: ${d.slice(0,3).map(r=>r.community).join(', ')}. Easiest: ${d.slice(-3).map(r=>r.community).join(', ')}. Grey = heuristic floor, blue = human ceiling.`;
+}
+document.querySelectorAll('.toggle button').forEach(b=>b.onclick=()=>{
+ document.querySelectorAll('.toggle button').forEach(x=>x.classList.remove('on'));b.classList.add('on');
+ SC=b.dataset.sc;renderLB();});
+renderLB();
+
+// how models fail — stacked horizontal bars, models ordered by Correct desc
+(function(){
+ const rows=[...D.failmodes].reverse();  // plotly draws bottom-up
+ const y=rows.map(r=>r.model);
+ const traces=D.fail_order.map(cat=>({
+   x:rows.map(r=>r[cat]||0),y,name:cat,type:'bar',orientation:'h',
+   marker:{color:D.fail_colors[cat]},
+   hovertemplate:'%{y}<br>'+cat+': %{x:.1f}%<extra></extra>'}));
+ Plotly.newPlot('failchart',traces,{...L,barmode:'stack',height:Math.max(440,28*rows.length),
+   xaxis:{...L.xaxis,title:'Share of human-scored responses (%)',range:[0,100]},
+   legend:{orientation:'h',y:1.06,font:{size:11}}},{displayModeBar:false,responsive:true});
+ const top=D.failmodes[0],bot=D.failmodes[D.failmodes.length-1];
+ document.getElementById('failcap').innerHTML=`How each model's answers break down when a human judges them. <b>Hallucination (red) is the dominant failure mode and grows as capability drops</b> — from ${top.Hallucinated}% (${top.model}) to ${bot.Hallucinated}% (${bot.model}). Models almost never admit uncertainty (blue, ≤1%) — they produce a confident wrong answer instead. "Wrong entity" = right category, wrong specific answer; "too broad" = a more general answer.`;
+})();
+
+// kappa
+const k=D.kappa;
+document.getElementById('kcards').innerHTML=[["Overall Cohen's κ",k.overall],['Human-correct the judge missed',k.misses.toLocaleString()],['Judge over-credits',k.overcredit.toLocaleString()]].map(c=>`<div class="card"><div class="v">${c[1]}</div><div class="l">${c[0]}</div></div>`).join('');
+const kp=[...k.per];
+Plotly.newPlot('kchart',[{x:kp.map(r=>r.kappa),y:kp.map(r=>r.community),type:'bar',orientation:'h',marker:{color:kp.map(r=>r.kappa<0.4?'#d1552f':CEIL)},hovertemplate:'%{y}<br>κ %{x:.3f}<extra></extra>'}],{...L,height:Math.max(420,26*kp.length),xaxis:{...L.xaxis,title:"Cohen's κ (judge vs human)"},shapes:[{type:'line',x0:k.overall,x1:k.overall,y0:-0.5,y1:kp.length-0.5,line:{color:'#5b6673',dash:'dash',width:1.4}}]},{displayModeBar:false,responsive:true});
+document.getElementById('kcap').innerHTML=`Two raters = the automatic judge and the human (not two humans — we have one annotator per riddle). On ${k.n.toLocaleString()} doubly-labelled rows. Red κ&lt;0.4 = string-matching unreliable (non-Latin scripts). Dashed line = overall κ ${k.overall}. The judge misses far more correct answers (${k.misses.toLocaleString()}) than it over-credits (${k.overcredit.toLocaleString()}).`;
+
+// coverage
+const ord={complete:0,partial:1,thin:2,not_started:3};
+const cv=[...D.coverage].sort((a,b)=>ord[a.status]-ord[b.status]||b.pct_complete-a.pct_complete);
+document.getElementById('covtable').innerHTML='<table><tr><th>Community</th><th>Status</th><th>% complete</th><th>human</th><th>pending</th><th>eligible</th></tr>'+cv.map(r=>`<tr><td>${r.community}</td><td><span class="badge ${r.status}">${r.status.replace('_',' ')}</span></td><td>${r.pct_complete??''}</td><td>${r.human_done}</td><td>${r.pending}</td><td>${r.human_eligible}</td></tr>`).join('')+'</table>';
+const chase=cv.filter(r=>r.status=='not_started'||r.status=='thin').map(r=>r.community);
+document.getElementById('covnote').textContent=chase.length?('Chase list (not started / thin): '+chase.join(', ')):'All started.';
+
+// leaderboard CSV download
+document.getElementById('lbdl').onclick=()=>{
+ const lb=D.leaderboard[SC],cols=['rank','model','Human (ceiling) %','Heuristic (floor) %','human-scored %','n'];
+ const csv=[cols.join(',')].concat(lb.map(r=>cols.map(c=>r[c]).join(','))).join('\n');
+ const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([csv],{type:'text/csv'}));
+ a.download='human_leaderboard_'+SC+'.csv';a.click();};
+
+document.getElementById('foot').innerHTML='Methodology: single human annotator per riddle (no inter-annotator agreement). Leaderboard and error types restricted to communities with complete human coverage. Source of truth: model_riddle_results_flat.csv (Henry Gagnier). Heuristic judge is reference-based string matching (no LLM). Cuba riddle spanish_cuba_history_016 excluded (redone in v2). Provisional until coverage completes.';
+</script></body></html>"""
+
+
+if __name__ == "__main__":
+    main()

index.html

@@ -4,84 +4,173 @@
 <title>Multicultural Riddles — Human Evaluation</title>
 <script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
 <style>
- :root{--floor:#9aa0a6;--ceil:#3b6fb6;--ink:#1c1e21;--mut:#6b7280;--line:#e5e7eb}
+ :root{
+   --ceil:#2f6f9f; --floor:#b8bfc7; --ink:#14181f; --mut:#5b6673; --line:#e7eaee;
+   --bg:#fbfcfd; --card:#ffffff; --accent:#2f6f9f;
+   --font:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
+ }
  *{box-sizing:border-box}
- body{font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;
-   color:var(--ink);margin:0;padding:24px;max-width:1040px;margin:0 auto}
- h1{font-size:22px;margin:0 0 4px} .cap{color:var(--mut);font-size:13px;margin-bottom:16px}
- .cards{display:flex;gap:12px;flex-wrap:wrap;margin:12px 0 18px}
- .card{border:1px solid var(--line);border-radius:10px;padding:10px 14px;min-width:150px}
- .card .v{font-size:22px;font-weight:600} .card .l{color:var(--mut);font-size:12px}
- .tabs{display:flex;gap:6px;flex-wrap:wrap;border-bottom:1px solid var(--line);margin-bottom:14px}
- .tab{padding:8px 12px;cursor:pointer;border:none;background:none;font-size:14px;color:var(--mut)}
- .tab.on{color:var(--ceil);border-bottom:2px solid var(--ceil);font-weight:600}
- .pane{display:none} .pane.on{display:block}
- .toggle{margin:0 0 10px} .toggle label{margin-right:14px;font-size:14px;cursor:pointer}
+ body{font-family:var(--font);color:var(--ink);background:var(--bg);margin:0;
+   line-height:1.5;-webkit-font-smoothing:antialiased}
+ .wrap{max-width:1080px;margin:0 auto;padding:32px 24px 56px}
+ header{border-bottom:1px solid var(--line);padding-bottom:18px;margin-bottom:22px}
+ h1{font-size:24px;font-weight:700;letter-spacing:-.01em;margin:0 0 6px}
+ .sub{color:var(--mut);font-size:13.5px;max-width:760px}
+ .pill{display:inline-block;background:#eaf2f8;color:#245b82;font-size:12px;
+   font-weight:600;padding:3px 10px;border-radius:999px;margin-bottom:10px}
+ .cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:12px;margin:20px 0}
+ .card{background:var(--card);border:1px solid var(--line);border-radius:12px;
+   padding:14px 16px;box-shadow:0 1px 2px rgba(20,24,31,.03)}
+ .card .v{font-size:24px;font-weight:700;letter-spacing:-.02em}
+ .card .l{color:var(--mut);font-size:12px;margin-top:2px}
+ .tabs{display:flex;gap:4px;flex-wrap:wrap;border-bottom:1px solid var(--line);margin:8px 0 6px}
+ .tab{padding:9px 14px;cursor:pointer;border:none;background:none;font-size:14px;
+   font-weight:500;color:var(--mut);border-bottom:2px solid transparent;margin-bottom:-1px}
+ .tab:hover{color:var(--ink)}
+ .tab.on{color:var(--accent);border-bottom-color:var(--accent);font-weight:600}
+ .toggle{display:flex;gap:6px;margin:16px 0 4px}
+ .toggle button{border:1px solid var(--line);background:var(--card);color:var(--mut);
+   font-size:13px;padding:6px 12px;border-radius:8px;cursor:pointer;font-family:var(--font)}
+ .toggle button.on{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600}
+ .pane{display:none;padding-top:10px} .pane.on{display:block}
+ .h2{font-size:16px;font-weight:600;margin:6px 0 2px}
+ .cap{color:var(--mut);font-size:12.5px;margin:8px 0 4px}
+ .dl{display:inline-block;margin-top:10px;font-size:12.5px;color:var(--accent);
+   border:1px solid var(--line);border-radius:8px;padding:5px 11px;text-decoration:none;cursor:pointer}
  table{border-collapse:collapse;width:100%;font-size:13px;margin-top:12px}
- th,td{border-bottom:1px solid var(--line);padding:6px 8px;text-align:left}
- th{color:var(--mut);font-weight:600} .foot{color:var(--mut);font-size:12px;margin-top:18px;line-height:1.5}
- .info{background:#eef4fb;border:1px solid #cfe0f4;border-radius:8px;padding:12px;font-size:14px;color:#274b73}
- .badge{padding:2px 8px;border-radius:10px;font-size:12px}
+ th,td{border-bottom:1px solid var(--line);padding:7px 9px;text-align:left}
+ th{color:var(--mut);font-weight:600;font-size:12px;text-transform:uppercase;letter-spacing:.03em}
+ tr:hover td{background:#f7f9fb}
+ .badge{padding:2px 9px;border-radius:999px;font-size:11.5px;font-weight:600}
  .complete{background:#e3f4e8;color:#1c7a3f}.partial{background:#fdf3e0;color:#9a6a12}
- .thin{background:#fdeaea;color:#b23}.not_started{background:#f1f1f3;color:#666}
-</style></head><body>
-<h1>🧩 Multicultural Riddles Benchmark — Human Evaluation</h1>
-<div class="cap" id="cap"></div>
+ .thin{background:#fdeaea;color:#b23}.not_started{background:#eef0f2;color:#697}
+ .note{background:#f4f8fb;border:1px solid #dde8f1;border-radius:10px;padding:12px 14px;
+   font-size:13px;color:#2c4a63;margin-top:12px}
+ .foot{color:var(--mut);font-size:12px;margin-top:26px;padding-top:16px;
+   border-top:1px solid var(--line);line-height:1.6}
+</style></head><body><div class="wrap">
+<header>
+ <div class="pill">Human evaluation · provisional</div>
+ <h1>🧩 Multicultural Riddles Benchmark — Human Evaluation</h1>
+ <div class="sub" id="sub"></div>
+</header>
 <div class="cards" id="cards"></div>
-<div class="toggle" id="toggle">
- <label><input type="radio" name="sc" value="strict" checked> Strict (top score = 3)</label>
- <label><input type="radio" name="sc" value="lenient"> Lenient (score ≥ 2)</label>
-</div>
 <div class="tabs" id="tabs"></div>
-<div id="lb" class="pane on"><div id="lbchart"></div><div class="cap" id="lbcap"></div><div id="lbtable"></div></div>
-<div id="diff" class="pane"><div id="diffchart"></div><div class="cap" id="diffcap"></div></div>
-<div id="kappa" class="pane"><div class="cards" id="kcards"></div><div id="kchart"></div><div class="cap" id="kcap"></div></div>
-<div id="cov" class="pane"><div id="covtable"></div><div class="cap" id="covcap"></div></div>
-<div id="map" class="pane"><p style="font-size:14px;margin:0 0 10px">Coverage map — concept by <b>Pouya Sadeghi</b> (Cohere Labs): cultures covered per country, across 34 countries. Hover a country for its cultures.</p><div style="height:560px; width:100%;">                            <div id="mapdiv" class="plotly-graph-div" style="height:100%; width:100%;"></div>            <script>                window.PLOTLYENV=window.PLOTLYENV || {};                                if (document.getElementById("mapdiv")) {                    Plotly.newPlot(                        "mapdiv",                        [{"colorbar":{"len":0.6,"orientation":"h","thickness":14,"ticktext":["1","2","3","4","5\u201310"],"tickvals":[1,2,3,4,5],"title":{"side":"bottom","text":"Cultures covered per country"},"y":-0.12},"colorscale":[[0.0,"rgb(247,251,255)"],[0.125,"rgb(222,235,247)"],[0.25,"rgb(198,219,239)"],[0.375,"rgb(158,202,225)"],[0.5,"rgb(107,174,214)"],[0.625,"rgb(66,146,198)"],[0.75,"rgb(33,113,181)"],[0.875,"rgb(8,81,156)"],[1.0,"rgb(8,48,107)"]],"hovertemplate":"%{text}\u003cextra\u003e\u003c\u002fextra\u003e","locationmode":"country names","locations":["Indonesia","Bangladesh","Belarus","India","Bulgaria","China","United States","Hungary","Italy","Iran","Serbia","Argentina","Uganda","Peru","Nigeria","Sri Lanka","Uzbekistan","Thailand","Syria","Pakistan","Nepal","Germany","Poland","Russia","Portugal","Brazil","Cuba","Netherlands","Canada","France","Sudan","Morocco","Kenya","Egypt"],"marker":{"line":{"color":"white","width":0.5}},"text":["\u003cb\u003eIndonesia\u003c\u002fb\u003e\u003cbr\u003e3 cultures\u003cbr\u003ebugisnese_indonesia, indonesian_indonesia, mongondow_indonesia","\u003cb\u003eBangladesh\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003ebengali_bangladesh","\u003cb\u003eBelarus\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003ebelarusian_belarus","\u003cb\u003eIndia\u003c\u002fb\u003e\u003cbr\u003e6 cultures\u003cbr\u003ebengali_india, konkani_india, malayalam_india, marathi_india, tamil_india, telugu_india","\u003cb\u003eBulgaria\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003ebulgarian_bulgaria","\u003cb\u003eChina\u003c\u002fb\u003e\u003cbr\u003e3 cultures\u003cbr\u003ecantonese_china, chinese_china, chinese_simplified_china","\u003cb\u003eUnited States\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eenglish_usa","\u003cb\u003eHungary\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003ehungarian_hungary","\u003cb\u003eItaly\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eitalian_italy","\u003cb\u003eIran\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003epersian_iran","\u003cb\u003eSerbia\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eserbian_serbia","\u003cb\u003eArgentina\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003espanish_argentina","\u003cb\u003eUganda\u003c\u002fb\u003e\u003cbr\u003e2 cultures\u003cbr\u003eluganda_uganda, lumasaba_uganda","\u003cb\u003ePeru\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003espanish_peru","\u003cb\u003eNigeria\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eyoruba_nigeria","\u003cb\u003eSri Lanka\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003esinhala_sri_lanka","\u003cb\u003eUzbekistan\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003euzbek_uzbekistan","\u003cb\u003eThailand\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003ethai_thailand","\u003cb\u003eSyria\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003earabic_syria","\u003cb\u003ePakistan\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eurdu_pakistan","\u003cb\u003eNepal\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003enepali_nepal","\u003cb\u003eGermany\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003egerman_germany","\u003cb\u003ePoland\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003epolish_poland","\u003cb\u003eRussia\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003erussian_russia","\u003cb\u003ePortugal\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eportuguese_portugal","\u003cb\u003eBrazil\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003ebrazilian_portuguese","\u003cb\u003eCuba\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003espanish_cuba","\u003cb\u003eNetherlands\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003edutch_netherlands","\u003cb\u003eCanada\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eenglish_canada","\u003cb\u003eFrance\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003efrench_france","\u003cb\u003eSudan\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003earabic_sudan","\u003cb\u003eMorocco\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003earabic_morocco","\u003cb\u003eKenya\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003eswahili_kenya","\u003cb\u003eEgypt\u003c\u002fb\u003e\u003cbr\u003e1 culture\u003cbr\u003earabic_egypt"],"z":[3,1,1,5,1,3,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"zmax":5,"zmin":1,"type":"choropleth"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermap":[{"type":"scattermap","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"geo":{"projection":{"type":"natural earth"},"showcountries":true,"countrycolor":"#ffffff","showframe":true,"framecolor":"#888","showcoastlines":false,"landcolor":"#e5e7eb"},"margin":{"l":0,"r":0,"t":0,"b":0},"height":560,"paper_bgcolor":"#fff"},                        {"displayModeBar": false, "responsive": true}                    )                };            </script>        </div><div class="cap">Darker = more of the benchmark's cultures contributed by that country. Grey countries are not represented in the benchmark. Covers all 44 v0 cultures.</div></div>
+
+<div id="lb" class="pane on">
+ <div class="h2">Heuristic floor → human ceiling</div>
+ <div class="toggle" id="toggle">
+  <button data-sc="strict" class="on">Strict (correct = top score)</button>
+  <button data-sc="lenient">Lenient (score ≥ 2)</button>
+ </div>
+ <div id="lbchart"></div>
+ <div class="cap">Grey = automatic heuristic judge (lower bound). Blue = human judgement (upper bound). The gap is the lift from strict string-matching to human scoring.</div>
+ <div id="lbtable"></div>
+ <a class="dl" id="lbdl">Download leaderboard (CSV)</a>
+</div>
+
+<div id="fail" class="pane">
+ <div class="h2">How models fail (human error types)</div>
+ <div id="failchart"></div>
+ <div class="cap" id="failcap"></div>
+</div>
+
+<div id="diff" class="pane">
+ <div class="h2">How hard is each culture?</div>
+ <div id="diffchart"></div>
+ <div class="cap" id="diffcap"></div>
+</div>
+
+<div id="kappa" class="pane">
+ <div class="h2">Does the auto-judge agree with humans?</div>
+ <div class="cards" id="kcards"></div>
+ <div id="kchart"></div>
+ <div class="cap" id="kcap"></div>
+</div>
+
+<div id="cov" class="pane">
+ <div class="h2">Human-eval coverage by community</div>
+ <div id="covtable"></div>
+ <div class="note" id="covnote"></div>
+</div>
+
 <div class="foot" id="foot"></div>
+</div>
 <script>
-const D={"meta": {"mtime": "2026-08-07 19:55", "complete": 17, "total": 44, "models": 24, "human_rows": 63947, "pending": 18062}, "leaderboard": {"strict": [{"model": "gemini-3.1-pro-preview", "rank": 1, "Human (ceiling) %": 77.9, "Heuristic (floor) %": 65.1, "n": 1700, "human-scored %": 53.0}, {"model": "GPT5.5 [none]", "rank": 2, "Human (ceiling) %": 75.9, "Heuristic (floor) %": 60.6, "n": 1700, "human-scored %": 57.0}, {"model": "GPT5.5 [high]", "rank": 3, "Human (ceiling) %": 73.9, "Heuristic (floor) %": 59.4, "n": 1700, "human-scored %": 50.0}, {"model": "claude-opus-4-7 [thinking]", "rank": 4, "Human (ceiling) %": 64.1, "Heuristic (floor) %": 51.8, "n": 1700, "human-scored %": 61.0}, {"model": "deepseek-v4-pro-thinking", "rank": 5, "Human (ceiling) %": 61.5, "Heuristic (floor) %": 50.7, "n": 1700, "human-scored %": 64.0}, {"model": "claude-opus-4-7 [none]", "rank": 6, "Human (ceiling) %": 56.8, "Heuristic (floor) %": 46.4, "n": 1700, "human-scored %": 67.0}, {"model": "deepseek-v4-pro", "rank": 7, "Human (ceiling) %": 56.6, "Heuristic (floor) %": 46.4, "n": 1699, "human-scored %": 70.0}, {"model": "Gemini 2.5 Flash", "rank": 8, "Human (ceiling) %": 45.8, "Heuristic (floor) %": 36.2, "n": 1700, "human-scored %": 78.0}, {"model": "Mistral Large 3", "rank": 9, "Human (ceiling) %": 37.4, "Heuristic (floor) %": 29.9, "n": 1700, "human-scored %": 81.0}, {"model": "DeepSeek V4 Flash", "rank": 10, "Human (ceiling) %": 34.4, "Heuristic (floor) %": 27.4, "n": 1700, "human-scored %": 83.0}, {"model": "Gemma 4 31B IT", "rank": 11, "Human (ceiling) %": 34.2, "Heuristic (floor) %": 27.1, "n": 1700, "human-scored %": 83.0}, {"model": "Gemini 2.5 Flash Lite", "rank": 12, "Human (ceiling) %": 31.5, "Heuristic (floor) %": 25.8, "n": 1700, "human-scored %": 84.0}, {"model": "Qwen-SEA-LION-v4.5-27B-IT", "rank": 13, "Human (ceiling) %": 31.2, "Heuristic (floor) %": 25.1, "n": 1700, "human-scored %": 72.0}, {"model": "Cohere Command A Plus", "rank": 14, "Human (ceiling) %": 29.1, "Heuristic (floor) %": 23.1, "n": 1700, "human-scored %": 52.0}, {"model": "Qwen3.5 27B", "rank": 15, "Human (ceiling) %": 24.6, "Heuristic (floor) %": 19.3, "n": 1700, "human-scored %": 88.0}, {"model": "Amazon Nova Lite v1", "rank": 16, "Human (ceiling) %": 12.8, "Heuristic (floor) %": 9.6, "n": 1699, "human-scored %": 94.0}, {"model": "EuroLLM-22B-Instruct-2512", "rank": 17, "Human (ceiling) %": 11.8, "Heuristic (floor) %": 9.7, "n": 1700, "human-scored %": 95.0}, {"model": "OLMo-3-32B-Think", "rank": 18, "Human (ceiling) %": 10.4, "Heuristic (floor) %": 8.2, "n": 1699, "human-scored %": 70.0}, {"model": "Amazon Nova Micro v1", "rank": 19, "Human (ceiling) %": 6.4, "Heuristic (floor) %": 5.9, "n": 1699, "human-scored %": 88.0}, {"model": "Tiny-Aya fire", "rank": 20, "Human (ceiling) %": 6.2, "Heuristic (floor) %": 4.6, "n": 1698, "human-scored %": 97.0}, {"model": "Tiny-Aya water", "rank": 21, "Human (ceiling) %": 6.2, "Heuristic (floor) %": 4.6, "n": 1699, "human-scored %": 97.0}, {"model": "Tiny-Aya earth", "rank": 22, "Human (ceiling) %": 6.1, "Heuristic (floor) %": 4.7, "n": 1698, "human-scored %": 98.0}, {"model": "Tiny-Aya global", "rank": 23, "Human (ceiling) %": 5.7, "Heuristic (floor) %": 4.9, "n": 1698, "human-scored %": 98.0}, {"model": "Qwen/Qwen3.5-0.8B", "rank": 24, "Human (ceiling) %": 0.2, "Heuristic (floor) %": 0.3, "n": 1699, "human-scored %": 100.0}], "lenient": [{"model": "gemini-3.1-pro-preview", "rank": 1, "Human (ceiling) %": 84.7, "Heuristic (floor) %": 65.1, "n": 1700, "human-scored %": 53.0}, {"model": "GPT5.5 [none]", "rank": 2, "Human (ceiling) %": 82.1, "Heuristic (floor) %": 60.6, "n": 1700, "human-scored %": 57.0}, {"model": "GPT5.5 [high]", "rank": 3, "Human (ceiling) %": 78.8, "Heuristic (floor) %": 59.4, "n": 1700, "human-scored %": 50.0}, {"model": "claude-opus-4-7 [thinking]", "rank": 4, "Human (ceiling) %": 70.6, "Heuristic (floor) %": 51.8, "n": 1700, "human-scored %": 61.0}, {"model": "deepseek-v4-pro-thinking", "rank": 5, "Human (ceiling) %": 68.8, "Heuristic (floor) %": 50.7, "n": 1700, "human-scored %": 64.0}, {"model": "deepseek-v4-pro", "rank": 6, "Human (ceiling) %": 64.3, "Heuristic (floor) %": 46.4, "n": 1699, "human-scored %": 70.0}, {"model": "claude-opus-4-7 [none]", "rank": 7, "Human (ceiling) %": 64.2, "Heuristic (floor) %": 46.4, "n": 1700, "human-scored %": 67.0}, {"model": "Gemini 2.5 Flash", "rank": 8, "Human (ceiling) %": 53.5, "Heuristic (floor) %": 36.2, "n": 1700, "human-scored %": 78.0}, {"model": "Mistral Large 3", "rank": 9, "Human (ceiling) %": 44.7, "Heuristic (floor) %": 29.9, "n": 1700, "human-scored %": 81.0}, {"model": "DeepSeek V4 Flash", "rank": 10, "Human (ceiling) %": 43.4, "Heuristic (floor) %": 27.4, "n": 1700, "human-scored %": 83.0}, {"model": "Gemma 4 31B IT", "rank": 11, "Human (ceiling) %": 42.7, "Heuristic (floor) %": 27.1, "n": 1700, "human-scored %": 83.0}, {"model": "Gemini 2.5 Flash Lite", "rank": 12, "Human (ceiling) %": 39.9, "Heuristic (floor) %": 25.8, "n": 1700, "human-scored %": 84.0}, {"model": "Qwen-SEA-LION-v4.5-27B-IT", "rank": 13, "Human (ceiling) %": 37.9, "Heuristic (floor) %": 25.1, "n": 1700, "human-scored %": 72.0}, {"model": "Cohere Command A Plus", "rank": 14, "Human (ceiling) %": 35.9, "Heuristic (floor) %": 23.1, "n": 1700, "human-scored %": 52.0}, {"model": "Qwen3.5 27B", "rank": 15, "Human (ceiling) %": 33.4, "Heuristic (floor) %": 19.3, "n": 1700, "human-scored %": 88.0}, {"model": "Amazon Nova Lite v1", "rank": 16, "Human (ceiling) %": 18.9, "Heuristic (floor) %": 9.6, "n": 1699, "human-scored %": 94.0}, {"model": "EuroLLM-22B-Instruct-2512", "rank": 17, "Human (ceiling) %": 18.7, "Heuristic (floor) %": 9.7, "n": 1700, "human-scored %": 95.0}, {"model": "OLMo-3-32B-Think", "rank": 18, "Human (ceiling) %": 13.9, "Heuristic (floor) %": 8.2, "n": 1699, "human-scored %": 70.0}, {"model": "Tiny-Aya water", "rank": 19, "Human (ceiling) %": 12.5, "Heuristic (floor) %": 4.6, "n": 1699, "human-scored %": 97.0}, {"model": "Tiny-Aya earth", "rank": 20, "Human (ceiling) %": 12.4, "Heuristic (floor) %": 4.7, "n": 1698, "human-scored %": 98.0}, {"model": "Tiny-Aya fire", "rank": 21, "Human (ceiling) %": 12.4, "Heuristic (floor) %": 4.6, "n": 1698, "human-scored %": 97.0}, {"model": "Tiny-Aya global", "rank": 22, "Human (ceiling) %": 11.4, "Heuristic (floor) %": 4.9, "n": 1698, "human-scored %": 98.0}, {"model": "Amazon Nova Micro v1", "rank": 23, "Human (ceiling) %": 10.8, "Heuristic (floor) %": 5.9, "n": 1699, "human-scored %": 88.0}, {"model": "Qwen/Qwen3.5-0.8B", "rank": 24, "Human (ceiling) %": 1.2, "Heuristic (floor) %": 0.3, "n": 1699, "human-scored %": 100.0}]}, "difficulty": {"strict": [{"community": "mongondow_indonesia", "Human %": 5.2, "Heuristic %": 4.0}, {"community": "bugisnese_indonesia", "Human %": 6.1, "Heuristic %": 3.5}, {"community": "belarusian_belarus", "Human %": 23.4, "Heuristic %": 21.9}, {"community": "malayalam_india", "Human %": 23.6, "Heuristic %": 19.8}, {"community": "hungarian_hungary", "Human %": 24.0, "Heuristic %": 21.5}, {"community": "persian_iran", "Human %": 29.4, "Heuristic %": 24.9}, {"community": "spanish_argentina", "Human %": 30.9, "Heuristic %": 26.5}, {"community": "telugu_india", "Human %": 31.2, "Heuristic %": 22.0}, {"community": "cantonese_china", "Human %": 31.5, "Heuristic %": 20.3}, {"community": "indonesian_indonesia", "Human %": 35.5, "Heuristic %": 29.5}, {"community": "serbian_serbia", "Human %": 37.1, "Heuristic %": 19.8}, {"community": "tamil_india", "Human %": 42.0, "Heuristic %": 35.9}, {"community": "bengali_bangladesh", "Human %": 44.2, "Heuristic %": 37.4}, {"community": "bulgarian_bulgaria", "Human %": 44.3, "Heuristic %": 39.0}, {"community": "marathi_india", "Human %": 44.5, "Heuristic %": 29.1}, {"community": "italian_italy", "Human %": 46.0, "Heuristic %": 41.9}, {"community": "english_usa", "Human %": 68.4, "Heuristic %": 61.3}], "lenient": [{"community": "mongondow_indonesia", "Human %": 7.4, "Heuristic %": 4.0}, {"community": "bugisnese_indonesia", "Human %": 8.6, "Heuristic %": 3.5}, {"community": "belarusian_belarus", "Human %": 25.3, "Heuristic %": 21.9}, {"community": "hungarian_hungary", "Human %": 29.0, "Heuristic %": 21.5}, {"community": "malayalam_india", "Human %": 30.1, "Heuristic %": 19.8}, {"community": "persian_iran", "Human %": 37.2, "Heuristic %": 24.9}, {"community": "spanish_argentina", "Human %": 37.4, "Heuristic %": 26.5}, {"community": "indonesian_indonesia", "Human %": 37.7, "Heuristic %": 29.5}, {"community": "serbian_serbia", "Human %": 41.7, "Heuristic %": 19.8}, {"community": "cantonese_china", "Human %": 41.8, "Heuristic %": 20.3}, {"community": "tamil_india", "Human %": 46.4, "Heuristic %": 35.9}, {"community": "italian_italy", "Human %": 48.4, "Heuristic %": 41.9}, {"community": "bengali_bangladesh", "Human %": 49.0, "Heuristic %": 37.4}, {"community": "telugu_india", "Human %": 50.7, "Heuristic %": 22.0}, {"community": "marathi_india", "Human %": 57.5, "Heuristic %": 29.1}, {"community": "bulgarian_bulgaria", "Human %": 58.4, "Heuristic %": 39.0}, {"community": "english_usa", "Human %": 71.5, "Heuristic %": 61.3}]}, "kappa": {"overall": 0.603, "misses": 3188, "overcredit": 540, "n": 31965, "per": [{"community": "cantonese_china", "kappa": 0.096}, {"community": "serbian_serbia", "kappa": 0.212}, {"community": "marathi_india", "kappa": 0.303}, {"community": "telugu_india", "kappa": 0.314}, {"community": "mongondow_indonesia", "kappa": 0.43}, {"community": "bugisnese_indonesia", "kappa": 0.448}, {"community": "malayalam_india", "kappa": 0.48}, {"community": "bulgarian_bulgaria", "kappa": 0.551}, {"community": "tamil_india", "kappa": 0.568}, {"community": "bengali_bangladesh", "kappa": 0.582}, {"community": "indonesian_indonesia", "kappa": 0.625}, {"community": "hungarian_hungary", "kappa": 0.631}, {"community": "persian_iran", "kappa": 0.636}, {"community": "english_usa", "kappa": 0.715}, {"community": "spanish_argentina", "kappa": 0.819}, {"community": "italian_italy", "kappa": 0.823}, {"community": "belarusian_belarus", "kappa": 0.828}]}, "coverage": [{"community": "bugisnese_indonesia", "status": "complete", "pct_complete": 99.7, "human_done": 1855, "pending": 5, "human_eligible": 1860}, {"community": "bengali_bangladesh", "status": "complete", "pct_complete": 99.8, "human_done": 1707, "pending": 3, "human_eligible": 1710}, {"community": "belarusian_belarus", "status": "complete", "pct_complete": 99.9, "human_done": 1910, "pending": 1, "human_eligible": 1911}, {"community": "malayalam_india", "status": "complete", "pct_complete": 99.9, "human_done": 1928, "pending": 1, "human_eligible": 1929}, {"community": "marathi_india", "status": "complete", "pct_complete": 99.9, "human_done": 1785, "pending": 2, "human_eligible": 1787}, {"community": "bulgarian_bulgaria", "status": "complete", "pct_complete": 100.0, "human_done": 1742, "pending": 0, "human_eligible": 1742}, {"community": "cantonese_china", "status": "complete", "pct_complete": 100.0, "human_done": 1834, "pending": 0, "human_eligible": 1834}, {"community": "english_usa", "status": "complete", "pct_complete": 100.0, "human_done": 1569, "pending": 0, "human_eligible": 1569}, {"community": "hungarian_hungary", "status": "complete", "pct_complete": 100.0, "human_done": 1904, "pending": 0, "human_eligible": 1904}, {"community": "indonesian_indonesia", "status": "complete", "pct_complete": 100.0, "human_done": 1914, "pending": 0, "human_eligible": 1914}, {"community": "italian_italy", "status": "complete", "pct_complete": 100.0, "human_done": 1855, "pending": 0, "human_eligible": 1855}, {"community": "mongondow_indonesia", "status": "complete", "pct_complete": 100.0, "human_done": 2113, "pending": 0, "human_eligible": 2113}, {"community": "persian_iran", "status": "complete", "pct_complete": 100.0, "human_done": 1954, "pending": 0, "human_eligible": 1954}, {"community": "serbian_serbia", "status": "complete", "pct_complete": 100.0, "human_done": 1901, "pending": 0, "human_eligible": 1901}, {"community": "spanish_argentina", "status": "complete", "pct_complete": 100.0, "human_done": 2244, "pending": 0, "human_eligible": 2244}, {"community": "tamil_india", "status": "complete", "pct_complete": 100.0, "human_done": 1818, "pending": 0, "human_eligible": 1818}, {"community": "telugu_india", "status": "complete", "pct_complete": 100.0, "human_done": 1932, "pending": 0, "human_eligible": 1932}, {"community": "luganda_uganda", "status": "not_started", "pct_complete": 0.0, "human_done": 0, "pending": 2172, "human_eligible": 2172}, {"community": "lumasaba_uganda", "status": "not_started", "pct_complete": 0.0, "human_done": 0, "pending": 2137, "human_eligible": 2137}, {"community": "spanish_peru", "status": "partial", "pct_complete": 49.6, "human_done": 909, "pending": 923, "human_eligible": 1832}, {"community": "yoruba_nigeria", "status": "partial", "pct_complete": 63.7, "human_done": 1336, "pending": 762, "human_eligible": 2098}, {"community": "sinhala_sri_lanka", "status": "partial", "pct_complete": 74.5, "human_done": 1361, "pending": 465, "human_eligible": 1826}, {"community": "uzbek_uzbekistan", "status": "partial", "pct_complete": 76.3, "human_done": 1409, "pending": 437, "human_eligible": 1846}, {"community": "thai_thailand", "status": "partial", "pct_complete": 79.6, "human_done": 1451, "pending": 373, "human_eligible": 1824}, {"community": "arabic_syria", "status": "partial", "pct_complete": 80.7, "human_done": 1650, "pending": 395, "human_eligible": 2045}, {"community": "urdu_pakistan", "status": "partial", "pct_complete": 81.1, "human_done": 1631, "pending": 381, "human_eligible": 2012}, {"community": "nepali_nepal", "status": "partial", "pct_complete": 81.2, "human_done": 1485, "pending": 343, "human_eligible": 1828}, {"community": "chinese_china", "status": "partial", "pct_complete": 81.5, "human_done": 1725, "pending": 392, "human_eligible": 2117}, {"community": "german_germany", "status": "partial", "pct_complete": 82.8, "human_done": 1550, "pending": 321, "human_eligible": 1871}, {"community": "polish_poland", "status": "partial", "pct_complete": 83.1, "human_done": 1484, "pending": 301, "human_eligible": 1785}, {"community": "konkani_india", "status": "partial", "pct_complete": 83.8, "human_done": 1751, "pending": 339, "human_eligible": 2090}, {"community": "chinese_simplified_china", "status": "partial", "pct_complete": 85.3, "human_done": 1497, "pending": 258, "human_eligible": 1755}, {"community": "russian_russia", "status": "partial", "pct_complete": 85.9, "human_done": 1563, "pending": 257, "human_eligible": 1820}, {"community": "portuguese_portugal", "status": "partial", "pct_complete": 86.5, "human_done": 1400, "pending": 219, "human_eligible": 1619}, {"community": "brazilian_portuguese", "status": "partial", "pct_complete": 89.3, "human_done": 1175, "pending": 141, "human_eligible": 1316}, {"community": "spanish_cuba", "status": "partial", "pct_complete": 89.6, "human_done": 1477, "pending": 171, "human_eligible": 1648}, {"community": "bengali_india", "status": "partial", "pct_complete": 90.9, "human_done": 1471, "pending": 147, "human_eligible": 1618}, {"community": "dutch_netherlands", "status": "partial", "pct_complete": 91.4, "human_done": 1322, "pending": 125, "human_eligible": 1447}, {"community": "english_canada", "status": "partial", "pct_complete": 94.0, "human_done": 1841, "pending": 118, "human_eligible": 1959}, {"community": "french_france", "status": "partial", "pct_complete": 94.9, "human_done": 1334, "pending": 72, "human_eligible": 1406}, {"community": "arabic_sudan", "status": "thin", "pct_complete": 13.4, "human_done": 289, "pending": 1874, "human_eligible": 2163}, {"community": "arabic_morocco", "status": "thin", "pct_complete": 14.6, "human_done": 280, "pending": 1637, "human_eligible": 1917}, {"community": "swahili_kenya", "status": "thin", "pct_complete": 14.6, "human_done": 301, "pending": 1767, "human_eligible": 2068}, {"community": "arabic_egypt", "status": "thin", "pct_complete": 16.0, "human_done": 290, "pending": 1523, "human_eligible": 1813}]};
-const FLOOR="#9aa0a6",CEIL="#3b6fb6";
-const cap=document.getElementById('cap');
-cap.textContent=`Provisional leaderboard from Henry's flat file (updated ${D.meta.mtime}). Every score traces to the flat file; pending rows dropped, never guessed. Numbers shift as coverage completes.`;
+const D={"meta": {"mtime": "2026-08-07 19:55", "complete": 17, "total": 44, "models": 24, "human_rows": 63947, "pending": 18062}, "leaderboard": {"strict": [{"model": "gemini-3.1-pro-preview", "rank": 1, "Human (ceiling) %": 77.9, "Heuristic (floor) %": 65.1, "n": 1700, "human-scored %": 53.0}, {"model": "GPT5.5 [none]", "rank": 2, "Human (ceiling) %": 75.9, "Heuristic (floor) %": 60.6, "n": 1700, "human-scored %": 57.0}, {"model": "GPT5.5 [high]", "rank": 3, "Human (ceiling) %": 73.9, "Heuristic (floor) %": 59.4, "n": 1700, "human-scored %": 50.0}, {"model": "claude-opus-4-7 [thinking]", "rank": 4, "Human (ceiling) %": 64.1, "Heuristic (floor) %": 51.8, "n": 1700, "human-scored %": 61.0}, {"model": "deepseek-v4-pro-thinking", "rank": 5, "Human (ceiling) %": 61.5, "Heuristic (floor) %": 50.7, "n": 1700, "human-scored %": 64.0}, {"model": "claude-opus-4-7 [none]", "rank": 6, "Human (ceiling) %": 56.8, "Heuristic (floor) %": 46.4, "n": 1700, "human-scored %": 67.0}, {"model": "deepseek-v4-pro", "rank": 7, "Human (ceiling) %": 56.6, "Heuristic (floor) %": 46.4, "n": 1699, "human-scored %": 70.0}, {"model": "Gemini 2.5 Flash", "rank": 8, "Human (ceiling) %": 45.8, "Heuristic (floor) %": 36.2, "n": 1700, "human-scored %": 78.0}, {"model": "Mistral Large 3", "rank": 9, "Human (ceiling) %": 37.4, "Heuristic (floor) %": 29.9, "n": 1700, "human-scored %": 81.0}, {"model": "DeepSeek V4 Flash", "rank": 10, "Human (ceiling) %": 34.4, "Heuristic (floor) %": 27.4, "n": 1700, "human-scored %": 83.0}, {"model": "Gemma 4 31B IT", "rank": 11, "Human (ceiling) %": 34.2, "Heuristic (floor) %": 27.1, "n": 1700, "human-scored %": 83.0}, {"model": "Gemini 2.5 Flash Lite", "rank": 12, "Human (ceiling) %": 31.5, "Heuristic (floor) %": 25.8, "n": 1700, "human-scored %": 84.0}, {"model": "Qwen-SEA-LION-v4.5-27B-IT", "rank": 13, "Human (ceiling) %": 31.2, "Heuristic (floor) %": 25.1, "n": 1700, "human-scored %": 72.0}, {"model": "Cohere Command A Plus", "rank": 14, "Human (ceiling) %": 29.1, "Heuristic (floor) %": 23.1, "n": 1700, "human-scored %": 52.0}, {"model": "Qwen3.5 27B", "rank": 15, "Human (ceiling) %": 24.6, "Heuristic (floor) %": 19.3, "n": 1700, "human-scored %": 88.0}, {"model": "Amazon Nova Lite v1", "rank": 16, "Human (ceiling) %": 12.8, "Heuristic (floor) %": 9.6, "n": 1699, "human-scored %": 94.0}, {"model": "EuroLLM-22B-Instruct-2512", "rank": 17, "Human (ceiling) %": 11.8, "Heuristic (floor) %": 9.7, "n": 1700, "human-scored %": 95.0}, {"model": "OLMo-3-32B-Think", "rank": 18, "Human (ceiling) %": 10.4, "Heuristic (floor) %": 8.2, "n": 1699, "human-scored %": 70.0}, {"model": "Amazon Nova Micro v1", "rank": 19, "Human (ceiling) %": 6.4, "Heuristic (floor) %": 5.9, "n": 1699, "human-scored %": 88.0}, {"model": "Tiny-Aya fire", "rank": 20, "Human (ceiling) %": 6.2, "Heuristic (floor) %": 4.6, "n": 1698, "human-scored %": 97.0}, {"model": "Tiny-Aya water", "rank": 21, "Human (ceiling) %": 6.2, "Heuristic (floor) %": 4.6, "n": 1699, "human-scored %": 97.0}, {"model": "Tiny-Aya earth", "rank": 22, "Human (ceiling) %": 6.1, "Heuristic (floor) %": 4.7, "n": 1698, "human-scored %": 98.0}, {"model": "Tiny-Aya global", "rank": 23, "Human (ceiling) %": 5.7, "Heuristic (floor) %": 4.9, "n": 1698, "human-scored %": 98.0}, {"model": "Qwen/Qwen3.5-0.8B", "rank": 24, "Human (ceiling) %": 0.2, "Heuristic (floor) %": 0.3, "n": 1699, "human-scored %": 100.0}], "lenient": [{"model": "gemini-3.1-pro-preview", "rank": 1, "Human (ceiling) %": 84.7, "Heuristic (floor) %": 65.1, "n": 1700, "human-scored %": 53.0}, {"model": "GPT5.5 [none]", "rank": 2, "Human (ceiling) %": 82.1, "Heuristic (floor) %": 60.6, "n": 1700, "human-scored %": 57.0}, {"model": "GPT5.5 [high]", "rank": 3, "Human (ceiling) %": 78.8, "Heuristic (floor) %": 59.4, "n": 1700, "human-scored %": 50.0}, {"model": "claude-opus-4-7 [thinking]", "rank": 4, "Human (ceiling) %": 70.6, "Heuristic (floor) %": 51.8, "n": 1700, "human-scored %": 61.0}, {"model": "deepseek-v4-pro-thinking", "rank": 5, "Human (ceiling) %": 68.8, "Heuristic (floor) %": 50.7, "n": 1700, "human-scored %": 64.0}, {"model": "deepseek-v4-pro", "rank": 6, "Human (ceiling) %": 64.3, "Heuristic (floor) %": 46.4, "n": 1699, "human-scored %": 70.0}, {"model": "claude-opus-4-7 [none]", "rank": 7, "Human (ceiling) %": 64.2, "Heuristic (floor) %": 46.4, "n": 1700, "human-scored %": 67.0}, {"model": "Gemini 2.5 Flash", "rank": 8, "Human (ceiling) %": 53.5, "Heuristic (floor) %": 36.2, "n": 1700, "human-scored %": 78.0}, {"model": "Mistral Large 3", "rank": 9, "Human (ceiling) %": 44.7, "Heuristic (floor) %": 29.9, "n": 1700, "human-scored %": 81.0}, {"model": "DeepSeek V4 Flash", "rank": 10, "Human (ceiling) %": 43.4, "Heuristic (floor) %": 27.4, "n": 1700, "human-scored %": 83.0}, {"model": "Gemma 4 31B IT", "rank": 11, "Human (ceiling) %": 42.7, "Heuristic (floor) %": 27.1, "n": 1700, "human-scored %": 83.0}, {"model": "Gemini 2.5 Flash Lite", "rank": 12, "Human (ceiling) %": 39.9, "Heuristic (floor) %": 25.8, "n": 1700, "human-scored %": 84.0}, {"model": "Qwen-SEA-LION-v4.5-27B-IT", "rank": 13, "Human (ceiling) %": 37.9, "Heuristic (floor) %": 25.1, "n": 1700, "human-scored %": 72.0}, {"model": "Cohere Command A Plus", "rank": 14, "Human (ceiling) %": 35.9, "Heuristic (floor) %": 23.1, "n": 1700, "human-scored %": 52.0}, {"model": "Qwen3.5 27B", "rank": 15, "Human (ceiling) %": 33.4, "Heuristic (floor) %": 19.3, "n": 1700, "human-scored %": 88.0}, {"model": "Amazon Nova Lite v1", "rank": 16, "Human (ceiling) %": 18.9, "Heuristic (floor) %": 9.6, "n": 1699, "human-scored %": 94.0}, {"model": "EuroLLM-22B-Instruct-2512", "rank": 17, "Human (ceiling) %": 18.7, "Heuristic (floor) %": 9.7, "n": 1700, "human-scored %": 95.0}, {"model": "OLMo-3-32B-Think", "rank": 18, "Human (ceiling) %": 13.9, "Heuristic (floor) %": 8.2, "n": 1699, "human-scored %": 70.0}, {"model": "Tiny-Aya water", "rank": 19, "Human (ceiling) %": 12.5, "Heuristic (floor) %": 4.6, "n": 1699, "human-scored %": 97.0}, {"model": "Tiny-Aya earth", "rank": 20, "Human (ceiling) %": 12.4, "Heuristic (floor) %": 4.7, "n": 1698, "human-scored %": 98.0}, {"model": "Tiny-Aya fire", "rank": 21, "Human (ceiling) %": 12.4, "Heuristic (floor) %": 4.6, "n": 1698, "human-scored %": 97.0}, {"model": "Tiny-Aya global", "rank": 22, "Human (ceiling) %": 11.4, "Heuristic (floor) %": 4.9, "n": 1698, "human-scored %": 98.0}, {"model": "Amazon Nova Micro v1", "rank": 23, "Human (ceiling) %": 10.8, "Heuristic (floor) %": 5.9, "n": 1699, "human-scored %": 88.0}, {"model": "Qwen/Qwen3.5-0.8B", "rank": 24, "Human (ceiling) %": 1.2, "Heuristic (floor) %": 0.3, "n": 1699, "human-scored %": 100.0}]}, "difficulty": {"strict": [{"community": "mongondow_indonesia", "Human %": 5.2, "Heuristic %": 4.0}, {"community": "bugisnese_indonesia", "Human %": 6.1, "Heuristic %": 3.5}, {"community": "belarusian_belarus", "Human %": 23.4, "Heuristic %": 21.9}, {"community": "malayalam_india", "Human %": 23.6, "Heuristic %": 19.8}, {"community": "hungarian_hungary", "Human %": 24.0, "Heuristic %": 21.5}, {"community": "persian_iran", "Human %": 29.4, "Heuristic %": 24.9}, {"community": "spanish_argentina", "Human %": 30.9, "Heuristic %": 26.5}, {"community": "telugu_india", "Human %": 31.2, "Heuristic %": 22.0}, {"community": "cantonese_china", "Human %": 31.5, "Heuristic %": 20.3}, {"community": "indonesian_indonesia", "Human %": 35.5, "Heuristic %": 29.5}, {"community": "serbian_serbia", "Human %": 37.1, "Heuristic %": 19.8}, {"community": "tamil_india", "Human %": 42.0, "Heuristic %": 35.9}, {"community": "bengali_bangladesh", "Human %": 44.2, "Heuristic %": 37.4}, {"community": "bulgarian_bulgaria", "Human %": 44.3, "Heuristic %": 39.0}, {"community": "marathi_india", "Human %": 44.5, "Heuristic %": 29.1}, {"community": "italian_italy", "Human %": 46.0, "Heuristic %": 41.9}, {"community": "english_usa", "Human %": 68.4, "Heuristic %": 61.3}], "lenient": [{"community": "mongondow_indonesia", "Human %": 7.4, "Heuristic %": 4.0}, {"community": "bugisnese_indonesia", "Human %": 8.6, "Heuristic %": 3.5}, {"community": "belarusian_belarus", "Human %": 25.3, "Heuristic %": 21.9}, {"community": "hungarian_hungary", "Human %": 29.0, "Heuristic %": 21.5}, {"community": "malayalam_india", "Human %": 30.1, "Heuristic %": 19.8}, {"community": "persian_iran", "Human %": 37.2, "Heuristic %": 24.9}, {"community": "spanish_argentina", "Human %": 37.4, "Heuristic %": 26.5}, {"community": "indonesian_indonesia", "Human %": 37.7, "Heuristic %": 29.5}, {"community": "serbian_serbia", "Human %": 41.7, "Heuristic %": 19.8}, {"community": "cantonese_china", "Human %": 41.8, "Heuristic %": 20.3}, {"community": "tamil_india", "Human %": 46.4, "Heuristic %": 35.9}, {"community": "italian_italy", "Human %": 48.4, "Heuristic %": 41.9}, {"community": "bengali_bangladesh", "Human %": 49.0, "Heuristic %": 37.4}, {"community": "telugu_india", "Human %": 50.7, "Heuristic %": 22.0}, {"community": "marathi_india", "Human %": 57.5, "Heuristic %": 29.1}, {"community": "bulgarian_bulgaria", "Human %": 58.4, "Heuristic %": 39.0}, {"community": "english_usa", "Human %": 71.5, "Heuristic %": 61.3}]}, "fail_order": ["Correct", "Partially correct", "Explicit uncertainty", "Wrong entity", "Too broad", "Hallucinated", "Other incorrect"], "fail_colors": {"Correct": "#1a9e76", "Partially correct": "#8fcfb8", "Explicit uncertainty": "#4c9be8", "Wrong entity": "#e79a3c", "Too broad": "#efc84a", "Hallucinated": "#d1552f", "Other incorrect": "#a7adb5"}, "kappa": {"overall": 0.603, "misses": 3188, "overcredit": 540, "n": 31965, "per": [{"community": "cantonese_china", "kappa": 0.096}, {"community": "serbian_serbia", "kappa": 0.212}, {"community": "marathi_india", "kappa": 0.303}, {"community": "telugu_india", "kappa": 0.314}, {"community": "mongondow_indonesia", "kappa": 0.43}, {"community": "bugisnese_indonesia", "kappa": 0.448}, {"community": "malayalam_india", "kappa": 0.48}, {"community": "bulgarian_bulgaria", "kappa": 0.551}, {"community": "tamil_india", "kappa": 0.568}, {"community": "bengali_bangladesh", "kappa": 0.582}, {"community": "indonesian_indonesia", "kappa": 0.625}, {"community": "hungarian_hungary", "kappa": 0.631}, {"community": "persian_iran", "kappa": 0.636}, {"community": "english_usa", "kappa": 0.715}, {"community": "spanish_argentina", "kappa": 0.819}, {"community": "italian_italy", "kappa": 0.823}, {"community": "belarusian_belarus", "kappa": 0.828}]}, "failmodes": [{"model": "GPT5.5 [high]", "Correct": 74.8, "Partially correct": 9.6, "Explicit uncertainty": 0.0, "Wrong entity": 4.2, "Too broad": 3.0, "Hallucinated": 7.1, "Other incorrect": 1.3}, {"model": "gemini-3.1-pro-preview", "Correct": 65.7, "Partially correct": 12.9, "Explicit uncertainty": 0.0, "Wrong entity": 5.7, "Too broad": 2.8, "Hallucinated": 11.6, "Other incorrect": 1.3}, {"model": "GPT5.5 [none]", "Correct": 64.9, "Partially correct": 10.6, "Explicit uncertainty": 0.0, "Wrong entity": 7.6, "Too broad": 2.7, "Hallucinated": 12.9, "Other incorrect": 1.3}, {"model": "claude-opus-4-7 [thinking]", "Correct": 55.6, "Partially correct": 10.8, "Explicit uncertainty": 0.5, "Wrong entity": 14.0, "Too broad": 4.6, "Hallucinated": 12.1, "Other incorrect": 2.3}, {"model": "deepseek-v4-pro-thinking", "Correct": 45.8, "Partially correct": 11.4, "Explicit uncertainty": 0.0, "Wrong entity": 12.9, "Too broad": 5.4, "Hallucinated": 21.4, "Other incorrect": 3.1}, {"model": "claude-opus-4-7 [none]", "Correct": 42.9, "Partially correct": 11.2, "Explicit uncertainty": 0.2, "Wrong entity": 18.5, "Too broad": 5.2, "Hallucinated": 19.6, "Other incorrect": 2.4}, {"model": "deepseek-v4-pro", "Correct": 41.3, "Partially correct": 11.0, "Explicit uncertainty": 0.0, "Wrong entity": 15.5, "Too broad": 6.6, "Hallucinated": 22.4, "Other incorrect": 3.2}, {"model": "Gemini 2.5 Flash", "Correct": 31.5, "Partially correct": 9.8, "Explicit uncertainty": 0.2, "Wrong entity": 21.6, "Too broad": 6.8, "Hallucinated": 27.1, "Other incorrect": 3.0}, {"model": "Cohere Command A Plus", "Correct": 30.0, "Partially correct": 13.0, "Explicit uncertainty": 0.9, "Wrong entity": 14.0, "Too broad": 11.3, "Hallucinated": 24.7, "Other incorrect": 6.1}, {"model": "Mistral Large 3", "Correct": 23.1, "Partially correct": 9.0, "Explicit uncertainty": 0.1, "Wrong entity": 24.3, "Too broad": 9.2, "Hallucinated": 29.9, "Other incorrect": 4.5}, {"model": "Qwen-SEA-LION-v4.5-27B-IT", "Correct": 22.2, "Partially correct": 9.4, "Explicit uncertainty": 1.1, "Wrong entity": 19.0, "Too broad": 9.1, "Hallucinated": 34.0, "Other incorrect": 5.2}, {"model": "Gemma 4 31B IT", "Correct": 21.7, "Partially correct": 10.2, "Explicit uncertainty": 0.0, "Wrong entity": 20.0, "Too broad": 8.5, "Hallucinated": 34.3, "Other incorrect": 5.3}, {"model": "DeepSeek V4 Flash", "Correct": 21.0, "Partially correct": 10.9, "Explicit uncertainty": 0.1, "Wrong entity": 20.0, "Too broad": 9.3, "Hallucinated": 33.9, "Other incorrect": 4.7}, {"model": "Gemini 2.5 Flash Lite", "Correct": 18.4, "Partially correct": 10.1, "Explicit uncertainty": 0.2, "Wrong entity": 22.9, "Too broad": 9.9, "Hallucinated": 35.2, "Other incorrect": 3.2}, {"model": "Qwen3.5 27B", "Correct": 15.5, "Partially correct": 9.9, "Explicit uncertainty": 0.1, "Wrong entity": 23.5, "Too broad": 9.3, "Hallucinated": 37.2, "Other incorrect": 4.5}, {"model": "OLMo-3-32B-Think", "Correct": 8.6, "Partially correct": 5.2, "Explicit uncertainty": 0.2, "Wrong entity": 17.4, "Too broad": 14.9, "Hallucinated": 46.2, "Other incorrect": 7.5}, {"model": "Amazon Nova Lite v1", "Correct": 7.9, "Partially correct": 6.5, "Explicit uncertainty": 0.1, "Wrong entity": 21.0, "Too broad": 12.7, "Hallucinated": 47.0, "Other incorrect": 4.8}, {"model": "EuroLLM-22B-Instruct-2512", "Correct": 7.3, "Partially correct": 7.3, "Explicit uncertainty": 0.1, "Wrong entity": 16.2, "Too broad": 14.9, "Hallucinated": 47.3, "Other incorrect": 7.0}, {"model": "Amazon Nova Micro v1", "Correct": 4.4, "Partially correct": 4.9, "Explicit uncertainty": 0.3, "Wrong entity": 16.6, "Too broad": 13.2, "Hallucinated": 54.5, "Other incorrect": 6.1}, {"model": "Tiny-Aya earth", "Correct": 3.8, "Partially correct": 6.5, "Explicit uncertainty": 0.0, "Wrong entity": 19.3, "Too broad": 14.6, "Hallucinated": 47.8, "Other incorrect": 8.1}, {"model": "Tiny-Aya fire", "Correct": 3.7, "Partially correct": 6.3, "Explicit uncertainty": 0.0, "Wrong entity": 19.4, "Too broad": 14.6, "Hallucinated": 48.5, "Other incorrect": 7.4}, {"model": "Tiny-Aya water", "Correct": 3.7, "Partially correct": 6.5, "Explicit uncertainty": 0.1, "Wrong entity": 19.1, "Too broad": 14.0, "Hallucinated": 48.7, "Other incorrect": 8.0}, {"model": "Tiny-Aya global", "Correct": 3.6, "Partially correct": 5.8, "Explicit uncertainty": 0.0, "Wrong entity": 20.3, "Too broad": 14.9, "Hallucinated": 47.8, "Other incorrect": 7.7}, {"model": "Qwen/Qwen3.5-0.8B", "Correct": 0.0, "Partially correct": 1.0, "Explicit uncertainty": 0.2, "Wrong entity": 4.4, "Too broad": 10.8, "Hallucinated": 77.3, "Other incorrect": 6.3}], "coverage": [{"community": "bugisnese_indonesia", "status": "complete", "pct_complete": 99.7, "human_done": 1855, "pending": 5, "human_eligible": 1860}, {"community": "bengali_bangladesh", "status": "complete", "pct_complete": 99.8, "human_done": 1707, "pending": 3, "human_eligible": 1710}, {"community": "belarusian_belarus", "status": "complete", "pct_complete": 99.9, "human_done": 1910, "pending": 1, "human_eligible": 1911}, {"community": "malayalam_india", "status": "complete", "pct_complete": 99.9, "human_done": 1928, "pending": 1, "human_eligible": 1929}, {"community": "marathi_india", "status": "complete", "pct_complete": 99.9, "human_done": 1785, "pending": 2, "human_eligible": 1787}, {"community": "bulgarian_bulgaria", "status": "complete", "pct_complete": 100.0, "human_done": 1742, "pending": 0, "human_eligible": 1742}, {"community": "cantonese_china", "status": "complete", "pct_complete": 100.0, "human_done": 1834, "pending": 0, "human_eligible": 1834}, {"community": "english_usa", "status": "complete", "pct_complete": 100.0, "human_done": 1569, "pending": 0, "human_eligible": 1569}, {"community": "hungarian_hungary", "status": "complete", "pct_complete": 100.0, "human_done": 1904, "pending": 0, "human_eligible": 1904}, {"community": "indonesian_indonesia", "status": "complete", "pct_complete": 100.0, "human_done": 1914, "pending": 0, "human_eligible": 1914}, {"community": "italian_italy", "status": "complete", "pct_complete": 100.0, "human_done": 1855, "pending": 0, "human_eligible": 1855}, {"community": "mongondow_indonesia", "status": "complete", "pct_complete": 100.0, "human_done": 2113, "pending": 0, "human_eligible": 2113}, {"community": "persian_iran", "status": "complete", "pct_complete": 100.0, "human_done": 1954, "pending": 0, "human_eligible": 1954}, {"community": "serbian_serbia", "status": "complete", "pct_complete": 100.0, "human_done": 1901, "pending": 0, "human_eligible": 1901}, {"community": "spanish_argentina", "status": "complete", "pct_complete": 100.0, "human_done": 2244, "pending": 0, "human_eligible": 2244}, {"community": "tamil_india", "status": "complete", "pct_complete": 100.0, "human_done": 1818, "pending": 0, "human_eligible": 1818}, {"community": "telugu_india", "status": "complete", "pct_complete": 100.0, "human_done": 1932, "pending": 0, "human_eligible": 1932}, {"community": "luganda_uganda", "status": "not_started", "pct_complete": 0.0, "human_done": 0, "pending": 2172, "human_eligible": 2172}, {"community": "lumasaba_uganda", "status": "not_started", "pct_complete": 0.0, "human_done": 0, "pending": 2137, "human_eligible": 2137}, {"community": "spanish_peru", "status": "partial", "pct_complete": 49.6, "human_done": 909, "pending": 923, "human_eligible": 1832}, {"community": "yoruba_nigeria", "status": "partial", "pct_complete": 63.7, "human_done": 1336, "pending": 762, "human_eligible": 2098}, {"community": "sinhala_sri_lanka", "status": "partial", "pct_complete": 74.5, "human_done": 1361, "pending": 465, "human_eligible": 1826}, {"community": "uzbek_uzbekistan", "status": "partial", "pct_complete": 76.3, "human_done": 1409, "pending": 437, "human_eligible": 1846}, {"community": "thai_thailand", "status": "partial", "pct_complete": 79.6, "human_done": 1451, "pending": 373, "human_eligible": 1824}, {"community": "arabic_syria", "status": "partial", "pct_complete": 80.7, "human_done": 1650, "pending": 395, "human_eligible": 2045}, {"community": "urdu_pakistan", "status": "partial", "pct_complete": 81.1, "human_done": 1631, "pending": 381, "human_eligible": 2012}, {"community": "nepali_nepal", "status": "partial", "pct_complete": 81.2, "human_done": 1485, "pending": 343, "human_eligible": 1828}, {"community": "chinese_china", "status": "partial", "pct_complete": 81.5, "human_done": 1725, "pending": 392, "human_eligible": 2117}, {"community": "german_germany", "status": "partial", "pct_complete": 82.8, "human_done": 1550, "pending": 321, "human_eligible": 1871}, {"community": "polish_poland", "status": "partial", "pct_complete": 83.1, "human_done": 1484, "pending": 301, "human_eligible": 1785}, {"community": "konkani_india", "status": "partial", "pct_complete": 83.8, "human_done": 1751, "pending": 339, "human_eligible": 2090}, {"community": "chinese_simplified_china", "status": "partial", "pct_complete": 85.3, "human_done": 1497, "pending": 258, "human_eligible": 1755}, {"community": "russian_russia", "status": "partial", "pct_complete": 85.9, "human_done": 1563, "pending": 257, "human_eligible": 1820}, {"community": "portuguese_portugal", "status": "partial", "pct_complete": 86.5, "human_done": 1400, "pending": 219, "human_eligible": 1619}, {"community": "brazilian_portuguese", "status": "partial", "pct_complete": 89.3, "human_done": 1175, "pending": 141, "human_eligible": 1316}, {"community": "spanish_cuba", "status": "partial", "pct_complete": 89.6, "human_done": 1477, "pending": 171, "human_eligible": 1648}, {"community": "bengali_india", "status": "partial", "pct_complete": 90.9, "human_done": 1471, "pending": 147, "human_eligible": 1618}, {"community": "dutch_netherlands", "status": "partial", "pct_complete": 91.4, "human_done": 1322, "pending": 125, "human_eligible": 1447}, {"community": "english_canada", "status": "partial", "pct_complete": 94.0, "human_done": 1841, "pending": 118, "human_eligible": 1959}, {"community": "french_france", "status": "partial", "pct_complete": 94.9, "human_done": 1334, "pending": 72, "human_eligible": 1406}, {"community": "arabic_sudan", "status": "thin", "pct_complete": 13.4, "human_done": 289, "pending": 1874, "human_eligible": 2163}, {"community": "arabic_morocco", "status": "thin", "pct_complete": 14.6, "human_done": 280, "pending": 1637, "human_eligible": 1917}, {"community": "swahili_kenya", "status": "thin", "pct_complete": 14.6, "human_done": 301, "pending": 1767, "human_eligible": 2068}, {"community": "arabic_egypt", "status": "thin", "pct_complete": 16.0, "human_done": 290, "pending": 1523, "human_eligible": 1813}]};
+const CEIL="#2f6f9f",FLOOR="#b8bfc7";
+let SC="strict";
+
+document.getElementById('sub').textContent=`Live from the human-eval flat file (updated ${D.meta.mtime}). Every score traces to the flat file — pending rows are dropped, never guessed. This is the human ceiling that complements the team's full-corpus heuristic floor. Provisional; numbers shift as coverage completes.`;
 const cards=[['Communities complete',`${D.meta.complete} / ${D.meta.total}`],['Models',D.meta.models],['Human-scored rows',D.meta.human_rows.toLocaleString()],['Pending',D.meta.pending.toLocaleString()]];
 document.getElementById('cards').innerHTML=cards.map(c=>`<div class="card"><div class="v">${c[1]}</div><div class="l">${c[0]}</div></div>`).join('');
-const TABS=[['lb','🏆 Leaderboard'],['diff','🌍 Community difficulty'],['kappa','🤝 Judge vs human'],['cov','📊 Coverage'],['map','🗺️ Coverage map']];
+
+const TABS=[['lb','🏆 Leaderboard'],['fail','🔍 How models fail'],['diff','🌍 Community difficulty'],['kappa','🤝 Judge vs human'],['cov','📊 Coverage']];
 document.getElementById('tabs').innerHTML=TABS.map((t,i)=>`<button class="tab${i==0?' on':''}" data-p="${t[0]}">${t[1]}</button>`).join('');
 document.querySelectorAll('.tab').forEach(b=>b.onclick=()=>{
  document.querySelectorAll('.tab').forEach(x=>x.classList.remove('on'));b.classList.add('on');
  document.querySelectorAll('.pane').forEach(p=>p.classList.remove('on'));
  document.getElementById(b.dataset.p).classList.add('on');window.dispatchEvent(new Event('resize'));});
-const L={margin:{l:10,r:16,t:10,b:44},plot_bgcolor:'#fff',paper_bgcolor:'#fff',font:{size:12},xaxis:{gridcolor:'#eef0f2',zeroline:false},yaxis:{automargin:true,ticklabelposition:'outside'}};
-function dumbbell(el,rows,keyF,keyC,labels,xlab){
- const y=rows.map(labels),lo=rows.map(r=>r[keyF]),hi=rows.map(r=>r[keyC]);
- const segs=[];rows.forEach((r,i)=>segs.push({x:[r[keyF],r[keyC]],y:[y[i],y[i]],mode:'lines',line:{color:'#d0d3d8',width:3},showlegend:false,hoverinfo:'skip',type:'scatter'}));
- const f={x:lo,y,mode:'markers',type:'scatter',name:'Heuristic (floor)',marker:{color:FLOOR,size:10}};
- const c={x:hi,y,mode:'markers',type:'scatter',name:'Human (ceiling)',marker:{color:CEIL,size:10}};
- Plotly.newPlot(el,[...segs,f,c],{...L,height:Math.max(420,26*rows.length),xaxis:{...L.xaxis,title:xlab},legend:{orientation:'h',y:1.06}},{displayModeBar:false,responsive:true});
+
+const L={margin:{l:10,r:16,t:8,b:44},plot_bgcolor:'#fff',paper_bgcolor:'#fff',
+ font:{family:'-apple-system,Segoe UI,Roboto,sans-serif',size:12,color:'#14181f'},
+ xaxis:{gridcolor:'#eef0f2',zeroline:false},yaxis:{automargin:true}};
+
+function dumbbell(el,rows,keyF,keyC,label,xlab){
+ const y=rows.map(label);
+ const segs=rows.map((r,i)=>({x:[r[keyF],r[keyC]],y:[y[i],y[i]],mode:'lines',line:{color:'#dfe3e8',width:3},showlegend:false,hoverinfo:'skip',type:'scatter'}));
+ const f={x:rows.map(r=>r[keyF]),y,mode:'markers',type:'scatter',name:'Heuristic (floor)',marker:{color:FLOOR,size:10,line:{color:'#9aa0a6',width:1}}};
+ const c={x:rows.map(r=>r[keyC]),y,mode:'markers',type:'scatter',name:'Human (ceiling)',marker:{color:CEIL,size:10}};
+ Plotly.newPlot(el,[...segs,f,c],{...L,height:Math.max(400,27*rows.length),xaxis:{...L.xaxis,title:xlab,range:[0,100]},legend:{orientation:'h',y:1.07}},{displayModeBar:false,responsive:true});
 }
-function render(sc){
- const lb=D.leaderboard[sc];
- dumbbell('lbchart',[...lb].sort((a,b)=>a['Human (ceiling) %']-b['Human (ceiling) %']),'Heuristic (floor) %','Human (ceiling) %',r=>r.model,`Accuracy (%), ${sc}`);
- document.getElementById('lbcap').textContent='Grey = heuristic auto-judge (lower bound). Blue = human judgement (upper bound).';
- document.getElementById('lbtable').innerHTML='<table><tr><th>#</th><th>Model</th><th>Human %</th><th>Heuristic %</th><th>human-scored %</th><th>n</th></tr>'+lb.map(r=>`<tr><td>${r.rank}</td><td>${r.model}</td><td>${r['Human (ceiling) %']}</td><td>${r['Heuristic (floor) %']}</td><td>${r['human-scored %']}</td><td>${r.n}</td></tr>`).join('')+'</table>';
- const d=D.difficulty[sc];
- dumbbell('diffchart',d,'Heuristic %','Human %',r=>r.community,`Mean accuracy across models (%), ${sc}`);
- document.getElementById('diffcap').textContent=`Hardest: ${d.slice(0,3).map(r=>r.community).join(', ')}. Easiest: ${d.slice(-3).map(r=>r.community).join(', ')}.`;
+function renderLB(){
+ const lb=D.leaderboard[SC];
+ dumbbell('lbchart',[...lb].sort((a,b)=>a['Human (ceiling) %']-b['Human (ceiling) %']),'Heuristic (floor) %','Human (ceiling) %',r=>r.model,`Accuracy (%), ${SC}`);
+ document.getElementById('lbtable').innerHTML='<table><tr><th>#</th><th>Model</th><th>Human %</th><th>Heuristic %</th><th>human-judged %</th><th>n</th></tr>'+lb.map(r=>`<tr><td>${r.rank}</td><td>${r.model}</td><td><b>${r['Human (ceiling) %']}</b></td><td>${r['Heuristic (floor) %']}</td><td>${r['human-scored %']}</td><td>${r.n}</td></tr>`).join('')+'</table>';
+ const d=D.difficulty[SC];
+ dumbbell('diffchart',d,'Heuristic %','Human %',r=>r.community,`Mean accuracy across models (%), ${SC}`);
+ document.getElementById('diffcap').textContent=`Mean accuracy across all ${D.meta.models} models per community. Hardest: ${d.slice(0,3).map(r=>r.community).join(', ')}. Easiest: ${d.slice(-3).map(r=>r.community).join(', ')}. Grey = heuristic floor, blue = human ceiling.`;
 }
-document.querySelectorAll('input[name=sc]').forEach(r=>r.onchange=e=>render(e.target.value));
-render('strict');
+document.querySelectorAll('.toggle button').forEach(b=>b.onclick=()=>{
+ document.querySelectorAll('.toggle button').forEach(x=>x.classList.remove('on'));b.classList.add('on');
+ SC=b.dataset.sc;renderLB();});
+renderLB();
+
+// how models fail — stacked horizontal bars, models ordered by Correct desc
+(function(){
+ const rows=[...D.failmodes].reverse();  // plotly draws bottom-up
+ const y=rows.map(r=>r.model);
+ const traces=D.fail_order.map(cat=>({
+   x:rows.map(r=>r[cat]||0),y,name:cat,type:'bar',orientation:'h',
+   marker:{color:D.fail_colors[cat]},
+   hovertemplate:'%{y}<br>'+cat+': %{x:.1f}%<extra></extra>'}));
+ Plotly.newPlot('failchart',traces,{...L,barmode:'stack',height:Math.max(440,28*rows.length),
+   xaxis:{...L.xaxis,title:'Share of human-scored responses (%)',range:[0,100]},
+   legend:{orientation:'h',y:1.06,font:{size:11}}},{displayModeBar:false,responsive:true});
+ const top=D.failmodes[0],bot=D.failmodes[D.failmodes.length-1];
+ document.getElementById('failcap').innerHTML=`How each model's answers break down when a human judges them. <b>Hallucination (red) is the dominant failure mode and grows as capability drops</b> — from ${top.Hallucinated}% (${top.model}) to ${bot.Hallucinated}% (${bot.model}). Models almost never admit uncertainty (blue, ≤1%) — they produce a confident wrong answer instead. "Wrong entity" = right category, wrong specific answer; "too broad" = a more general answer.`;
+})();
+
 // kappa
 const k=D.kappa;
 document.getElementById('kcards').innerHTML=[["Overall Cohen's κ",k.overall],['Human-correct the judge missed',k.misses.toLocaleString()],['Judge over-credits',k.overcredit.toLocaleString()]].map(c=>`<div class="card"><div class="v">${c[1]}</div><div class="l">${c[0]}</div></div>`).join('');
 const kp=[...k.per];
-Plotly.newPlot('kchart',[{x:kp.map(r=>r.kappa),y:kp.map(r=>r.community),type:'bar',orientation:'h',marker:{color:kp.map(r=>r.kappa<0.4?'#D55E00':CEIL)}}],{...L,height:Math.max(420,26*kp.length),xaxis:{...L.xaxis,title:"Cohen's κ (judge vs human)"},shapes:[{type:'line',x0:k.overall,x1:k.overall,y0:-0.5,y1:kp.length-0.5,line:{color:'#7F7F7F',dash:'dash',width:1.4}}]},{displayModeBar:false,responsive:true});
-document.getElementById('kcap').textContent=`On ${k.n.toLocaleString()} human-scored rows. Red κ<0.4 = string-matching unreliable (non-Latin scripts). Dashed line = overall κ ${k.overall}. The judge misses far more correct answers (${k.misses.toLocaleString()}) than it over-credits (${k.overcredit.toLocaleString()}) — why the human ceiling sits above the heuristic floor.`;
+Plotly.newPlot('kchart',[{x:kp.map(r=>r.kappa),y:kp.map(r=>r.community),type:'bar',orientation:'h',marker:{color:kp.map(r=>r.kappa<0.4?'#d1552f':CEIL)},hovertemplate:'%{y}<br>κ %{x:.3f}<extra></extra>'}],{...L,height:Math.max(420,26*kp.length),xaxis:{...L.xaxis,title:"Cohen's κ (judge vs human)"},shapes:[{type:'line',x0:k.overall,x1:k.overall,y0:-0.5,y1:kp.length-0.5,line:{color:'#5b6673',dash:'dash',width:1.4}}]},{displayModeBar:false,responsive:true});
+document.getElementById('kcap').innerHTML=`Two raters = the automatic judge and the human (not two humans — we have one annotator per riddle). On ${k.n.toLocaleString()} doubly-labelled rows. Red κ&lt;0.4 = string-matching unreliable (non-Latin scripts). Dashed line = overall κ ${k.overall}. The judge misses far more correct answers (${k.misses.toLocaleString()}) than it over-credits (${k.overcredit.toLocaleString()}).`;
+
 // coverage
 const ord={complete:0,partial:1,thin:2,not_started:3};
 const cv=[...D.coverage].sort((a,b)=>ord[a.status]-ord[b.status]||b.pct_complete-a.pct_complete);
-document.getElementById('covtable').innerHTML='<table><tr><th>Community</th><th>Status</th><th>% complete</th><th>human</th><th>pending</th><th>eligible</th></tr>'+cv.map(r=>`<tr><td>${r.community}</td><td><span class="badge ${r.status}">${r.status}</span></td><td>${r.pct_complete??''}</td><td>${r.human_done}</td><td>${r.pending}</td><td>${r.human_eligible}</td></tr>`).join('')+'</table>';
+document.getElementById('covtable').innerHTML='<table><tr><th>Community</th><th>Status</th><th>% complete</th><th>human</th><th>pending</th><th>eligible</th></tr>'+cv.map(r=>`<tr><td>${r.community}</td><td><span class="badge ${r.status}">${r.status.replace('_',' ')}</span></td><td>${r.pct_complete??''}</td><td>${r.human_done}</td><td>${r.pending}</td><td>${r.human_eligible}</td></tr>`).join('')+'</table>';
 const chase=cv.filter(r=>r.status=='not_started'||r.status=='thin').map(r=>r.community);
-document.getElementById('covcap').textContent=chase.length?('Chase list: '+chase.join(', ')):'';
-document.getElementById('foot').innerHTML='Methodology: single human annotator per riddle (no inter-annotator agreement). Leaderboard restricted to communities with complete human coverage. Source of truth: model_riddle_results_flat.csv (Henry). Cuba riddle spanish_cuba_history_016 excluded (redone in v2). Provisional until coverage completes.';
+document.getElementById('covnote').textContent=chase.length?('Chase list (not started / thin): '+chase.join(', ')):'All started.';
+
+// leaderboard CSV download
+document.getElementById('lbdl').onclick=()=>{
+ const lb=D.leaderboard[SC],cols=['rank','model','Human (ceiling) %','Heuristic (floor) %','human-scored %','n'];
+ const csv=[cols.join(',')].concat(lb.map(r=>cols.map(c=>r[c]).join(','))).join('\n');
+ const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([csv],{type:'text/csv'}));
+ a.download='human_leaderboard_'+SC+'.csv';a.click();};
+
+document.getElementById('foot').innerHTML='Methodology: single human annotator per riddle (no inter-annotator agreement). Leaderboard and error types restricted to communities with complete human coverage. Source of truth: model_riddle_results_flat.csv (Henry Gagnier). Heuristic judge is reference-based string matching (no LLM). Cuba riddle spanish_cuba_history_016 excluded (redone in v2). Provisional until coverage completes.';
 </script></body></html>

requirements.txt

@@ -0,0 +1,4 @@
+streamlit>=1.40
+pandas>=2.2
+pyarrow>=16
+plotly>=5.24