Initialize leaderboard

.env.example

@@ -0,0 +1 @@
+HF_TOKEN=

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+.venv/
+.env

README.md

@@ -1,5 +1,5 @@
 ---
-title: PLLuM SLURM Tutorial
+title: "Evaluating LLMs on Athena: a practical introduction to Slurm"
 emoji: 💻
 colorFrom: indigo
 colorTo: yellow

app.py

@@ -0,0 +1,286 @@
+import hashlib
+import json
+import logging
+import math
+import os
+import re
+from collections import Counter, defaultdict, deque
+from datetime import datetime, timezone
+from pathlib import Path
+
+import gradio as gr
+from dotenv import load_dotenv
+from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
+
+load_dotenv(Path(__file__).with_name(".env"))
+
+REPO_ID = "gamemeine/pllum-slurm-dataset"
+DATA_FILE = "data/evaluation_dataset.jsonl"
+TOLERANCE = 1e-6
+ERROR_TYPES = ("wrong_answer", "parsing_error", "tool_error", "call_budget_exceeded",
+               "context_limit", "missing_prediction", "evaluation_error")
+RECENT_LIMIT = 50
+api = HfApi(token=os.environ.get("HF_TOKEN"))
+
+
+def reject_constant(value):
+    raise ValueError(f"Invalid JSON value: {value}")
+
+
+def read_json(path):
+    return json.loads(Path(path).read_text(encoding="utf-8"), parse_constant=reject_constant)
+
+
+def download(filename, revision):
+    return hf_hub_download(REPO_ID, filename, repo_type="dataset", revision=revision, token=api.token)
+
+
+def load_benchmark(revision):
+    content = Path(download(DATA_FILE, revision)).read_bytes()
+    rows = [json.loads(line, parse_constant=reject_constant) for line in content.decode("utf-8").splitlines() if line.strip()]
+    if not rows:
+        raise ValueError("The benchmark is empty.")
+    return rows, hashlib.sha256(content).hexdigest()
+
+
+def number(value):
+    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+        raise ValueError("Prediction must be a number or null.")
+    if isinstance(value, str):
+        value = value.replace(",", ".")
+    value = float(value)
+    if not math.isfinite(value):
+        raise ValueError("Numbers must be finite.")
+    return value
+
+
+def classify_error(error):
+    """Classify evaluator messages; keep unknown failures as evaluation_error."""
+    text = str(error).lower()
+    if "tool call budget exhausted" in text:
+        return "call_budget_exceeded"
+    if "exceeds max-model-len" in text:
+        return "context_limit"
+    if "unknown tool:" in text or " requires arguments " in text or re.match(
+        r"^(add|subtract|multiply|divide|modulo|power|absolute_value|square_root):", text
+    ):
+        return "tool_error"
+    if any(marker in text for marker in (
+        "expected only a final number", "final answer does not contain a number",
+        "expected exactly one tagged", "tool call must contain", "tool name must",
+        "tool arguments must", "expected a finite real number", "floating-point range",
+        "expecting value", "expecting property name", "expecting ',' delimiter",
+        "expecting ':' delimiter", "unterminated string", "extra data", "invalid control character", "invalid escape",
+    )):
+        return "parsing_error"
+    return "evaluation_error"
+
+
+def score(samples, benchmark):
+    if not isinstance(samples, list) or not all(isinstance(sample, dict) for sample in samples):
+        raise ValueError("samples.json must contain a list of evaluator samples.")
+    # Match by question and category, so changing row order is harmless.
+    def key(row):
+        prompt = row.get("prompt", row.get("task"))
+        category = row.get("category")
+        if not isinstance(prompt, str) or not isinstance(category, str):
+            raise ValueError("Each sample must have a prompt and category.")
+        return prompt, category
+
+    if Counter(map(key, samples)) != Counter(map(key, benchmark)):
+        raise ValueError("The submission must match the complete benchmark, including duplicate counts.")
+    references = defaultdict(deque)
+    for row in benchmark:
+        references[key(row)].append(number(row["answer"]))
+    categories = defaultdict(lambda: {"total": 0, "correct": 0})
+    error_counts = dict.fromkeys(ERROR_TYPES, 0)
+    details = []
+    for index, sample in enumerate(samples):
+        answer = references[key(sample)].popleft()
+        prediction, absolute_error = None, None
+        prediction_error = False
+        if sample.get("prediction") is not None:
+            try:
+                prediction = number(sample["prediction"])
+                absolute_error = abs(prediction - answer)
+                if not math.isfinite(absolute_error):
+                    absolute_error = None
+            except (ValueError, TypeError, OverflowError):
+                prediction_error = True
+        if sample.get("error"):
+            outcome = classify_error(sample["error"])
+        elif prediction_error:
+            outcome = "parsing_error"
+        elif prediction is None:
+            outcome = "missing_prediction"
+        elif abs(prediction - answer) <= TOLERANCE:
+            outcome = "correct"
+        else:
+            outcome = "wrong_answer"
+        correct = outcome == "correct"
+        if not correct:
+            error_counts[outcome] += 1
+        details.append({
+            "index": index, "prompt": key(sample)[0], "category": sample["category"],
+            "expected_answer": answer, "prediction": prediction,
+            "absolute_error": absolute_error, "outcome": outcome, "error": sample.get("error"),
+        })
+        category = categories[sample["category"]]
+        category["total"] += 1
+        category["correct"] += int(correct)
+    for category in categories.values():
+        category["accuracy"] = category["correct"] / category["total"]
+    correct = sum(category["correct"] for category in categories.values())
+    return {
+        "total": len(samples), "correct": correct, "accuracy": correct / len(samples),
+        "by_category": dict(sorted(categories.items())),
+        "error_counts": error_counts, "sample_results": details,
+    }
+
+
+def tables(results, categories):
+    headers = ["Login", "Submission", "Accuracy (%)"] + [f"{category} (%)" for category in categories]
+
+    def row(result):
+        return [result["login"], result["submission_id"], round(100 * result["accuracy"], 2)] + [
+            round(100 * result["by_category"][category]["accuracy"], 2) for category in categories
+        ]
+
+    recent = sorted(results, key=lambda result: result["created_at"], reverse=True)[:RECENT_LIMIT]
+    submissions = gr.update(
+        headers=["Submitted (UTC)"] + headers + ["Correct / total"] + list(ERROR_TYPES),
+        value=[
+            [result["created_at"]] + row(result) + [f"{result['correct']} / {result['total']}"]
+            + [result["error_counts"][error] for error in ERROR_TYPES]
+            for result in recent
+        ],
+    )
+    best = {}
+    # Highest accuracy wins; the earliest submission wins an exact tie.
+    for result in sorted(results, key=lambda result: (-result["accuracy"], result["created_at"], result["submission_id"])):
+        best.setdefault(result["login"], result)
+    leaderboard = gr.update(
+        headers=["#"] + headers,
+        value=[[rank] + row(result) for rank, result in enumerate(best.values(), 1)],
+    )
+    return submissions, leaderboard
+
+
+def refresh_tables():
+    try:
+        revision = api.dataset_info(REPO_ID).sha
+        benchmark, digest = load_benchmark(revision)
+        categories = sorted({row["category"] for row in benchmark})
+        results = []
+        for filename in api.list_repo_files(REPO_ID, repo_type="dataset", revision=revision):
+            if filename.startswith("submissions/") and filename.endswith("/results.json"):
+                result = read_json(download(filename, revision))
+                if result.get("dataset_sha256") == digest:
+                    # Compute diagnostics for submissions saved before error classification existed.
+                    if "error_counts" not in result:
+                        samples_file = filename.removesuffix("results.json") + "samples.json"
+                        result.update(score(read_json(download(samples_file, revision)), benchmark))
+                    results.append(result)
+        return tables(results, categories)
+    except Exception as exc:
+        logging.exception("Could not load submissions")
+        raise gr.Error("Could not load results. Check HF_TOKEN access and the benchmark file in the dataset.") from exc
+
+
+def submit(login, filepath):
+    login = (login or "").strip()
+    if not re.fullmatch(r"tutorial0(?:0[1-9]|[12][0-9]|3[0-3])", login):
+        raise gr.Error("Use your assigned login: tutorial001–tutorial033.")
+    if not filepath:
+        raise gr.Error("Upload samples.json.")
+    try:
+        samples = read_json(filepath)
+        revision = api.dataset_info(REPO_ID).sha
+        benchmark, digest = load_benchmark(revision)
+        result = score(samples, benchmark)
+    except (ValueError, TypeError, KeyError, OverflowError) as exc:
+        raise gr.Error(f"Invalid submission: {exc}") from exc
+    except Exception as exc:
+        logging.exception("Could not load the benchmark")
+        raise gr.Error("Could not load the benchmark. Check dataset access.") from exc
+
+    now = datetime.now(timezone.utc)
+    submission_id = f"{login}-{now:%Y%m%dT%H%M%S%fZ}"
+    result.update(login=login, submission_id=submission_id, created_at=now.isoformat(),
+                  dataset=DATA_FILE, dataset_sha256=digest, tolerance=TOLERANCE)
+    try:
+        api.create_commit(
+            repo_id=REPO_ID, repo_type="dataset", commit_message=f"Submission {submission_id}",
+            operations=[
+                CommitOperationAdd(path_in_repo=f"submissions/{submission_id}/samples.json", path_or_fileobj=Path(filepath)),
+                CommitOperationAdd(path_in_repo=f"submissions/{submission_id}/results.json",
+                                   path_or_fileobj=json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False).encode("utf-8")),
+            ],
+        )
+    except Exception as exc:
+        logging.exception("Could not save the submission")
+        raise gr.Error("Could not save the submission. HF_TOKEN needs write access to the dataset.") from exc
+    errors = ", ".join(f"{name}: {count}" for name, count in result["error_counts"].items() if count)
+    status = (f"Saved **{submission_id}**. Accuracy: **{result['accuracy']:.2%}** "
+              f"({result['correct']} / {result['total']} correct).\n\nErrors: {errors or 'none'}.")
+    try:
+        return status, *refresh_tables()
+    except gr.Error:
+        return status + " Results are temporarily unavailable — click Refresh.", gr.skip(), gr.skip()
+
+
+CSS = """
+.gradio-container { max-width: 1280px !important; margin: auto !important; }
+#workshop { width: 100%; margin: auto; }
+#intro, #scoring, #status { text-align: center; }
+#workshop input, #workshop th, #workshop td { text-align: center !important; }
+#workshop .tab-nav { justify-content: center; }
+"""
+
+with gr.Blocks(title="Evaluating LLMs on Athena: a practical introduction to Slurm") as demo:
+    with gr.Column(elem_id="workshop"):
+        gr.Markdown("""# Evaluating LLMs on Athena: a practical introduction to Slurm
+Enter your tutorial login and upload **samples.json** from a full benchmark evaluation
+(e.g. `$SCRATCH/pllum-slurm-lab/evaluation/samples.json`). Click **Submit** to score your run.
+**Leaderboard** shows the best run per login; **Submissions** shows the 50 latest runs.
+All accuracy values are percentages. For equal scores, the earliest run wins.
+""", elem_id="intro")
+        with gr.Row():
+            login = gr.Textbox(label="Login", placeholder="tutorial001", info="tutorial001–tutorial033")
+            upload = gr.File(label="samples.json", file_types=[".json"], type="filepath")
+        send = gr.Button("Submit", variant="primary")
+        status = gr.Markdown(elem_id="status")
+        with gr.Accordion("How scoring works", open=False):
+            gr.Markdown("""Upload the full benchmark. A prediction is correct if it matches the reference within **1e-6**
+and has no evaluator error. **Accuracy = correct / all questions**, overall and per category.
+
+Each failed question receives one error type:
+
+| Error type | Meaning |
+| --- | --- |
+| `wrong_answer` | A valid numerical prediction falls outside the tolerance. |
+| `parsing_error` | The evaluator could not parse the response, or the prediction is not a finite number. |
+| `tool_error` | An unknown tool, wrong arguments, or a failed calculator operation. |
+| `call_budget_exceeded` | The model exceeded the allowed number of tool calls. |
+| `context_limit` | The conversation exceeded the model context limit. |
+| `missing_prediction` | No final prediction and no reported evaluator error. |
+| `evaluation_error` | Another evaluator failure, including unrecognized error messages. |
+
+Error types are inferred from the evaluator's error messages. A reported evaluator error takes precedence
+over the prediction. Error columns show **counts of failed questions**, so correct + all error counts = total.
+The saved `results.json` includes per-question outcomes, reference answers and absolute differences.
+Both tabs show results for the current benchmark only.
+""", elem_id="scoring")
+        with gr.Tabs():
+            with gr.Tab("Leaderboard"):
+                leaderboard_table = gr.Dataframe(interactive=False, label="Best result per login")
+            with gr.Tab("Submissions"):
+                submissions_table = gr.Dataframe(interactive=False, label="Latest submissions")
+        refresh = gr.Button("Refresh")
+        outputs = [submissions_table, leaderboard_table]
+        send.click(submit, inputs=[login, upload], outputs=[status, *outputs], concurrency_limit=1)
+        refresh.click(refresh_tables, outputs=outputs)
+        demo.load(refresh_tables, outputs=outputs)
+
+if __name__ == "__main__":
+    demo.queue().launch(css=CSS)

requirements.txt

@@ -0,0 +1,3 @@
+gradio==6.30.0
+huggingface_hub>=1.0,<2
+python-dotenv>=1.0,<2