app.py

"""Sequential Forgetting Leaderboard — every number links to a raw run file."""
from pathlib import Path

import gradio as gr
import pandas as pd

HERE = Path(__file__).parent
DATASET_URL = "https://huggingface.co/datasets/ModelBrew/sequential-forgetting-benchmark"

df = pd.read_csv(HERE / "results.csv").fillna("")

SUITE_LABELS = {
    "5-domain-realworld": "Suite A — 5 real-world domains · Mistral-7B · 3 seeds",
    "4-domain-MLCF": "Suite B — Medical→Legal→Code→Finance",
    "4-domain-MLCF-history": "Suite B history — CL-technique stacks (within-version comparisons only)",
    "mquake-5skill-vault": "Suite C — MQuAKE 5-skill retention · Qwen3-4B",
}


def linkify(source: str) -> str:
    return f"[{source}]({DATASET_URL}/blob/main/results/{source})"


def suite_table(suite: str, valid_only: bool) -> pd.DataFrame:
    sub = df[df.suite == suite].copy()
    if valid_only:
        sub = sub[sub.status.str.startswith("valid")]
    else:
        sub = sub[~sub.status.str.startswith("valid")]
    sub = sub.sort_values("retention_value_pct", key=lambda s: s.abs())
    sub["source"] = sub["source_file"].map(linkify)
    cols = ["method", "base_model", "n_domains", "n_seeds",
            "retention_metric", "retention_value_pct", "status", "source", "notes"]
    return sub[cols].reset_index(drop=True)


with gr.Blocks(title="Sequential Forgetting Leaderboard") as demo:
    gr.Markdown(
        "# 📉 Sequential Forgetting Leaderboard\n"
        "How much does sequential fine-tuning destroy what the model already learned? "
        "Lower magnitude = better retention. **Every number links to the raw run file** "
        f"in the [benchmark dataset]({DATASET_URL}); transcriptions are hand-checked "
        f"([provenance]({DATASET_URL}/blob/main/results/PROVENANCE.md)).\n\n"
        "Suites are **not** cross-comparable; each table ranks within its own protocol."
    )

    with gr.Tab("Leaderboard"):
        for suite, label in SUITE_LABELS.items():
            table = suite_table(suite, valid_only=True)
            if len(table):
                gr.Markdown(f"### {label}")
                gr.Dataframe(table, interactive=False, wrap=True,
                             datatype=["str"] * len(table.columns))

    with gr.Tab("Invalid & incomplete runs (disclosed)"):
        gr.Markdown(
            "Buggy or unfinished runs are **relabeled, not deleted**. Highlight: our "
            "early O-LoRA arm appeared to win (−2.0% forgetting) until we found a "
            "gradient-clipping bug that had frozen the model — so O-LoRA is listed as "
            "*invalid, never validly measured here*, not as *beaten*."
        )
        for suite in SUITE_LABELS:
            table = suite_table(suite, valid_only=False)
            if len(table):
                gr.Markdown(f"### {SUITE_LABELS[suite]}")
                gr.Dataframe(table, interactive=False, wrap=True,
                             datatype=["str"] * len(table.columns))

    with gr.Tab("Submit your method"):
        gr.Markdown(
            "1. Run your method on a suite "
            f"([protocol]({DATASET_URL}/blob/main/protocol/PROTOCOL.md)).\n"
            "2. Score it with "
            f"[`scoring/score.py`]({DATASET_URL}/blob/main/scoring/score.py) "
            "(`--nll` or `--matrix`).\n"
            "3. Open a PR on the dataset repo adding your raw log, a `results.csv` row, "
            "and a provenance line.\n\n"
            "Single-seed submissions are accepted and labeled `valid_single_run`. "
            "If your run later turns out buggy, it moves to the disclosed tab — "
            "that's the deal for everyone, including us."
        )

    gr.Markdown(
        "---\nMaintained by [ModelBrew](https://modelbrew.ai) — fine-tuning without "
        "catastrophic forgetting (patent-pending CRMA adapters). The `modular_crma` "
        "rows are our method; independent replications welcome."
    )

if __name__ == "__main__":
    demo.launch()