Spaces:
Running
Running
Add v8 SmolTalkBench progress chart with SmolLM baselines
Browse filesExclude the November 7 outage interval and keep generated evaluation data outside Git.
Co-authored-by: Codex <noreply@openai.com>
- README.md +16 -0
- app.py +9 -0
- build_progress.py +59 -0
- site/index.html +13 -0
- site/progress.js +40 -0
README.md
CHANGED
|
@@ -27,3 +27,19 @@ uvicorn app:app --host 127.0.0.1 --port 7860
|
|
| 27 |
|
| 28 |
Open http://127.0.0.1:7860. The health endpoint is `/health`.
|
| 29 |
Keep local logs and temporary files in `scratch/`, which is ignored by Git.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
Open http://127.0.0.1:7860. The health endpoint is `/health`.
|
| 29 |
Keep local logs and temporary files in `scratch/`, which is ignored by Git.
|
| 30 |
+
|
| 31 |
+
### Progress chart
|
| 32 |
+
|
| 33 |
+
Generate the chart snapshot from the leaderboard's v3 cache before starting the app:
|
| 34 |
+
|
| 35 |
+
```sh
|
| 36 |
+
python build_progress.py /path/to/swarmchat-leaderboard-v3.json
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
The generated data stays in `scratch/progress.json` and is never committed. Supply
|
| 40 |
+
that file separately when deploying (at `/home/user/app/scratch/progress.json` in
|
| 41 |
+
the Docker image). The chart shows an unavailable message if no snapshot is present.
|
| 42 |
+
It uses the leaderboard's length-penalised percentage metric and latest successful
|
| 43 |
+
1,000-prompt run per revision, with all three SmolLM baselines. The contiguous outage
|
| 44 |
+
plateau from November 7, 2026, 12:12:47 through 13:15:11 UTC (commit times) is omitted;
|
| 45 |
+
other score fluctuations and the original revision positions are preserved.
|
app.py
CHANGED
|
@@ -14,6 +14,7 @@ from typing import Literal
|
|
| 14 |
|
| 15 |
from fastapi import FastAPI, HTTPException
|
| 16 |
from fastapi.staticfiles import StaticFiles
|
|
|
|
| 17 |
from pydantic import BaseModel, Field
|
| 18 |
|
| 19 |
from bot_runner import Bot, user_lines
|
|
@@ -108,4 +109,12 @@ def health():
|
|
| 108 |
return {"ok": True}
|
| 109 |
|
| 110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
app.mount("/", StaticFiles(directory=HERE / "site", html=True), name="site")
|
|
|
|
| 14 |
|
| 15 |
from fastapi import FastAPI, HTTPException
|
| 16 |
from fastapi.staticfiles import StaticFiles
|
| 17 |
+
from fastapi.responses import FileResponse
|
| 18 |
from pydantic import BaseModel, Field
|
| 19 |
|
| 20 |
from bot_runner import Bot, user_lines
|
|
|
|
| 109 |
return {"ok": True}
|
| 110 |
|
| 111 |
|
| 112 |
+
@app.get("/api/progress")
|
| 113 |
+
def progress():
|
| 114 |
+
snapshot = HERE / "scratch" / "progress.json"
|
| 115 |
+
if not snapshot.is_file():
|
| 116 |
+
raise HTTPException(503, "The progress chart data is unavailable.")
|
| 117 |
+
return FileResponse(snapshot, media_type="application/json")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
app.mount("/", StaticFiles(directory=HERE / "site", html=True), name="site")
|
build_progress.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build the local chart snapshot from the leaderboard's cached summaries."""
|
| 2 |
+
|
| 3 |
+
import argparse
|
| 4 |
+
import json
|
| 5 |
+
import math
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
# The abrupt outage plateau, bounded by the last/first affected commit timestamps.
|
| 9 |
+
# Explicit bounds retain genuine regressions elsewhere; never filter on score.
|
| 10 |
+
OUTAGE_START = "20261107T121247Z"
|
| 11 |
+
OUTAGE_END = "20261107T131511Z"
|
| 12 |
+
MODEL = "HuggingFaceSwarms/SwarmChat-v8"
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def build_progress(cache):
|
| 16 |
+
latest = {}
|
| 17 |
+
for path, summary in cache["summaries"].items():
|
| 18 |
+
meta = summary.get("metadata", {})
|
| 19 |
+
metric = summary.get("primary_metric", {})
|
| 20 |
+
score = metric.get("value")
|
| 21 |
+
if (meta.get("task") != "smoltalk_bench_gold"
|
| 22 |
+
or meta.get("status") != "success" or meta.get("task_args")
|
| 23 |
+
or meta.get("total_samples") != 1000
|
| 24 |
+
or metric.get("name") != "length_penalised_win_rate"
|
| 25 |
+
or not isinstance(score, (int, float)) or not math.isfinite(score)):
|
| 26 |
+
continue
|
| 27 |
+
key = (meta["model_id"], meta["revision"])
|
| 28 |
+
order = (meta["completed_at"], path)
|
| 29 |
+
if key not in latest or order > latest[key][0]:
|
| 30 |
+
latest[key] = (order, summary)
|
| 31 |
+
revisions = sorted((revision, summary) for (model, revision), (_, summary)
|
| 32 |
+
in latest.items() if model == MODEL)
|
| 33 |
+
points, excluded = [], 0
|
| 34 |
+
for index, (revision, summary) in enumerate(revisions, 1):
|
| 35 |
+
if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END:
|
| 36 |
+
excluded += 1
|
| 37 |
+
continue
|
| 38 |
+
points.append({"revision": index, "commit": revision.split("-")[1],
|
| 39 |
+
"score": summary["primary_metric"]["value"]})
|
| 40 |
+
baselines = sorted([
|
| 41 |
+
{"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]}
|
| 42 |
+
for (model, revision), (_, summary) in latest.items()
|
| 43 |
+
if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main"
|
| 44 |
+
], key=lambda baseline: baseline["score"])
|
| 45 |
+
if not points or len(baselines) != 3:
|
| 46 |
+
raise ValueError("Expected v8 evaluations and all three SmolLM baselines")
|
| 47 |
+
return {"points": points, "baselines": baselines, "excluded": excluded,
|
| 48 |
+
"fetched_at": cache["fetched_at"]}
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
if __name__ == "__main__":
|
| 52 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 53 |
+
parser.add_argument("cache", type=Path)
|
| 54 |
+
args = parser.parse_args()
|
| 55 |
+
result = build_progress(json.loads(args.cache.read_text()))
|
| 56 |
+
output = Path(__file__).parent / "scratch" / "progress.json"
|
| 57 |
+
output.parent.mkdir(exist_ok=True)
|
| 58 |
+
output.write_text(json.dumps(result, separators=(",", ":")) + "\n")
|
| 59 |
+
print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations")
|
site/index.html
CHANGED
|
@@ -41,6 +41,10 @@
|
|
| 41 |
.sw-win { background:var(--accent); border-width:2px !important; } .sw-src { background:var(--ink); } .sw-used { background:#8a8a8a; }
|
| 42 |
.sw-prop { background:var(--paper); border-width:2px !important; } .sw-off { background:var(--off); border-color:var(--faint) !important; }
|
| 43 |
.foot { font:13px var(--mono); color:var(--mute); margin-top:14px; } .foot a { color:inherit; }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
@media (max-width:640px) { body { font-size:17px; } #map { height:420px; } #log { height:200px; } .msg { max-width:90%; } }
|
| 45 |
</style>
|
| 46 |
</head>
|
|
@@ -78,6 +82,14 @@
|
|
| 78 |
</div>
|
| 79 |
</section>
|
| 80 |
<p class="foot">PyBot at commit <a id="commit" target="_blank" rel="noopener"></a>. Click a file to open it on the Hub.</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
</main>
|
| 82 |
<script>/**
|
| 83 |
* plotly.js v2.35.2
|
|
@@ -167,5 +179,6 @@ document.getElementById("reset").addEventListener("click", () => {
|
|
| 167 |
q.value = ""; q.disabled = send.disabled = false; q.focus({preventScroll:true});
|
| 168 |
});
|
| 169 |
</script>
|
|
|
|
| 170 |
</body>
|
| 171 |
</html>
|
|
|
|
| 41 |
.sw-win { background:var(--accent); border-width:2px !important; } .sw-src { background:var(--ink); } .sw-used { background:#8a8a8a; }
|
| 42 |
.sw-prop { background:var(--paper); border-width:2px !important; } .sw-off { background:var(--off); border-color:var(--faint) !important; }
|
| 43 |
.foot { font:13px var(--mono); color:var(--mute); margin-top:14px; } .foot a { color:inherit; }
|
| 44 |
+
.progress { margin-top:48px; border-top:1px solid var(--ink); padding-top:24px; }
|
| 45 |
+
.progress h2 { font:400 32px/1.2 var(--lead); margin:0 0 8px; }
|
| 46 |
+
.progress p { max-width:680px; }
|
| 47 |
+
#progress-chart { width:100%; height:460px; }
|
| 48 |
@media (max-width:640px) { body { font-size:17px; } #map { height:420px; } #log { height:200px; } .msg { max-width:90%; } }
|
| 49 |
</style>
|
| 50 |
</head>
|
|
|
|
| 82 |
</div>
|
| 83 |
</section>
|
| 84 |
<p class="foot">PyBot at commit <a id="commit" target="_blank" rel="noopener"></a>. Click a file to open it on the Hub.</p>
|
| 85 |
+
<section class="progress" aria-labelledby="progress-title">
|
| 86 |
+
<h2 id="progress-title">Learning to converse, one revision at a time</h2>
|
| 87 |
+
<p>SwarmChat-v8 on SmolTalkBench, compared with the SmolLM Instruct models.</p>
|
| 88 |
+
<div id="progress-chart" role="img" aria-label="Line chart of SwarmChat-v8 win rate over evaluated revisions, with three SmolLM baselines"></div>
|
| 89 |
+
<p class="foot" id="progress-status" aria-live="polite">Loading evaluation history…</p>
|
| 90 |
+
<p class="foot">Length-penalised win rate on 1,000 prompts. November 7 outage evaluations are excluded; revision numbers are preserved.
|
| 91 |
+
Source: <a href="https://huggingface.co/spaces/HuggingFaceSwarms/swarmchat-leaderboard" target="_blank" rel="noopener">SwarmChat leaderboard</a>.</p>
|
| 92 |
+
</section>
|
| 93 |
</main>
|
| 94 |
<script>/**
|
| 95 |
* plotly.js v2.35.2
|
|
|
|
| 179 |
q.value = ""; q.disabled = send.disabled = false; q.focus({preventScroll:true});
|
| 180 |
});
|
| 181 |
</script>
|
| 182 |
+
<script src="progress.js"></script>
|
| 183 |
</body>
|
| 184 |
</html>
|
site/progress.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
async function drawProgress() {
|
| 2 |
+
const status = document.getElementById("progress-status");
|
| 3 |
+
try {
|
| 4 |
+
const response = await fetch("api/progress");
|
| 5 |
+
if (!response.ok) throw new Error("Progress data unavailable");
|
| 6 |
+
const data = await response.json();
|
| 7 |
+
const points = data.points;
|
| 8 |
+
const extent = [points[0].revision, points[points.length - 1].revision];
|
| 9 |
+
const traces = [{
|
| 10 |
+
type: "scatter", mode: "lines", name: "SwarmChat-v8",
|
| 11 |
+
x: points.map(p => p.revision), y: points.map(p => p.score),
|
| 12 |
+
customdata: points.map(p => p.commit),
|
| 13 |
+
line: {color: "#eb6834", width: 2},
|
| 14 |
+
hovertemplate: "v8 · revision %{x:,}<br>Win rate: %{y:.2f}%<br>Commit %{customdata}<extra></extra>"
|
| 15 |
+
}, ...data.baselines.map((baseline, index) => ({
|
| 16 |
+
type: "scatter", mode: "lines",
|
| 17 |
+
name: `${baseline.name} (${baseline.score.toFixed(1)}%)`,
|
| 18 |
+
x: extent, y: [baseline.score, baseline.score],
|
| 19 |
+
line: {color: ["#999999", "#555555", "#000000"][index], width: 1.5,
|
| 20 |
+
dash: ["dot", "dash", "dashdot"][index]},
|
| 21 |
+
hovertemplate: `${baseline.name}<br>Win rate: %{y:.2f}%<extra></extra>`
|
| 22 |
+
}))];
|
| 23 |
+
await Plotly.newPlot("progress-chart", traces, {
|
| 24 |
+
margin: {l: 60, r: 12, t: 12, b: 130},
|
| 25 |
+
paper_bgcolor: "#ffffff", plot_bgcolor: "#ffffff",
|
| 26 |
+
font: {family: "'Letter Gothic', monospace", size: 12, color: "#000000"},
|
| 27 |
+
xaxis: {title: {text: "Evaluated revisions", standoff: 12},
|
| 28 |
+
range: extent, gridcolor: "#eeeeee", zeroline: false, automargin: true},
|
| 29 |
+
yaxis: {title: {text: "Win rate (%)", standoff: 10}, rangemode: "tozero",
|
| 30 |
+
gridcolor: "#eeeeee", zerolinecolor: "#dddddd", automargin: true},
|
| 31 |
+
legend: {orientation: "h", x: 0, y: -0.25, yanchor: "top", font: {size: 11}},
|
| 32 |
+
hoverlabel: {bgcolor: "#ffffff", bordercolor: "#000000"}
|
| 33 |
+
}, {responsive: true, displayModeBar: false, displaylogo: false});
|
| 34 |
+
status.textContent = `${points.length.toLocaleString()} evaluations shown · ${data.excluded} outage evaluations excluded · Snapshot ${data.fetched_at.slice(0, 10)}.`;
|
| 35 |
+
} catch (error) {
|
| 36 |
+
document.getElementById("progress-chart").hidden = true;
|
| 37 |
+
status.textContent = "The progress chart is unavailable. View the evaluation history on the leaderboard below.";
|
| 38 |
+
}
|
| 39 |
+
}
|
| 40 |
+
drawProgress();
|