lewtun HF Staff Codex commited on
Commit
a5edae5
·
1 Parent(s): a727a3e

Add v8 SmolTalkBench progress chart with SmolLM baselines

Browse files

Exclude the November 7 outage interval and keep generated evaluation data outside Git.

Co-authored-by: Codex <noreply@openai.com>

Files changed (5) hide show
  1. README.md +16 -0
  2. app.py +9 -0
  3. build_progress.py +59 -0
  4. site/index.html +13 -0
  5. site/progress.js +40 -0
README.md CHANGED
@@ -27,3 +27,19 @@ uvicorn app:app --host 127.0.0.1 --port 7860
27
 
28
  Open http://127.0.0.1:7860. The health endpoint is `/health`.
29
  Keep local logs and temporary files in `scratch/`, which is ignored by Git.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
  Open http://127.0.0.1:7860. The health endpoint is `/health`.
29
  Keep local logs and temporary files in `scratch/`, which is ignored by Git.
30
+
31
+ ### Progress chart
32
+
33
+ Generate the chart snapshot from the leaderboard's v3 cache before starting the app:
34
+
35
+ ```sh
36
+ python build_progress.py /path/to/swarmchat-leaderboard-v3.json
37
+ ```
38
+
39
+ The generated data stays in `scratch/progress.json` and is never committed. Supply
40
+ that file separately when deploying (at `/home/user/app/scratch/progress.json` in
41
+ the Docker image). The chart shows an unavailable message if no snapshot is present.
42
+ It uses the leaderboard's length-penalised percentage metric and latest successful
43
+ 1,000-prompt run per revision, with all three SmolLM baselines. The contiguous outage
44
+ plateau from November 7, 2026, 12:12:47 through 13:15:11 UTC (commit times) is omitted;
45
+ other score fluctuations and the original revision positions are preserved.
app.py CHANGED
@@ -14,6 +14,7 @@ from typing import Literal
14
 
15
  from fastapi import FastAPI, HTTPException
16
  from fastapi.staticfiles import StaticFiles
 
17
  from pydantic import BaseModel, Field
18
 
19
  from bot_runner import Bot, user_lines
@@ -108,4 +109,12 @@ def health():
108
  return {"ok": True}
109
 
110
 
 
 
 
 
 
 
 
 
111
  app.mount("/", StaticFiles(directory=HERE / "site", html=True), name="site")
 
14
 
15
  from fastapi import FastAPI, HTTPException
16
  from fastapi.staticfiles import StaticFiles
17
+ from fastapi.responses import FileResponse
18
  from pydantic import BaseModel, Field
19
 
20
  from bot_runner import Bot, user_lines
 
109
  return {"ok": True}
110
 
111
 
112
+ @app.get("/api/progress")
113
+ def progress():
114
+ snapshot = HERE / "scratch" / "progress.json"
115
+ if not snapshot.is_file():
116
+ raise HTTPException(503, "The progress chart data is unavailable.")
117
+ return FileResponse(snapshot, media_type="application/json")
118
+
119
+
120
  app.mount("/", StaticFiles(directory=HERE / "site", html=True), name="site")
build_progress.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Build the local chart snapshot from the leaderboard's cached summaries."""
2
+
3
+ import argparse
4
+ import json
5
+ import math
6
+ from pathlib import Path
7
+
8
+ # The abrupt outage plateau, bounded by the last/first affected commit timestamps.
9
+ # Explicit bounds retain genuine regressions elsewhere; never filter on score.
10
+ OUTAGE_START = "20261107T121247Z"
11
+ OUTAGE_END = "20261107T131511Z"
12
+ MODEL = "HuggingFaceSwarms/SwarmChat-v8"
13
+
14
+
15
+ def build_progress(cache):
16
+ latest = {}
17
+ for path, summary in cache["summaries"].items():
18
+ meta = summary.get("metadata", {})
19
+ metric = summary.get("primary_metric", {})
20
+ score = metric.get("value")
21
+ if (meta.get("task") != "smoltalk_bench_gold"
22
+ or meta.get("status") != "success" or meta.get("task_args")
23
+ or meta.get("total_samples") != 1000
24
+ or metric.get("name") != "length_penalised_win_rate"
25
+ or not isinstance(score, (int, float)) or not math.isfinite(score)):
26
+ continue
27
+ key = (meta["model_id"], meta["revision"])
28
+ order = (meta["completed_at"], path)
29
+ if key not in latest or order > latest[key][0]:
30
+ latest[key] = (order, summary)
31
+ revisions = sorted((revision, summary) for (model, revision), (_, summary)
32
+ in latest.items() if model == MODEL)
33
+ points, excluded = [], 0
34
+ for index, (revision, summary) in enumerate(revisions, 1):
35
+ if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END:
36
+ excluded += 1
37
+ continue
38
+ points.append({"revision": index, "commit": revision.split("-")[1],
39
+ "score": summary["primary_metric"]["value"]})
40
+ baselines = sorted([
41
+ {"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]}
42
+ for (model, revision), (_, summary) in latest.items()
43
+ if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main"
44
+ ], key=lambda baseline: baseline["score"])
45
+ if not points or len(baselines) != 3:
46
+ raise ValueError("Expected v8 evaluations and all three SmolLM baselines")
47
+ return {"points": points, "baselines": baselines, "excluded": excluded,
48
+ "fetched_at": cache["fetched_at"]}
49
+
50
+
51
+ if __name__ == "__main__":
52
+ parser = argparse.ArgumentParser(description=__doc__)
53
+ parser.add_argument("cache", type=Path)
54
+ args = parser.parse_args()
55
+ result = build_progress(json.loads(args.cache.read_text()))
56
+ output = Path(__file__).parent / "scratch" / "progress.json"
57
+ output.parent.mkdir(exist_ok=True)
58
+ output.write_text(json.dumps(result, separators=(",", ":")) + "\n")
59
+ print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations")
site/index.html CHANGED
@@ -41,6 +41,10 @@
41
  .sw-win { background:var(--accent); border-width:2px !important; } .sw-src { background:var(--ink); } .sw-used { background:#8a8a8a; }
42
  .sw-prop { background:var(--paper); border-width:2px !important; } .sw-off { background:var(--off); border-color:var(--faint) !important; }
43
  .foot { font:13px var(--mono); color:var(--mute); margin-top:14px; } .foot a { color:inherit; }
 
 
 
 
44
  @media (max-width:640px) { body { font-size:17px; } #map { height:420px; } #log { height:200px; } .msg { max-width:90%; } }
45
  </style>
46
  </head>
@@ -78,6 +82,14 @@
78
  </div>
79
  </section>
80
  <p class="foot">PyBot at commit <a id="commit" target="_blank" rel="noopener"></a>. Click a file to open it on the Hub.</p>
 
 
 
 
 
 
 
 
81
  </main>
82
  <script>/**
83
  * plotly.js v2.35.2
@@ -167,5 +179,6 @@ document.getElementById("reset").addEventListener("click", () => {
167
  q.value = ""; q.disabled = send.disabled = false; q.focus({preventScroll:true});
168
  });
169
  </script>
 
170
  </body>
171
  </html>
 
41
  .sw-win { background:var(--accent); border-width:2px !important; } .sw-src { background:var(--ink); } .sw-used { background:#8a8a8a; }
42
  .sw-prop { background:var(--paper); border-width:2px !important; } .sw-off { background:var(--off); border-color:var(--faint) !important; }
43
  .foot { font:13px var(--mono); color:var(--mute); margin-top:14px; } .foot a { color:inherit; }
44
+ .progress { margin-top:48px; border-top:1px solid var(--ink); padding-top:24px; }
45
+ .progress h2 { font:400 32px/1.2 var(--lead); margin:0 0 8px; }
46
+ .progress p { max-width:680px; }
47
+ #progress-chart { width:100%; height:460px; }
48
  @media (max-width:640px) { body { font-size:17px; } #map { height:420px; } #log { height:200px; } .msg { max-width:90%; } }
49
  </style>
50
  </head>
 
82
  </div>
83
  </section>
84
  <p class="foot">PyBot at commit <a id="commit" target="_blank" rel="noopener"></a>. Click a file to open it on the Hub.</p>
85
+ <section class="progress" aria-labelledby="progress-title">
86
+ <h2 id="progress-title">Learning to converse, one revision at a time</h2>
87
+ <p>SwarmChat-v8 on SmolTalkBench, compared with the SmolLM Instruct models.</p>
88
+ <div id="progress-chart" role="img" aria-label="Line chart of SwarmChat-v8 win rate over evaluated revisions, with three SmolLM baselines"></div>
89
+ <p class="foot" id="progress-status" aria-live="polite">Loading evaluation history…</p>
90
+ <p class="foot">Length-penalised win rate on 1,000 prompts. November 7 outage evaluations are excluded; revision numbers are preserved.
91
+ Source: <a href="https://huggingface.co/spaces/HuggingFaceSwarms/swarmchat-leaderboard" target="_blank" rel="noopener">SwarmChat leaderboard</a>.</p>
92
+ </section>
93
  </main>
94
  <script>/**
95
  * plotly.js v2.35.2
 
179
  q.value = ""; q.disabled = send.disabled = false; q.focus({preventScroll:true});
180
  });
181
  </script>
182
+ <script src="progress.js"></script>
183
  </body>
184
  </html>
site/progress.js ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ async function drawProgress() {
2
+ const status = document.getElementById("progress-status");
3
+ try {
4
+ const response = await fetch("api/progress");
5
+ if (!response.ok) throw new Error("Progress data unavailable");
6
+ const data = await response.json();
7
+ const points = data.points;
8
+ const extent = [points[0].revision, points[points.length - 1].revision];
9
+ const traces = [{
10
+ type: "scatter", mode: "lines", name: "SwarmChat-v8",
11
+ x: points.map(p => p.revision), y: points.map(p => p.score),
12
+ customdata: points.map(p => p.commit),
13
+ line: {color: "#eb6834", width: 2},
14
+ hovertemplate: "v8 · revision %{x:,}<br>Win rate: %{y:.2f}%<br>Commit %{customdata}<extra></extra>"
15
+ }, ...data.baselines.map((baseline, index) => ({
16
+ type: "scatter", mode: "lines",
17
+ name: `${baseline.name} (${baseline.score.toFixed(1)}%)`,
18
+ x: extent, y: [baseline.score, baseline.score],
19
+ line: {color: ["#999999", "#555555", "#000000"][index], width: 1.5,
20
+ dash: ["dot", "dash", "dashdot"][index]},
21
+ hovertemplate: `${baseline.name}<br>Win rate: %{y:.2f}%<extra></extra>`
22
+ }))];
23
+ await Plotly.newPlot("progress-chart", traces, {
24
+ margin: {l: 60, r: 12, t: 12, b: 130},
25
+ paper_bgcolor: "#ffffff", plot_bgcolor: "#ffffff",
26
+ font: {family: "'Letter Gothic', monospace", size: 12, color: "#000000"},
27
+ xaxis: {title: {text: "Evaluated revisions", standoff: 12},
28
+ range: extent, gridcolor: "#eeeeee", zeroline: false, automargin: true},
29
+ yaxis: {title: {text: "Win rate (%)", standoff: 10}, rangemode: "tozero",
30
+ gridcolor: "#eeeeee", zerolinecolor: "#dddddd", automargin: true},
31
+ legend: {orientation: "h", x: 0, y: -0.25, yanchor: "top", font: {size: 11}},
32
+ hoverlabel: {bgcolor: "#ffffff", bordercolor: "#000000"}
33
+ }, {responsive: true, displayModeBar: false, displaylogo: false});
34
+ status.textContent = `${points.length.toLocaleString()} evaluations shown · ${data.excluded} outage evaluations excluded · Snapshot ${data.fetched_at.slice(0, 10)}.`;
35
+ } catch (error) {
36
+ document.getElementById("progress-chart").hidden = true;
37
+ status.textContent = "The progress chart is unavailable. View the evaluation history on the leaderboard below.";
38
+ }
39
+ }
40
+ drawProgress();