"""Build the local chart snapshot from the leaderboard's cached summaries.""" import argparse import json import math import subprocess from pathlib import Path # The abrupt outage plateau, bounded by the last/first affected commit timestamps. # Explicit bounds retain genuine regressions elsewhere; never filter on score. OUTAGE_START = "20261107T121247Z" OUTAGE_END = "20261107T131511Z" MODEL = "HuggingFaceSwarms/SwarmChat-v8" def count_source_tree(repository, commit): """Count a source tree directly for evaluated commits outside the main history.""" tree = subprocess.check_output( ["git", "-C", str(repository), "ls-tree", "-r", commit, "--", "bot"], text=True) objects = [line.split()[2] for line in tree.splitlines() if line.endswith(".py")] total = 0 with subprocess.Popen(["git", "-C", str(repository), "cat-file", "--batch"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) as process: for oid in objects: process.stdin.write((oid + "\n").encode()) process.stdin.flush() size = int(process.stdout.readline().split()[2]) blob = process.stdout.read(size) process.stdout.read(1) total += blob.count(b"\n") + bool(blob and not blob.endswith(b"\n")) process.stdin.close() if process.wait() != 0: raise RuntimeError(f"Could not read source tree for {commit}") return total def source_line_counts(repository): """Physical lines in bot/**/*.py, including blanks/comments, excluding tests. Accumulate first-parent diffs, including deletions and merge changes. Disabling rename detection makes moves across the bot/ boundary count correctly. """ command = ["git", "-C", str(repository), "log", "--first-parent", "--reverse", "--format=commit:%H", "--numstat", "--no-renames", "--diff-merges=first-parent", "HEAD"] counts, commit, total = {}, None, 0 with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process: for line in process.stdout: line = line.rstrip("\n") if line.startswith("commit:"): if commit is not None: counts[commit[:12]] = total commit = line.removeprefix("commit:") elif "\t" in line: added, removed, path = line.split("\t", 2) if path.startswith("bot/") and path.endswith(".py"): if added == "-" or removed == "-": raise ValueError(f"Cannot count binary Python file: {path}") total += int(added) - int(removed) if process.wait() != 0: raise RuntimeError("Could not read source history") if commit is not None: counts[commit[:12]] = total return counts def build_progress(cache): latest = {} for path, summary in cache["summaries"].items(): meta = summary.get("metadata", {}) metric = summary.get("primary_metric", {}) score = metric.get("value") if (meta.get("task") != "smoltalk_bench_gold" or meta.get("status") != "success" or meta.get("task_args") or meta.get("total_samples") != 1000 or metric.get("name") != "length_penalised_win_rate" or not isinstance(score, (int, float)) or not math.isfinite(score)): continue key = (meta["model_id"], meta["revision"]) order = (meta["completed_at"], path) if key not in latest or order > latest[key][0]: latest[key] = (order, summary) revisions = sorted((revision, summary) for (model, revision), (_, summary) in latest.items() if model == MODEL) points, excluded = [], 0 for index, (revision, summary) in enumerate(revisions, 1): if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END: excluded += 1 continue points.append({"revision": index, "commit": revision.split("-")[1], "score": summary["primary_metric"]["value"]}) baselines = sorted([ {"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]} for (model, revision), (_, summary) in latest.items() if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main" ], key=lambda baseline: baseline["score"]) if not points or len(baselines) != 3: raise ValueError("Expected v8 evaluations and all three SmolLM baselines") return {"points": points, "baselines": baselines, "excluded": excluded, "fetched_at": cache["fetched_at"]} if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("cache", type=Path) parser.add_argument("--source-repo", type=Path, required=True, help="Git repository containing the evaluated source history") args = parser.parse_args() result = build_progress(json.loads(args.cache.read_text())) counts = source_line_counts(args.source_repo) missing = [] for point in result["points"]: if point["commit"] not in counts: exists = subprocess.run( ["git", "-C", str(args.source_repo), "cat-file", "-e", point["commit"]], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 counts[point["commit"]] = count_source_tree(args.source_repo, point["commit"]) if exists else None if counts[point["commit"]] is None: missing.append(point["commit"]) point["lines_of_code"] = counts[point["commit"]] result["missing_source_commits"] = missing if missing: print(f"No source available for {len(missing)} commits; leaving code-size gaps: {', '.join(missing)}") output = Path(__file__).parent / "scratch" / "progress.json" output.parent.mkdir(exist_ok=True) output.write_text(json.dumps(result, separators=(",", ":")) + "\n") print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations")