PyBot / build_progress.py
lewtun's picture
lewtun HF Staff
Overlay measured Python source lines on progress chart
5c9ff26
Raw History Blame
6.1 kB
"""Build the local chart snapshot from the leaderboard's cached summaries."""
import argparse
import json
import math
import subprocess
from pathlib import Path
# The abrupt outage plateau, bounded by the last/first affected commit timestamps.
# Explicit bounds retain genuine regressions elsewhere; never filter on score.
OUTAGE_START = "20261107T121247Z"
OUTAGE_END = "20261107T131511Z"
MODEL = "HuggingFaceSwarms/SwarmChat-v8"
def count_source_tree(repository, commit):
"""Count a source tree directly for evaluated commits outside the main history."""
tree = subprocess.check_output(
["git", "-C", str(repository), "ls-tree", "-r", commit, "--", "bot"], text=True)
objects = [line.split()[2] for line in tree.splitlines() if line.endswith(".py")]
total = 0
with subprocess.Popen(["git", "-C", str(repository), "cat-file", "--batch"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE) as process:
for oid in objects:
process.stdin.write((oid + "\n").encode())
process.stdin.flush()
size = int(process.stdout.readline().split()[2])
blob = process.stdout.read(size)
process.stdout.read(1)
total += blob.count(b"\n") + bool(blob and not blob.endswith(b"\n"))
process.stdin.close()
if process.wait() != 0:
raise RuntimeError(f"Could not read source tree for {commit}")
return total
def source_line_counts(repository):
"""Physical lines in bot/**/*.py, including blanks/comments, excluding tests.
Accumulate first-parent diffs, including deletions and merge changes. Disabling
rename detection makes moves across the bot/ boundary count correctly.
"""
command = ["git", "-C", str(repository), "log", "--first-parent", "--reverse",
"--format=commit:%H", "--numstat", "--no-renames",
"--diff-merges=first-parent", "HEAD"]
counts, commit, total = {}, None, 0
with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process:
for line in process.stdout:
line = line.rstrip("\n")
if line.startswith("commit:"):
if commit is not None:
counts[commit[:12]] = total
commit = line.removeprefix("commit:")
elif "\t" in line:
added, removed, path = line.split("\t", 2)
if path.startswith("bot/") and path.endswith(".py"):
if added == "-" or removed == "-":
raise ValueError(f"Cannot count binary Python file: {path}")
total += int(added) - int(removed)
if process.wait() != 0:
raise RuntimeError("Could not read source history")
if commit is not None:
counts[commit[:12]] = total
return counts
def build_progress(cache):
latest = {}
for path, summary in cache["summaries"].items():
meta = summary.get("metadata", {})
metric = summary.get("primary_metric", {})
score = metric.get("value")
if (meta.get("task") != "smoltalk_bench_gold"
or meta.get("status") != "success" or meta.get("task_args")
or meta.get("total_samples") != 1000
or metric.get("name") != "length_penalised_win_rate"
or not isinstance(score, (int, float)) or not math.isfinite(score)):
continue
key = (meta["model_id"], meta["revision"])
order = (meta["completed_at"], path)
if key not in latest or order > latest[key][0]:
latest[key] = (order, summary)
revisions = sorted((revision, summary) for (model, revision), (_, summary)
in latest.items() if model == MODEL)
points, excluded = [], 0
for index, (revision, summary) in enumerate(revisions, 1):
if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END:
excluded += 1
continue
points.append({"revision": index, "commit": revision.split("-")[1],
"score": summary["primary_metric"]["value"]})
baselines = sorted([
{"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]}
for (model, revision), (_, summary) in latest.items()
if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main"
], key=lambda baseline: baseline["score"])
if not points or len(baselines) != 3:
raise ValueError("Expected v8 evaluations and all three SmolLM baselines")
return {"points": points, "baselines": baselines, "excluded": excluded,
"fetched_at": cache["fetched_at"]}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("cache", type=Path)
parser.add_argument("--source-repo", type=Path, required=True,
help="Git repository containing the evaluated source history")
args = parser.parse_args()
result = build_progress(json.loads(args.cache.read_text()))
counts = source_line_counts(args.source_repo)
missing = []
for point in result["points"]:
if point["commit"] not in counts:
exists = subprocess.run(
["git", "-C", str(args.source_repo), "cat-file", "-e", point["commit"]],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
counts[point["commit"]] = count_source_tree(args.source_repo, point["commit"]) if exists else None
if counts[point["commit"]] is None:
missing.append(point["commit"])
point["lines_of_code"] = counts[point["commit"]]
result["missing_source_commits"] = missing
if missing:
print(f"No source available for {len(missing)} commits; leaving code-size gaps: {', '.join(missing)}")
output = Path(__file__).parent / "scratch" / "progress.json"
output.parent.mkdir(exist_ok=True)
output.write_text(json.dumps(result, separators=(",", ":")) + "\n")
print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations")