Spaces:
Running
Running
File size: 6,102 Bytes
a5edae5 5c9ff26 a5edae5 5c9ff26 a5edae5 5c9ff26 a5edae5 5c9ff26 a5edae5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | """Build the local chart snapshot from the leaderboard's cached summaries."""
import argparse
import json
import math
import subprocess
from pathlib import Path
# The abrupt outage plateau, bounded by the last/first affected commit timestamps.
# Explicit bounds retain genuine regressions elsewhere; never filter on score.
OUTAGE_START = "20261107T121247Z"
OUTAGE_END = "20261107T131511Z"
MODEL = "HuggingFaceSwarms/SwarmChat-v8"
def count_source_tree(repository, commit):
"""Count a source tree directly for evaluated commits outside the main history."""
tree = subprocess.check_output(
["git", "-C", str(repository), "ls-tree", "-r", commit, "--", "bot"], text=True)
objects = [line.split()[2] for line in tree.splitlines() if line.endswith(".py")]
total = 0
with subprocess.Popen(["git", "-C", str(repository), "cat-file", "--batch"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE) as process:
for oid in objects:
process.stdin.write((oid + "\n").encode())
process.stdin.flush()
size = int(process.stdout.readline().split()[2])
blob = process.stdout.read(size)
process.stdout.read(1)
total += blob.count(b"\n") + bool(blob and not blob.endswith(b"\n"))
process.stdin.close()
if process.wait() != 0:
raise RuntimeError(f"Could not read source tree for {commit}")
return total
def source_line_counts(repository):
"""Physical lines in bot/**/*.py, including blanks/comments, excluding tests.
Accumulate first-parent diffs, including deletions and merge changes. Disabling
rename detection makes moves across the bot/ boundary count correctly.
"""
command = ["git", "-C", str(repository), "log", "--first-parent", "--reverse",
"--format=commit:%H", "--numstat", "--no-renames",
"--diff-merges=first-parent", "HEAD"]
counts, commit, total = {}, None, 0
with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process:
for line in process.stdout:
line = line.rstrip("\n")
if line.startswith("commit:"):
if commit is not None:
counts[commit[:12]] = total
commit = line.removeprefix("commit:")
elif "\t" in line:
added, removed, path = line.split("\t", 2)
if path.startswith("bot/") and path.endswith(".py"):
if added == "-" or removed == "-":
raise ValueError(f"Cannot count binary Python file: {path}")
total += int(added) - int(removed)
if process.wait() != 0:
raise RuntimeError("Could not read source history")
if commit is not None:
counts[commit[:12]] = total
return counts
def build_progress(cache):
latest = {}
for path, summary in cache["summaries"].items():
meta = summary.get("metadata", {})
metric = summary.get("primary_metric", {})
score = metric.get("value")
if (meta.get("task") != "smoltalk_bench_gold"
or meta.get("status") != "success" or meta.get("task_args")
or meta.get("total_samples") != 1000
or metric.get("name") != "length_penalised_win_rate"
or not isinstance(score, (int, float)) or not math.isfinite(score)):
continue
key = (meta["model_id"], meta["revision"])
order = (meta["completed_at"], path)
if key not in latest or order > latest[key][0]:
latest[key] = (order, summary)
revisions = sorted((revision, summary) for (model, revision), (_, summary)
in latest.items() if model == MODEL)
points, excluded = [], 0
for index, (revision, summary) in enumerate(revisions, 1):
if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END:
excluded += 1
continue
points.append({"revision": index, "commit": revision.split("-")[1],
"score": summary["primary_metric"]["value"]})
baselines = sorted([
{"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]}
for (model, revision), (_, summary) in latest.items()
if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main"
], key=lambda baseline: baseline["score"])
if not points or len(baselines) != 3:
raise ValueError("Expected v8 evaluations and all three SmolLM baselines")
return {"points": points, "baselines": baselines, "excluded": excluded,
"fetched_at": cache["fetched_at"]}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("cache", type=Path)
parser.add_argument("--source-repo", type=Path, required=True,
help="Git repository containing the evaluated source history")
args = parser.parse_args()
result = build_progress(json.loads(args.cache.read_text()))
counts = source_line_counts(args.source_repo)
missing = []
for point in result["points"]:
if point["commit"] not in counts:
exists = subprocess.run(
["git", "-C", str(args.source_repo), "cat-file", "-e", point["commit"]],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
counts[point["commit"]] = count_source_tree(args.source_repo, point["commit"]) if exists else None
if counts[point["commit"]] is None:
missing.append(point["commit"])
point["lines_of_code"] = counts[point["commit"]]
result["missing_source_commits"] = missing
if missing:
print(f"No source available for {len(missing)} commits; leaving code-size gaps: {', '.join(missing)}")
output = Path(__file__).parent / "scratch" / "progress.json"
output.parent.mkdir(exist_ok=True)
output.write_text(json.dumps(result, separators=(",", ":")) + "\n")
print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations")
|