Spaces:
Running
Running
Download build_progress.py from HuggingFaceSwarms/PyBot: direct link, hf CLI and curl.
- Browser
- Download file 6.1 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/PyBot/resolve/f6eeecfaadc6fe7c25c122fe581fcaee809a3b23/build_progress.py
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/PyBot@f6eeecfaadc6fe7c25c122fe581fcaee809a3b23/build_progress.py
-
curl -L -o build_progress.py https://huggingface.co/spaces/HuggingFaceSwarms/PyBot/resolve/f6eeecfaadc6fe7c25c122fe581fcaee809a3b23/build_progress.py
6.1 kB
| """Build the local chart snapshot from the leaderboard's cached summaries.""" | |
| import argparse | |
| import json | |
| import math | |
| import subprocess | |
| from pathlib import Path | |
| # The abrupt outage plateau, bounded by the last/first affected commit timestamps. | |
| # Explicit bounds retain genuine regressions elsewhere; never filter on score. | |
| OUTAGE_START = "20261107T121247Z" | |
| OUTAGE_END = "20261107T131511Z" | |
| MODEL = "HuggingFaceSwarms/SwarmChat-v8" | |
| def count_source_tree(repository, commit): | |
| """Count a source tree directly for evaluated commits outside the main history.""" | |
| tree = subprocess.check_output( | |
| ["git", "-C", str(repository), "ls-tree", "-r", commit, "--", "bot"], text=True) | |
| objects = [line.split()[2] for line in tree.splitlines() if line.endswith(".py")] | |
| total = 0 | |
| with subprocess.Popen(["git", "-C", str(repository), "cat-file", "--batch"], | |
| stdin=subprocess.PIPE, stdout=subprocess.PIPE) as process: | |
| for oid in objects: | |
| process.stdin.write((oid + "\n").encode()) | |
| process.stdin.flush() | |
| size = int(process.stdout.readline().split()[2]) | |
| blob = process.stdout.read(size) | |
| process.stdout.read(1) | |
| total += blob.count(b"\n") + bool(blob and not blob.endswith(b"\n")) | |
| process.stdin.close() | |
| if process.wait() != 0: | |
| raise RuntimeError(f"Could not read source tree for {commit}") | |
| return total | |
| def source_line_counts(repository): | |
| """Physical lines in bot/**/*.py, including blanks/comments, excluding tests. | |
| Accumulate first-parent diffs, including deletions and merge changes. Disabling | |
| rename detection makes moves across the bot/ boundary count correctly. | |
| """ | |
| command = ["git", "-C", str(repository), "log", "--first-parent", "--reverse", | |
| "--format=commit:%H", "--numstat", "--no-renames", | |
| "--diff-merges=first-parent", "HEAD"] | |
| counts, commit, total = {}, None, 0 | |
| with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process: | |
| for line in process.stdout: | |
| line = line.rstrip("\n") | |
| if line.startswith("commit:"): | |
| if commit is not None: | |
| counts[commit[:12]] = total | |
| commit = line.removeprefix("commit:") | |
| elif "\t" in line: | |
| added, removed, path = line.split("\t", 2) | |
| if path.startswith("bot/") and path.endswith(".py"): | |
| if added == "-" or removed == "-": | |
| raise ValueError(f"Cannot count binary Python file: {path}") | |
| total += int(added) - int(removed) | |
| if process.wait() != 0: | |
| raise RuntimeError("Could not read source history") | |
| if commit is not None: | |
| counts[commit[:12]] = total | |
| return counts | |
| def build_progress(cache): | |
| latest = {} | |
| for path, summary in cache["summaries"].items(): | |
| meta = summary.get("metadata", {}) | |
| metric = summary.get("primary_metric", {}) | |
| score = metric.get("value") | |
| if (meta.get("task") != "smoltalk_bench_gold" | |
| or meta.get("status") != "success" or meta.get("task_args") | |
| or meta.get("total_samples") != 1000 | |
| or metric.get("name") != "length_penalised_win_rate" | |
| or not isinstance(score, (int, float)) or not math.isfinite(score)): | |
| continue | |
| key = (meta["model_id"], meta["revision"]) | |
| order = (meta["completed_at"], path) | |
| if key not in latest or order > latest[key][0]: | |
| latest[key] = (order, summary) | |
| revisions = sorted((revision, summary) for (model, revision), (_, summary) | |
| in latest.items() if model == MODEL) | |
| points, excluded = [], 0 | |
| for index, (revision, summary) in enumerate(revisions, 1): | |
| if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END: | |
| excluded += 1 | |
| continue | |
| points.append({"revision": index, "commit": revision.split("-")[1], | |
| "score": summary["primary_metric"]["value"]}) | |
| baselines = sorted([ | |
| {"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]} | |
| for (model, revision), (_, summary) in latest.items() | |
| if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main" | |
| ], key=lambda baseline: baseline["score"]) | |
| if not points or len(baselines) != 3: | |
| raise ValueError("Expected v8 evaluations and all three SmolLM baselines") | |
| return {"points": points, "baselines": baselines, "excluded": excluded, | |
| "fetched_at": cache["fetched_at"]} | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("cache", type=Path) | |
| parser.add_argument("--source-repo", type=Path, required=True, | |
| help="Git repository containing the evaluated source history") | |
| args = parser.parse_args() | |
| result = build_progress(json.loads(args.cache.read_text())) | |
| counts = source_line_counts(args.source_repo) | |
| missing = [] | |
| for point in result["points"]: | |
| if point["commit"] not in counts: | |
| exists = subprocess.run( | |
| ["git", "-C", str(args.source_repo), "cat-file", "-e", point["commit"]], | |
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 | |
| counts[point["commit"]] = count_source_tree(args.source_repo, point["commit"]) if exists else None | |
| if counts[point["commit"]] is None: | |
| missing.append(point["commit"]) | |
| point["lines_of_code"] = counts[point["commit"]] | |
| result["missing_source_commits"] = missing | |
| if missing: | |
| print(f"No source available for {len(missing)} commits; leaving code-size gaps: {', '.join(missing)}") | |
| output = Path(__file__).parent / "scratch" / "progress.json" | |
| output.parent.mkdir(exist_ok=True) | |
| output.write_text(json.dumps(result, separators=(",", ":")) + "\n") | |
| print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations") | |