File size: 6,102 Bytes
a5edae5
 
 
 
 
5c9ff26
a5edae5
 
 
 
 
 
 
 
 
5c9ff26
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a5edae5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5c9ff26
 
a5edae5
 
5c9ff26
 
 
 
 
 
 
 
 
 
 
 
 
 
a5edae5
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
"""Build the local chart snapshot from the leaderboard's cached summaries."""

import argparse
import json
import math
import subprocess
from pathlib import Path

# The abrupt outage plateau, bounded by the last/first affected commit timestamps.
# Explicit bounds retain genuine regressions elsewhere; never filter on score.
OUTAGE_START = "20261107T121247Z"
OUTAGE_END = "20261107T131511Z"
MODEL = "HuggingFaceSwarms/SwarmChat-v8"


def count_source_tree(repository, commit):
    """Count a source tree directly for evaluated commits outside the main history."""
    tree = subprocess.check_output(
        ["git", "-C", str(repository), "ls-tree", "-r", commit, "--", "bot"], text=True)
    objects = [line.split()[2] for line in tree.splitlines() if line.endswith(".py")]
    total = 0
    with subprocess.Popen(["git", "-C", str(repository), "cat-file", "--batch"],
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE) as process:
        for oid in objects:
            process.stdin.write((oid + "\n").encode())
            process.stdin.flush()
            size = int(process.stdout.readline().split()[2])
            blob = process.stdout.read(size)
            process.stdout.read(1)
            total += blob.count(b"\n") + bool(blob and not blob.endswith(b"\n"))
        process.stdin.close()
        if process.wait() != 0:
            raise RuntimeError(f"Could not read source tree for {commit}")
    return total


def source_line_counts(repository):
    """Physical lines in bot/**/*.py, including blanks/comments, excluding tests.

    Accumulate first-parent diffs, including deletions and merge changes. Disabling
    rename detection makes moves across the bot/ boundary count correctly.
    """
    command = ["git", "-C", str(repository), "log", "--first-parent", "--reverse",
               "--format=commit:%H", "--numstat", "--no-renames",
               "--diff-merges=first-parent", "HEAD"]
    counts, commit, total = {}, None, 0
    with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process:
        for line in process.stdout:
            line = line.rstrip("\n")
            if line.startswith("commit:"):
                if commit is not None:
                    counts[commit[:12]] = total
                commit = line.removeprefix("commit:")
            elif "\t" in line:
                added, removed, path = line.split("\t", 2)
                if path.startswith("bot/") and path.endswith(".py"):
                    if added == "-" or removed == "-":
                        raise ValueError(f"Cannot count binary Python file: {path}")
                    total += int(added) - int(removed)
        if process.wait() != 0:
            raise RuntimeError("Could not read source history")
    if commit is not None:
        counts[commit[:12]] = total
    return counts


def build_progress(cache):
    latest = {}
    for path, summary in cache["summaries"].items():
        meta = summary.get("metadata", {})
        metric = summary.get("primary_metric", {})
        score = metric.get("value")
        if (meta.get("task") != "smoltalk_bench_gold"
                or meta.get("status") != "success" or meta.get("task_args")
                or meta.get("total_samples") != 1000
                or metric.get("name") != "length_penalised_win_rate"
                or not isinstance(score, (int, float)) or not math.isfinite(score)):
            continue
        key = (meta["model_id"], meta["revision"])
        order = (meta["completed_at"], path)
        if key not in latest or order > latest[key][0]:
            latest[key] = (order, summary)
    revisions = sorted((revision, summary) for (model, revision), (_, summary)
                       in latest.items() if model == MODEL)
    points, excluded = [], 0
    for index, (revision, summary) in enumerate(revisions, 1):
        if OUTAGE_START <= revision.split("-")[0] <= OUTAGE_END:
            excluded += 1
            continue
        points.append({"revision": index, "commit": revision.split("-")[1],
                       "score": summary["primary_metric"]["value"]})
    baselines = sorted([
        {"name": model.split("/")[-1], "score": summary["primary_metric"]["value"]}
        for (model, revision), (_, summary) in latest.items()
        if model.startswith("HuggingFaceTB/SmolLM-") and revision == "main"
    ], key=lambda baseline: baseline["score"])
    if not points or len(baselines) != 3:
        raise ValueError("Expected v8 evaluations and all three SmolLM baselines")
    return {"points": points, "baselines": baselines, "excluded": excluded,
            "fetched_at": cache["fetched_at"]}


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("cache", type=Path)
    parser.add_argument("--source-repo", type=Path, required=True,
                        help="Git repository containing the evaluated source history")
    args = parser.parse_args()
    result = build_progress(json.loads(args.cache.read_text()))
    counts = source_line_counts(args.source_repo)
    missing = []
    for point in result["points"]:
        if point["commit"] not in counts:
            exists = subprocess.run(
                ["git", "-C", str(args.source_repo), "cat-file", "-e", point["commit"]],
                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
            counts[point["commit"]] = count_source_tree(args.source_repo, point["commit"]) if exists else None
        if counts[point["commit"]] is None:
            missing.append(point["commit"])
        point["lines_of_code"] = counts[point["commit"]]
    result["missing_source_commits"] = missing
    if missing:
        print(f"No source available for {len(missing)} commits; leaving code-size gaps: {', '.join(missing)}")
    output = Path(__file__).parent / "scratch" / "progress.json"
    output.parent.mkdir(exist_ok=True)
    output.write_text(json.dumps(result, separators=(",", ":")) + "\n")
    print(f"Wrote {len(result['points'])} points; excluded {result['excluded']} outage evaluations")