File size: 5,081 Bytes
079e13c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | """Chart: Beam's reasoning-effort dial next to Dory-Beam-4B's, on the evaluation set.
.venv/bin/python -I scripts/make_effort_chart.py # -> release/model/assets/beam_effort_dial.png (+ dataset copy)
Accuracy vs mean output tokens per answer, for three groups of benchmarks: all six, AIME 2024 + 2025 (hard), and
MATH-500 + AMC 2023 (routine). Beam (teacher) in categorical slot 2 (orange, as on the Space), Dory-Beam-4B in slot 1
(blue), base Dory as a de-emphasised gray point. Reads data/beam_effort/summary.json from scripts/beam_effort_study.py.
"""
import json
import shutil
import sys
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from matplotlib.ticker import FuncFormatter # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
from make_chart import SURFACE, INK, INK2, MUTED, GRID, AXIS, BLUE, ORANGE, BASE # noqa: E402
OUT = ROOT / "release" / "model" / "assets" / "beam_effort_dial.png"
GROUPS = [("All six benchmarks", "all"), ("AIME 2024 + 2025", "aime")]
# label offset (points) and alignment per point; Beam's top three sit close together
OFFSETS = {"minimal": ((8, -3), "left"), "low": ((8, -3), "left"), "medium": ((8, -3), "left"),
"high": ((-8, 4), "right"), "xhigh": ((8, 4), "left"), "max": ((8, -11), "left")}
DORY_OFFSETS = {"low": ((-8, -3), "right"), "medium": ((8, -3), "left"), "high": ((8, -3), "left")}
BEAM_LEVELS = ["minimal", "low", "medium", "high", "xhigh", "max"]
SUMMARY = ROOT / "data" / "beam_effort" / "summary.json" # written by scripts/beam_effort_study.py
def main():
chart = json.load(open(SUMMARY))["chart"]
k = lambda p: (p[0] / 1000, p[1])
plt.rcParams.update({"font.family": ["Helvetica Neue", "Helvetica", "Arial", "DejaVu Sans"],
"font.size": 10, "axes.edgecolor": AXIS, "axes.labelcolor": INK2,
"xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK})
fig, axes = plt.subplots(1, 2, figsize=(11, 5.6), facecolor=SURFACE)
fig.subplots_adjust(left=0.07, right=0.98, top=0.71, bottom=0.12, wspace=0.18)
for ax, (title, g) in zip(axes, GROUPS):
ax.set_facecolor(SURFACE)
for side in ("top", "right"):
ax.spines[side].set_visible(False)
ax.grid(True, color=GRID, linewidth=1, zorder=0)
ax.tick_params(length=0, pad=4)
series = [
("Beam-1.0-501B (teacher)", ORANGE, [(lv, k(chart[g]["beam"][lv])) for lv in BEAM_LEVELS]),
("Dory-Beam-4B (student) · 2 loops", BLUE,
[(lv, k(chart[g]["dory_beam"][lv])) for lv in ("low", "medium", "high")]),
("Dory1.0-4B (base) · 2 loops", BASE, [("base", k(chart[g]["base"]))]),
]
pts = []
for label, color, ps in series:
xs = [p[1][0] for p in ps]
ys = [p[1][1] for p in ps]
if len(ps) > 1:
ax.plot(xs, ys, color=color, linewidth=2, solid_capstyle="round", solid_joinstyle="round", zorder=3)
ax.plot(xs, ys, "o", color=color, markersize=8, markeredgecolor=SURFACE, markeredgewidth=2, zorder=4,
label=label)
for name, (x, y) in ps:
off, ha = (OFFSETS if color == ORANGE else DORY_OFFSETS).get(name, ((8, -3), "left"))
ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha)
pts.append((x, y))
ys = [p[1] for p in pts]
lo, hi = min(ys), max(ys)
pad = max(2.0, (hi - lo) * 0.12)
ax.set_ylim(max(0, lo - pad), min(101, hi + pad))
ax.set_xlim(0, max(p[0] for p in pts) * 1.14)
ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: "0" if v == 0 else f"{v:g}k"))
ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:g}%"))
ax.set_title(title, loc="left", fontsize=11.5, fontweight="semibold", color=INK, pad=8)
ax.set_xlabel("mean output tokens per answer", fontsize=9.5, color=INK2)
axes[0].set_ylabel("accuracy", fontsize=9.5, color=INK2)
fig.text(0.07, 0.95, "Beam's reasoning-effort dial, and the one Dory-Beam-4B learned from it",
fontsize=16, fontweight="semibold", color=INK)
fig.text(0.07, 0.915, "Same 1,836 evaluation answers at every level; labels give the reasoning effort. Beam: self-hosted "
"NVFP4,\ntemperature 0.7, up to 28k tokens. Dory: temperature 1.0, up to 64k.",
fontsize=10, color=INK2, va="top")
handles, labels = axes[0].get_legend_handles_labels()
fig.legend(handles, labels, loc="upper left", bbox_to_anchor=(0.063, 0.83), ncol=3, frameon=False,
fontsize=10, handletextpad=0.4, columnspacing=1.6)
OUT.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(OUT, dpi=200, facecolor=SURFACE)
dst = ROOT / "release" / "dataset" / "assets"
dst.mkdir(parents=True, exist_ok=True)
shutil.copy(OUT, dst / OUT.name)
print("wrote", OUT)
if __name__ == "__main__":
main()
|