File size: 4,536 Bytes
dafe8d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | """Budget-forcing chart for the model card: overall accuracy vs thinking budget at 2 loops, for base Dory,
Dory-Beam-4B (default effort) and Dory-Beam-4B-Accurate.
python3 scripts/make_budget_chart.py # -> release/model/assets/accuracy_vs_budget.png
At each budget, a sample still thinking is closed with "\n</think>" and gets up to 1,024 answer tokens
(modal_app/eval_dory.py::budget_force). The last point is the uncapped 64k run. Same emphasis form and colors as
scripts/make_chart.py: base in the de-emphasis gray, the two Dory-Beam models in categorical slots 1 and 2.
"""
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
BUDGET = ROOT / "data" / "eval" / "budget"
RES = ROOT / "data" / "eval" / "results"
OUT = ROOT / "release" / "model" / "assets" / "accuracy_vs_budget.png"
SURFACE, INK, INK2, MUTED, GRID, AXIS = "#fcfcfb", "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
BLUE, ORANGE, BASE = "#2a78d6", "#eb6834", "#898781"
CAPS = [2048, 4096, 8192, 16384]
# (legend label, direct label, color, tag), drawn back to front
SERIES = [("Dory1.0-4B (base)", "base", BASE, "base_loops2-64k"),
("Dory-Beam-4B-Accurate", "Accurate", ORANGE, "mix030_loops2-64k"),
("Dory-Beam-4B, default effort", "Dory-Beam", BLUE, "rE-t050_loops2-default-64k")]
def curve(tag):
hit = {c: [0, 0] for c in CAPS}
for r in map(json.loads, open(BUDGET / f"{tag}.jsonl")):
hit[r["cap"]][0] += r["correct"]
hit[r["cap"]][1] += 1
full = [json.loads(l)["correct"] for l in open(RES / f"{tag}.jsonl")]
return [100 * a / n for a, n in hit.values()] + [100 * sum(full) / len(full)]
def main():
plt.rcParams.update({"font.family": ["Helvetica Neue", "Helvetica", "Arial", "DejaVu Sans"],
"font.size": 10, "axes.edgecolor": AXIS, "axes.labelcolor": INK2,
"xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK})
fig, ax = plt.subplots(figsize=(9, 5.6), facecolor=SURFACE)
fig.subplots_adjust(left=0.08, right=0.74, top=0.74, bottom=0.12)
ax.set_facecolor(SURFACE)
for side in ("top", "right"):
ax.spines[side].set_visible(False)
ax.grid(True, axis="y", color=GRID, linewidth=1, zorder=0)
ax.tick_params(length=0, pad=6)
xs = list(range(len(CAPS) + 1))
ends = []
for label, short, color, tag in SERIES:
ys = curve(tag)
ax.plot(xs, ys, color=color, linewidth=2, solid_capstyle="round", solid_joinstyle="round", zorder=3)
ax.plot(xs, ys, "o", color=color, markersize=8, markeredgecolor=SURFACE, markeredgewidth=2, zorder=4,
label=label)
ends.append((ys[0], short))
# direct labels in muted ink at the left end, where the lines are furthest apart
for y, label in ends:
ax.annotate(label, (0, y), xytext=(-10, 0), textcoords="offset points", ha="right",
va="center", fontsize=9, color=MUTED)
ax.set_xlim(-0.75, len(CAPS) + 0.25)
ax.set_xticks(xs, ["2k", "4k", "8k", "16k", "no cap\n(64k)"])
ax.set_ylim(50, 92)
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda v, _: f"{v:g}%"))
ax.set_xlabel("thinking budget (tokens)", fontsize=9.5, color=INK2)
ax.set_ylabel("accuracy, all benchmarks", fontsize=9.5, color=INK2)
fig.text(0.08, 0.94, "On a thinking budget, Dory-Beam-4B answers more correctly", fontsize=15,
fontweight="semibold", color=INK)
fig.text(0.08, 0.895, "At the budget, a model still thinking is told to stop and answer (up to 1,024 more tokens). "
"2 loops,\n1,836 samples over GPQA, MATH-500, AIME 24/25, AMC 23 and 300 held-out problems.",
fontsize=9.5, color=INK2, va="top")
handles, labels = ax.get_legend_handles_labels()
fig.legend(handles[::-1], labels[::-1], loc="upper left", bbox_to_anchor=(0.073, 0.825), ncol=3, frameon=False,
fontsize=9.5, handletextpad=0.4, columnspacing=1.6)
fig.text(0.765, 0.70, "Dory-Beam-4B minus base,\n95% CI\n\n"
"2k +8.7 (+6.3 to +11.1)\n4k +7.7 (+5.3 to +10.1)\n8k +3.9 (+2.0 to +5.7)\n"
"16k +2.3 (+0.4 to +4.2)\n\nDefault effort. Paired\nbootstrap over problems.",
fontsize=8.5, color=INK2, va="top", linespacing=1.35)
OUT.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(OUT, dpi=200, facecolor=SURFACE)
print("wrote", OUT)
if __name__ == "__main__":
main()
|