File size: 3,857 Bytes
a3baf17 0282214 a3baf17 0282214 a3baf17 0282214 a3baf17 0282214 a3baf17 0282214 a3baf17 0282214 a3baf17 0282214 a3baf17 0282214 a3baf17 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Compare the four runs using recorded training token counts."""
import argparse
import csv
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def main():
parser = argparse.ArgumentParser()
here = Path(__file__).resolve().parent
output = here / "part2_results" / "output"
parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output")
parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens",
help="dataset uses one pass over the selected training subset as 1x.")
args = parser.parse_args()
runs = {}
directories = {}
for name in ("adamw", "mars", "lion", "muon"):
directory = args.output_dir / name
path = directory / "metrics.json"
if path.exists():
runs[name] = json.loads(path.read_text())
directories[name] = directory
if not runs:
raise FileNotFoundError("No metrics.json files found under the output directory.")
dataset_sizes = {}
if args.x_axis == "dataset":
records = []
for name, history in runs.items():
settings = json.loads((directories[name] / "settings.json").read_text())
size = settings["data"]["train_target_tokens"]
if size <= 0:
raise ValueError(f"{name}: selected training dataset must contain target tokens.")
dataset_sizes[name] = size
for row in history:
records.append({**row, "optimizer": name, "experiment_dataset_tokens": size,
"experiment_dataset_passes": row["train_tokens"] / size,
"original_train_split_tokens": settings["data"]["full_train_target_tokens"]})
print(f"{name}: 1x dataset = {size:,} selected target tokens; "
f"final coverage = {history[-1]['train_tokens'] / size:.5f}x")
fields = list(dict.fromkeys(key for row in records for key in row))
with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
writer.writerows(records)
def draw(axis, metric, label):
for name, history in runs.items():
denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1
axis.plot([row["train_tokens"] / denominator for row in history],
[row[metric] for row in history], marker="o", label=name)
if args.x_axis == "dataset":
axis.set_xlabel("Training tokens / selected training dataset tokens (passes)")
axis.set_xticks([i / 10 for i in range(11)])
axis.set_title("Optimizer comparison: selected subset is the experiment dataset")
else:
axis.set(xlabel="Training target tokens seen", title="Optimizer comparison")
axis.set_ylabel(label)
axis.grid(alpha=0.25)
axis.legend()
for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")):
figure, axis = plt.subplots(figsize=(8, 5), layout="constrained")
draw(axis, metric, label)
suffix = "_vs_dataset" if args.x_axis == "dataset" else ""
figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180)
plt.close(figure)
if args.x_axis == "dataset":
figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained")
draw(axes[0], "validation_loss", "Validation cross entropy")
draw(axes[1], "test_bleu", "Test BLEU (0-100)")
figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180)
plt.close(figure)
if __name__ == "__main__":
main()
|