"""Compare the four runs using recorded training token counts.""" import argparse import csv import json from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt def main(): parser = argparse.ArgumentParser() here = Path(__file__).resolve().parent output = here / "part2_results" / "output" parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output") parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens", help="dataset uses one pass over the selected training subset as 1x.") args = parser.parse_args() runs = {} directories = {} for name in ("adamw", "mars", "lion", "muon"): directory = args.output_dir / name path = directory / "metrics.json" if path.exists(): runs[name] = json.loads(path.read_text()) directories[name] = directory if not runs: raise FileNotFoundError("No metrics.json files found under the output directory.") dataset_sizes = {} if args.x_axis == "dataset": records = [] for name, history in runs.items(): settings = json.loads((directories[name] / "settings.json").read_text()) size = settings["data"]["train_target_tokens"] if size <= 0: raise ValueError(f"{name}: selected training dataset must contain target tokens.") dataset_sizes[name] = size for row in history: records.append({**row, "optimizer": name, "experiment_dataset_tokens": size, "experiment_dataset_passes": row["train_tokens"] / size, "original_train_split_tokens": settings["data"]["full_train_target_tokens"]}) print(f"{name}: 1x dataset = {size:,} selected target tokens; " f"final coverage = {history[-1]['train_tokens'] / size:.5f}x") fields = list(dict.fromkeys(key for row in records for key in row)) with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=fields) writer.writeheader() writer.writerows(records) def draw(axis, metric, label): for name, history in runs.items(): denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1 axis.plot([row["train_tokens"] / denominator for row in history], [row[metric] for row in history], marker="o", label=name) if args.x_axis == "dataset": axis.set_xlabel("Training tokens / selected training dataset tokens (passes)") axis.set_xticks([i / 10 for i in range(11)]) axis.set_title("Optimizer comparison: selected subset is the experiment dataset") else: axis.set(xlabel="Training target tokens seen", title="Optimizer comparison") axis.set_ylabel(label) axis.grid(alpha=0.25) axis.legend() for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")): figure, axis = plt.subplots(figsize=(8, 5), layout="constrained") draw(axis, metric, label) suffix = "_vs_dataset" if args.x_axis == "dataset" else "" figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180) plt.close(figure) if args.x_axis == "dataset": figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained") draw(axes[0], "validation_loss", "Validation cross entropy") draw(axes[1], "test_bleu", "Test BLEU (0-100)") figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180) plt.close(figure) if __name__ == "__main__": main()