File size: 3,857 Bytes
a3baf17
 
 
0282214
a3baf17
 
 
 
 
 
 
 
 
 
0282214
 
 
 
 
a3baf17
 
0282214
a3baf17
0282214
a3baf17
 
 
0282214
a3baf17
 
0282214
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a3baf17
0282214
 
 
 
 
 
 
 
 
 
 
a3baf17
0282214
 
 
 
 
 
 
 
 
 
 
 
a3baf17
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
"""Compare the four runs using recorded training token counts."""

import argparse
import csv
import json
from pathlib import Path

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt


def main():
    parser = argparse.ArgumentParser()
    here = Path(__file__).resolve().parent
    output = here / "part2_results" / "output"
    parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output")
    parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens",
                        help="dataset uses one pass over the selected training subset as 1x.")
    args = parser.parse_args()
    runs = {}
    directories = {}
    for name in ("adamw", "mars", "lion", "muon"):
        directory = args.output_dir / name
        path = directory / "metrics.json"
        if path.exists():
            runs[name] = json.loads(path.read_text())
            directories[name] = directory
    if not runs:
        raise FileNotFoundError("No metrics.json files found under the output directory.")
    dataset_sizes = {}
    if args.x_axis == "dataset":
        records = []
        for name, history in runs.items():
            settings = json.loads((directories[name] / "settings.json").read_text())
            size = settings["data"]["train_target_tokens"]
            if size <= 0:
                raise ValueError(f"{name}: selected training dataset must contain target tokens.")
            dataset_sizes[name] = size
            for row in history:
                records.append({**row, "optimizer": name, "experiment_dataset_tokens": size,
                                "experiment_dataset_passes": row["train_tokens"] / size,
                                "original_train_split_tokens": settings["data"]["full_train_target_tokens"]})
            print(f"{name}: 1x dataset = {size:,} selected target tokens; "
                  f"final coverage = {history[-1]['train_tokens'] / size:.5f}x")
        fields = list(dict.fromkeys(key for row in records for key in row))
        with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream:
            writer = csv.DictWriter(stream, fieldnames=fields)
            writer.writeheader()
            writer.writerows(records)

    def draw(axis, metric, label):
        for name, history in runs.items():
            denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1
            axis.plot([row["train_tokens"] / denominator for row in history],
                      [row[metric] for row in history], marker="o", label=name)
        if args.x_axis == "dataset":
            axis.set_xlabel("Training tokens / selected training dataset tokens (passes)")
            axis.set_xticks([i / 10 for i in range(11)])
            axis.set_title("Optimizer comparison: selected subset is the experiment dataset")
        else:
            axis.set(xlabel="Training target tokens seen", title="Optimizer comparison")
        axis.set_ylabel(label)
        axis.grid(alpha=0.25)
        axis.legend()

    for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")):
        figure, axis = plt.subplots(figsize=(8, 5), layout="constrained")
        draw(axis, metric, label)
        suffix = "_vs_dataset" if args.x_axis == "dataset" else ""
        figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180)
        plt.close(figure)
    if args.x_axis == "dataset":
        figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained")
        draw(axes[0], "validation_loss", "Validation cross entropy")
        draw(axes[1], "test_bleu", "Test BLEU (0-100)")
        figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180)
        plt.close(figure)


if __name__ == "__main__":
    main()