anlp-assignment2-part2 / source /part2 /plot_comparison.py
walke007's picture
Upload Part 2 checkpoints and recorded results
0282214 verified
Raw History Blame Contribute Delete
3.86 kB
"""Compare the four runs using recorded training token counts."""
import argparse
import csv
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def main():
parser = argparse.ArgumentParser()
here = Path(__file__).resolve().parent
output = here / "part2_results" / "output"
parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output")
parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens",
help="dataset uses one pass over the selected training subset as 1x.")
args = parser.parse_args()
runs = {}
directories = {}
for name in ("adamw", "mars", "lion", "muon"):
directory = args.output_dir / name
path = directory / "metrics.json"
if path.exists():
runs[name] = json.loads(path.read_text())
directories[name] = directory
if not runs:
raise FileNotFoundError("No metrics.json files found under the output directory.")
dataset_sizes = {}
if args.x_axis == "dataset":
records = []
for name, history in runs.items():
settings = json.loads((directories[name] / "settings.json").read_text())
size = settings["data"]["train_target_tokens"]
if size <= 0:
raise ValueError(f"{name}: selected training dataset must contain target tokens.")
dataset_sizes[name] = size
for row in history:
records.append({**row, "optimizer": name, "experiment_dataset_tokens": size,
"experiment_dataset_passes": row["train_tokens"] / size,
"original_train_split_tokens": settings["data"]["full_train_target_tokens"]})
print(f"{name}: 1x dataset = {size:,} selected target tokens; "
f"final coverage = {history[-1]['train_tokens'] / size:.5f}x")
fields = list(dict.fromkeys(key for row in records for key in row))
with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
writer.writerows(records)
def draw(axis, metric, label):
for name, history in runs.items():
denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1
axis.plot([row["train_tokens"] / denominator for row in history],
[row[metric] for row in history], marker="o", label=name)
if args.x_axis == "dataset":
axis.set_xlabel("Training tokens / selected training dataset tokens (passes)")
axis.set_xticks([i / 10 for i in range(11)])
axis.set_title("Optimizer comparison: selected subset is the experiment dataset")
else:
axis.set(xlabel="Training target tokens seen", title="Optimizer comparison")
axis.set_ylabel(label)
axis.grid(alpha=0.25)
axis.legend()
for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")):
figure, axis = plt.subplots(figsize=(8, 5), layout="constrained")
draw(axis, metric, label)
suffix = "_vs_dataset" if args.x_axis == "dataset" else ""
figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180)
plt.close(figure)
if args.x_axis == "dataset":
figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained")
draw(axes[0], "validation_loss", "Validation cross entropy")
draw(axes[1], "test_bleu", "Test BLEU (0-100)")
figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180)
plt.close(figure)
if __name__ == "__main__":
main()