Download source/part2/plot_comparison.py from walke007/anlp-assignment2-part2: direct link, hf CLI and curl.
- Browser
- Download file 3.86 kB
-
https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/plot_comparison.py
- Command line
-
hf download hf://walke007/anlp-assignment2-part2/source/part2/plot_comparison.py
-
curl -L -o plot_comparison.py https://huggingface.co/walke007/anlp-assignment2-part2/resolve/main/source/part2/plot_comparison.py
3.86 kB
| """Compare the four runs using recorded training token counts.""" | |
| import argparse | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| here = Path(__file__).resolve().parent | |
| output = here / "part2_results" / "output" | |
| parser.add_argument("--output-dir", type=Path, default=output if output.is_dir() else here / "output") | |
| parser.add_argument("--x-axis", choices=("tokens", "dataset"), default="tokens", | |
| help="dataset uses one pass over the selected training subset as 1x.") | |
| args = parser.parse_args() | |
| runs = {} | |
| directories = {} | |
| for name in ("adamw", "mars", "lion", "muon"): | |
| directory = args.output_dir / name | |
| path = directory / "metrics.json" | |
| if path.exists(): | |
| runs[name] = json.loads(path.read_text()) | |
| directories[name] = directory | |
| if not runs: | |
| raise FileNotFoundError("No metrics.json files found under the output directory.") | |
| dataset_sizes = {} | |
| if args.x_axis == "dataset": | |
| records = [] | |
| for name, history in runs.items(): | |
| settings = json.loads((directories[name] / "settings.json").read_text()) | |
| size = settings["data"]["train_target_tokens"] | |
| if size <= 0: | |
| raise ValueError(f"{name}: selected training dataset must contain target tokens.") | |
| dataset_sizes[name] = size | |
| for row in history: | |
| records.append({**row, "optimizer": name, "experiment_dataset_tokens": size, | |
| "experiment_dataset_passes": row["train_tokens"] / size, | |
| "original_train_split_tokens": settings["data"]["full_train_target_tokens"]}) | |
| print(f"{name}: 1x dataset = {size:,} selected target tokens; " | |
| f"final coverage = {history[-1]['train_tokens'] / size:.5f}x") | |
| fields = list(dict.fromkeys(key for row in records for key in row)) | |
| with (args.output_dir / "comparison_dataset_metrics.csv").open("w", newline="") as stream: | |
| writer = csv.DictWriter(stream, fieldnames=fields) | |
| writer.writeheader() | |
| writer.writerows(records) | |
| def draw(axis, metric, label): | |
| for name, history in runs.items(): | |
| denominator = dataset_sizes[name] if args.x_axis == "dataset" else 1 | |
| axis.plot([row["train_tokens"] / denominator for row in history], | |
| [row[metric] for row in history], marker="o", label=name) | |
| if args.x_axis == "dataset": | |
| axis.set_xlabel("Training tokens / selected training dataset tokens (passes)") | |
| axis.set_xticks([i / 10 for i in range(11)]) | |
| axis.set_title("Optimizer comparison: selected subset is the experiment dataset") | |
| else: | |
| axis.set(xlabel="Training target tokens seen", title="Optimizer comparison") | |
| axis.set_ylabel(label) | |
| axis.grid(alpha=0.25) | |
| axis.legend() | |
| for metric, label in (("validation_loss", "Validation cross entropy"), ("test_bleu", "Test BLEU (0-100)")): | |
| figure, axis = plt.subplots(figsize=(8, 5), layout="constrained") | |
| draw(axis, metric, label) | |
| suffix = "_vs_dataset" if args.x_axis == "dataset" else "" | |
| figure.savefig(args.output_dir / f"comparison_{metric}{suffix}.png", dpi=180) | |
| plt.close(figure) | |
| if args.x_axis == "dataset": | |
| figure, axes = plt.subplots(1, 2, figsize=(14, 5), layout="constrained") | |
| draw(axes[0], "validation_loss", "Validation cross entropy") | |
| draw(axes[1], "test_bleu", "Test BLEU (0-100)") | |
| figure.savefig(args.output_dir / "comparison_dataset_dynamics.png", dpi=180) | |
| plt.close(figure) | |
| if __name__ == "__main__": | |
| main() | |