dispatcher / scripts /plot_training.py
garg-tejas
feat: training tracker, HF Jobs launcher, inplace fix, visualization
27d2e57
Raw History Blame Contribute Delete
19.5 kB
#!/usr/bin/env python3
"""Plot DispatchR training curves from a saved metrics.json.
Reads the metrics.json written by train_unsloth_grpo.py and generates a
multi-panel figure covering reward curves, loss, action rates, fatalities,
epsilon decay, and curriculum phases.
Usage:
# Single run:
python scripts/plot_training.py --metrics outputs/unsloth_grpo/metrics.json
# Compare two runs (before vs after, or two different configs):
python scripts/plot_training.py \\
--metrics outputs/run_a/metrics.json \\
--compare outputs/run_b/metrics.json \\
--labels "Baseline" "With curriculum"
# Quick reward-only view:
python scripts/plot_training.py --metrics outputs/metrics.json --quick
"""
import argparse
import json
import sys
from pathlib import Path
from typing import Dict, List, Optional
import matplotlib
matplotlib.use("Agg")
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
# ── Style constants ────────────────────────────────────────────────────────────
BG_DARK = "#0F172A"
BG_PANEL = "#1E293B"
GRID_COLOR = "#334155"
TEXT_COLOR = "white"
C_REWARD = "#22C55E"
C_MA5 = "#F59E0B"
C_MA10 = "#3B82F6"
C_LOSS = "#EF4444"
C_DISPATCH = "#06B6D4"
C_HOLD = "#6B7280"
C_INVALID = "#DC2626"
C_VALID = "#10B981"
C_EPSILON = "#8B5CF6"
C_FATALITY = "#DC2626"
DIFF_COLORS = {
"warmup": "#10B981",
"learning": "#3B82F6",
"advanced": "#F59E0B",
"expert": "#EF4444",
}
COMPARE_COLORS = [
("#22C55E", "#3B82F6"), # pair 0: green / blue
("#F59E0B", "#EC4899"), # pair 1: amber / pink
]
# ── Helpers ────────────────────────────────────────────────────────────────────
def load_metrics(path: str) -> Dict:
with open(path) as fh:
return json.load(fh)
def _moving_avg(data: List[float], window: int) -> List[float]:
ma = []
for i in range(len(data)):
start = max(0, i - window + 1)
ma.append(float(np.mean(data[start:i + 1])))
return ma
def _ax_style(ax):
ax.set_facecolor(BG_PANEL)
ax.tick_params(colors=TEXT_COLOR, labelsize=8)
for spine in ax.spines.values():
spine.set_color(GRID_COLOR)
ax.xaxis.label.set_color(TEXT_COLOR)
ax.yaxis.label.set_color(TEXT_COLOR)
ax.title.set_color(TEXT_COLOR)
ax.grid(color=GRID_COLOR, linewidth=0.4, alpha=0.6)
def _shade_curriculum(ax, records: List[Dict]) -> None:
"""Shade background by curriculum difficulty phase."""
if not records:
return
difficulties = [r["difficulty"] for r in records]
batches = [r["batch"] + 1 for r in records]
start_b, cur_d = batches[0], difficulties[0]
for b, d in zip(batches[1:], difficulties[1:]):
if d != cur_d:
ax.axvspan(start_b - 0.5, b - 0.5,
alpha=0.08, color=DIFF_COLORS.get(cur_d, GRID_COLOR),
linewidth=0)
cur_d, start_b = d, b
ax.axvspan(start_b - 0.5, batches[-1] + 0.5,
alpha=0.08, color=DIFF_COLORS.get(cur_d, GRID_COLOR), linewidth=0)
# ── Single-run full plot ───────────────────────────────────────────────────────
def plot_full(metrics: Dict, output: str, run_label: str = "") -> None:
records = metrics.get("batch_records", [])
ep_rewards = metrics.get("rewards", [])
if not records:
print("ERROR: metrics.json has no batch_records β€” training may not have run.")
sys.exit(1)
batches = [r["batch"] + 1 for r in records]
fig = plt.figure(figsize=(18, 14), facecolor=BG_DARK)
gs = gridspec.GridSpec(3, 2, figure=fig, hspace=0.42, wspace=0.28)
axes = [[fig.add_subplot(gs[r, c]) for c in range(2)] for r in range(3)]
title = f"DispatchR GRPO β€” Training Progress"
if run_label:
title += f" [{run_label}]"
cfg = metrics.get("config", {})
subtitle = (
f"model: {cfg.get('model', '?').split('/')[-1]} Β· "
f"episodes: {cfg.get('episodes', '?')} Β· "
f"batch_size: {cfg.get('batch_size', '?')} Β· "
f"lr: {cfg.get('learning_rate', '?')}"
)
fig.suptitle(title, color=TEXT_COLOR, fontsize=15, fontweight="bold", y=0.98)
fig.text(0.5, 0.955, subtitle, ha="center", color="#94A3B8", fontsize=9)
# ── 1. Episode rewards (scatter + MA) ─────────────────────────────────────
ax = axes[0][0]
_ax_style(ax)
if ep_rewards:
ep_idx = list(range(len(ep_rewards)))
ax.scatter(ep_idx, ep_rewards, s=6, alpha=0.35, color=C_REWARD, label="Episode reward")
if len(ep_rewards) >= 10:
ax.plot(ep_idx, _moving_avg(ep_rewards, 10),
color=C_MA10, lw=1.8, label="MA-10")
if len(ep_rewards) >= 5:
ax.plot(ep_idx, _moving_avg(ep_rewards, 5),
color=C_MA5, lw=1.2, linestyle="--", label="MA-5")
ax.axhline(0, color=TEXT_COLOR, lw=0.5, alpha=0.3)
ax.set_xlabel("Episode")
ax.set_ylabel("Reward")
ax.set_title("Episode Rewards")
ax.legend(fontsize=7, facecolor=BG_PANEL, edgecolor=GRID_COLOR,
labelcolor=TEXT_COLOR)
# ── 2. Batch mean reward + loss (dual axis) ────────────────────────────────
ax = axes[0][1]
_ax_style(ax)
_shade_curriculum(ax, records)
mean_r = [r["mean_reward"] for r in records]
ax.plot(batches, mean_r, color=C_REWARD, lw=1.8, label="Mean reward")
if len(records) >= 5:
ax.plot(batches, [r["reward_ma_5"] for r in records],
color=C_MA5, lw=1.2, ls="--", label="MA-5")
ax.fill_between(batches, [r["min_reward"] for r in records],
[r["max_reward"] for r in records],
color=C_REWARD, alpha=0.08, label="min–max band")
ax.axhline(0, color=TEXT_COLOR, lw=0.5, alpha=0.3)
ax.set_ylabel("Reward", color=C_REWARD)
ax.set_xlabel("Batch")
ax.set_title("Batch Reward + Loss")
ax2 = ax.twinx()
ax2.tick_params(colors=TEXT_COLOR, labelsize=8)
ax2.yaxis.label.set_color(TEXT_COLOR)
loss_pairs = [(batches[i], r["loss"]) for i, r in enumerate(records) if r["loss"] > 0]
if loss_pairs:
lx, ly = zip(*loss_pairs)
ax2.plot(lx, ly, color=C_LOSS, lw=1.5, marker="o",
markersize=2.5, label="Loss", alpha=0.85)
ax2.set_ylabel("Loss", color=C_LOSS)
ax2.legend(loc="upper right", fontsize=7,
facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
ax.legend(loc="upper left", fontsize=7,
facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
# ── 3. Valid action rate ───────────────────────────────────────────────────
ax = axes[1][0]
_ax_style(ax)
_shade_curriculum(ax, records)
vr = [r["valid_action_rate"] for r in records]
ax.plot(batches, vr, color=C_VALID, lw=1.8)
ax.fill_between(batches, vr, alpha=0.15, color=C_VALID)
ax.set_ylim(0, 1.05)
ax.set_xlabel("Batch")
ax.set_ylabel("Valid Action Rate")
ax.set_title("Valid Action Rate")
# ── 4. Action type breakdown ───────────────────────────────────────────────
ax = axes[1][1]
_ax_style(ax)
_shade_curriculum(ax, records)
ax.plot(batches, [r["dispatch_rate"] for r in records],
color=C_DISPATCH, lw=1.5, label="Dispatch")
ax.plot(batches, [r["hold_rate"] for r in records],
color=C_HOLD, lw=1.5, label="Hold")
ax.plot(batches, [r["invalid_action_rate"] for r in records],
color=C_INVALID, lw=1.5, label="Invalid")
ax.set_ylim(0, 1.05)
ax.set_xlabel("Batch")
ax.set_ylabel("Rate")
ax.set_title("Action Type Distribution")
ax.legend(fontsize=7, facecolor=BG_PANEL, edgecolor=GRID_COLOR,
labelcolor=TEXT_COLOR)
# ── 5. Fatalities per batch ────────────────────────────────────────────────
ax = axes[2][0]
_ax_style(ax)
_shade_curriculum(ax, records)
fatalities = [r["fatality_count"] for r in records]
ax.bar(batches, fatalities, color=C_FATALITY, alpha=0.7, width=0.8)
if len(records) >= 5:
ax.plot(batches, _moving_avg(fatalities, 5),
color="#FBBF24", lw=1.5, label="MA-5")
ax.legend(fontsize=7, facecolor=BG_PANEL, edgecolor=GRID_COLOR,
labelcolor=TEXT_COLOR)
ax.set_xlabel("Batch")
ax.set_ylabel("Fatalities")
ax.set_title("Fatalities per Batch")
# ── 6. Epsilon decay + curriculum phases ──────────────────────────────────
ax = axes[2][1]
_ax_style(ax)
epsilons = [r["epsilon"] for r in records]
ax.plot(batches, epsilons, color=C_EPSILON, lw=1.8, label="Epsilon")
ax.set_ylim(-0.05, 1.1)
ax.set_xlabel("Batch")
ax.set_ylabel("Epsilon", color=C_EPSILON)
ax.set_title("Exploration & Curriculum Phases")
# Curriculum legend patches
seen_diffs = dict.fromkeys(r["difficulty"] for r in records)
import matplotlib.patches as mpatches
patches = [
mpatches.Patch(color=DIFF_COLORS.get(d, GRID_COLOR), alpha=0.5, label=d.title())
for d in seen_diffs
]
if patches:
leg = ax.legend(handles=patches, loc="upper right", fontsize=7,
facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR,
title="Phase", title_fontsize=7)
ax.add_artist(leg)
ax.legend(handles=[
plt.Line2D([0], [0], color=C_EPSILON, lw=1.5, label="Epsilon")
], loc="upper left", fontsize=7,
facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
_shade_curriculum(ax, records)
Path(output).parent.mkdir(parents=True, exist_ok=True)
plt.savefig(output, dpi=150, facecolor=BG_DARK, bbox_inches="tight")
plt.close(fig)
print(f"Saved β†’ {output}")
# ── Comparison plot ────────────────────────────────────────────────────────────
def plot_compare(
metrics_a: Dict,
metrics_b: Dict,
output: str,
label_a: str = "Run A",
label_b: str = "Run B",
) -> None:
"""Side-by-side comparison of two runs: reward curves, loss, action rates."""
rec_a = metrics_a.get("batch_records", [])
rec_b = metrics_b.get("batch_records", [])
rew_a = metrics_a.get("rewards", [])
rew_b = metrics_b.get("rewards", [])
fig, axes = plt.subplots(2, 2, figsize=(16, 10), facecolor=BG_DARK)
fig.suptitle(
f"Training Comparison Β· {label_a} vs {label_b}",
color=TEXT_COLOR, fontsize=14, fontweight="bold",
)
ca, cb = "#22C55E", "#3B82F6" # green / blue
# ── 1. Episode reward curves ──────────────────────────────────────────────
ax = axes[0][0]
_ax_style(ax)
for rew, col, lbl in [(rew_a, ca, label_a), (rew_b, cb, label_b)]:
if rew:
idx = list(range(len(rew)))
ax.scatter(idx, rew, s=5, alpha=0.25, color=col)
if len(rew) >= 10:
ax.plot(idx, _moving_avg(rew, 10), color=col, lw=2.0, label=f"{lbl} MA-10")
ax.axhline(0, color=TEXT_COLOR, lw=0.4, alpha=0.3)
ax.set_xlabel("Episode")
ax.set_ylabel("Reward")
ax.set_title("Episode Rewards")
ax.legend(fontsize=8, facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
# ── 2. Batch mean reward ──────────────────────────────────────────────────
ax = axes[0][1]
_ax_style(ax)
for rec, col, lbl in [(rec_a, ca, label_a), (rec_b, cb, label_b)]:
if rec:
bs = [r["batch"] + 1 for r in rec]
ax.plot(bs, [r["mean_reward"] for r in rec], color=col, lw=1.8, label=lbl)
ax.fill_between(
bs,
[r["min_reward"] for r in rec],
[r["max_reward"] for r in rec],
color=col, alpha=0.07,
)
ax.axhline(0, color=TEXT_COLOR, lw=0.4, alpha=0.3)
ax.set_xlabel("Batch")
ax.set_ylabel("Mean Reward")
ax.set_title("Batch Mean Reward")
ax.legend(fontsize=8, facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
# ── 3. Loss ───────────────────────────────────────────────────────────────
ax = axes[1][0]
_ax_style(ax)
for rec, col, lbl in [(rec_a, ca, label_a), (rec_b, cb, label_b)]:
if rec:
pairs = [(r["batch"] + 1, r["loss"]) for r in rec if r["loss"] > 0]
if pairs:
lx, ly = zip(*pairs)
ax.plot(lx, ly, color=col, lw=1.5, marker="o",
markersize=2, alpha=0.9, label=lbl)
ax.set_xlabel("Batch")
ax.set_ylabel("GRPO Loss")
ax.set_title("Policy Loss")
ax.legend(fontsize=8, facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
# ── 4. Valid action rate + fatalities ────────────────────────────────────
ax = axes[1][1]
_ax_style(ax)
ax2 = ax.twinx()
ax2.tick_params(colors=TEXT_COLOR, labelsize=8)
for rec, col, lbl in [(rec_a, ca, label_a), (rec_b, cb, label_b)]:
if rec:
bs = [r["batch"] + 1 for r in rec]
ax.plot(bs, [r["valid_action_rate"] for r in rec],
color=col, lw=1.8, label=f"{lbl} valid")
ax2.plot(bs, _moving_avg([r["fatality_count"] for r in rec], 5),
color=col, lw=1.2, ls="--", alpha=0.7)
ax.set_ylim(0, 1.05)
ax.set_xlabel("Batch")
ax.set_ylabel("Valid Action Rate")
ax2.set_ylabel("Fatalities (MA-5)", color="#94A3B8")
ax.set_title("Valid Actions & Fatalities (dashed)")
ax.legend(fontsize=8, facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
plt.tight_layout(pad=2.0)
Path(output).parent.mkdir(parents=True, exist_ok=True)
plt.savefig(output, dpi=150, facecolor=BG_DARK, bbox_inches="tight")
plt.close(fig)
print(f"Saved β†’ {output}")
# ── Quick 2-panel view ─────────────────────────────────────────────────────────
def plot_quick(metrics: Dict, output: str, run_label: str = "") -> None:
"""Minimal reward + loss for a fast sanity check."""
records = metrics.get("batch_records", [])
if not records:
print("ERROR: no batch_records in metrics.json")
sys.exit(1)
batches = [r["batch"] + 1 for r in records]
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 4), facecolor=BG_DARK)
for ax in [ax1, ax2]:
_ax_style(ax)
# Reward
ax1.plot(batches, [r["mean_reward"] for r in records],
color=C_REWARD, lw=2, label="Mean reward")
ax1.fill_between(
batches,
[r["min_reward"] for r in records],
[r["max_reward"] for r in records],
color=C_REWARD, alpha=0.12, label="min–max"
)
if len(records) >= 5:
ax1.plot(batches, [r["reward_ma_5"] for r in records],
color=C_MA5, lw=1.4, ls="--", label="MA-5")
ax1.axhline(0, color=TEXT_COLOR, lw=0.5, alpha=0.3)
ax1.set_xlabel("Batch")
ax1.set_ylabel("Reward")
ax1.set_title(f"Reward Curve{' β€” ' + run_label if run_label else ''}")
ax1.legend(fontsize=8, facecolor=BG_PANEL, edgecolor=GRID_COLOR, labelcolor=TEXT_COLOR)
# Loss
loss_pairs = [(batches[i], r["loss"]) for i, r in enumerate(records) if r["loss"] > 0]
if loss_pairs:
lx, ly = zip(*loss_pairs)
ax2.plot(lx, ly, color=C_LOSS, lw=1.8, marker="o", markersize=3)
ax2.set_xlabel("Batch")
ax2.set_ylabel("GRPO Loss")
ax2.set_title("Loss Curve")
plt.tight_layout()
Path(output).parent.mkdir(parents=True, exist_ok=True)
plt.savefig(output, dpi=150, facecolor=BG_DARK, bbox_inches="tight")
plt.close(fig)
print(f"Saved β†’ {output}")
# ── Entry point ────────────────────────────────────────────────────────────────
def main() -> None:
parser = argparse.ArgumentParser(
description="Plot DispatchR training curves from metrics.json",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
parser.add_argument("--metrics", required=True,
help="Path to metrics.json (primary run)")
parser.add_argument("--compare", default=None,
help="Path to a second metrics.json to compare against")
parser.add_argument("--labels", nargs=2, default=["Run A", "Run B"],
metavar=("LABEL_A", "LABEL_B"),
help="Labels for the two runs when using --compare")
parser.add_argument("--output", default=None,
help="Output PNG path (default: same dir as --metrics)")
parser.add_argument("--quick", action="store_true",
help="2-panel quick view instead of full 6-panel plot")
args = parser.parse_args()
m = load_metrics(args.metrics)
# Summary to stdout
summary = m.get("summary", {})
cfg = m.get("config", {})
print(f"\nRun summary")
print(f" model: {cfg.get('model', '?').split('/')[-1]}")
print(f" episodes: {m.get('episodes_done', len(m.get('rewards', [])))}")
print(f" batches: {m.get('batches_done', len(m.get('batch_records', [])))}")
if summary:
print(f" mean_reward: {summary.get('mean_reward', 0):.4f}")
print(f" best_reward: {summary.get('best_reward', 0):.4f}")
print(f" final_loss: {summary.get('final_loss', 0):.4f}")
metrics_dir = Path(args.metrics).parent
if args.compare:
m2 = load_metrics(args.compare)
out = args.output or str(metrics_dir / "comparison.png")
plot_compare(m, m2, out, label_a=args.labels[0], label_b=args.labels[1])
elif args.quick:
out = args.output or str(metrics_dir / "training_curves_quick.png")
label = Path(args.metrics).parent.name
plot_quick(m, out, run_label=label)
else:
out = args.output or str(metrics_dir / "training_progress.png")
label = Path(args.metrics).parent.name
plot_full(m, out, run_label=label)
if __name__ == "__main__":
main()