Spaces:
Sleeping
Sleeping
File size: 4,003 Bytes
2aed081 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import numpy as np
import re
import os
import json
REFERENCE_BASELINES = {
"Random": {"f1": 48.5, "step_acc": 8.7, "color": "gray", "ls": ":"},
"Open-src avg": {"f1": 45.8, "step_acc": 10.9, "color": "orange", "ls": "-."},
"Gemini-2.5-Pro": {"f1": 64.6, "step_acc": 41.1, "color": "purple", "ls": "--"},
}
def parse_log(log_path):
"""Parse training log into lists of (epoch, train_loss, val_f1, val_step_acc)."""
with open(log_path) as f:
content = f.read()
epochs = []
train_losses = []
val_f1s = []
val_accs = []
blocks = re.split(r'── Epoch (\d+)/\d+ ──+', content)
# blocks[0] = preamble, then alternating: epoch_num, block_content
for i in range(1, len(blocks) - 1, 2):
ep = int(blocks[i])
blk = blocks[i + 1]
loss_m = re.search(r'Train loss\s*:\s*([0-9.]+)', blk)
f1_m = re.search(r'F1\s*:\s*([0-9.]+)%', blk)
acc_m = re.search(r'step-acc\s*:\s*([0-9.]+)%', blk, re.IGNORECASE)
if loss_m and f1_m and acc_m:
epochs.append(ep)
train_losses.append(float(loss_m.group(1)))
val_f1s.append(float(f1_m.group(1)))
val_accs.append(float(acc_m.group(1)))
return epochs, train_losses, val_f1s, val_accs
def plot_learning_curves(log_path, output_path, meta_path=None):
if not os.path.exists(log_path):
print(f"Log file not found: {log_path}")
return
epochs, train_losses, val_f1s, val_accs = parse_log(log_path)
if not epochs:
print("Could not parse any epoch data from the log.")
return
best_epoch = None
if meta_path and os.path.exists(meta_path):
with open(meta_path) as f:
meta = json.load(f)
best_epoch = meta.get("epoch")
plt.style.use("seaborn-v0_8-whitegrid")
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(11, 8), sharex=True)
fig.suptitle("AgentSight — Training Curves", fontsize=14, fontweight="bold")
# ── Top panel: training loss ──────────────────────────────────────────────
ax1.plot(epochs, train_losses, color="tab:red", marker="o", ms=3,
label="Train loss (BCE)")
ax1.set_ylabel("Train Loss", fontweight="bold")
ax1.yaxis.set_major_formatter(mticker.FormatStrFormatter("%.3f"))
if best_epoch:
ax1.axvline(best_epoch, color="green", lw=1.5, ls="--", label=f"Best epoch ({best_epoch})")
ax1.legend(fontsize=9)
# ── Bottom panel: val metrics + baselines ─────────────────────────────────
ax2.plot(epochs, val_accs, color="tab:blue", marker="s", ms=3,
label="Val Step-Acc (%)")
ax2.plot(epochs, val_f1s, color="tab:green", marker="^", ms=3, ls="--",
label="Val Macro-F1 (%)")
for name, ref in REFERENCE_BASELINES.items():
ax2.axhline(ref["step_acc"], color=ref["color"], ls=ref["ls"], lw=1.5,
label=f"{name} Step-Acc ({ref['step_acc']}%)")
if best_epoch:
ax2.axvline(best_epoch, color="green", lw=1.5, ls="--")
ax2.set_xlabel("Epoch", fontweight="bold")
ax2.set_ylabel("Metric (%)", fontweight="bold")
ax2.set_ylim(0, 100)
ax2.legend(fontsize=8, loc="lower right")
plt.tight_layout()
os.makedirs(os.path.dirname(output_path), exist_ok=True)
plt.savefig(output_path, dpi=300)
print(f"Plot saved → {output_path}")
if __name__ == "__main__":
root = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")
log_file = os.path.join(root, "full_training_log.txt")
meta = os.path.join(root, "src", "models", "best_agentsight_meta.json")
out_png = os.path.join(root, "notebooks", "training_curve.png")
plot_learning_curves(log_file, out_png, meta_path=meta)
|