Spaces:
Sleeping
Sleeping
File size: 4,062 Bytes
c613bf9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | import argparse
import json
from pathlib import Path
def main():
parser = argparse.ArgumentParser(description="Plot OversightForge metrics from rollout artifact")
parser.add_argument("--input", default="artifacts/training_rollout.json")
parser.add_argument("--eval-input", default="artifacts/level4_eval.json")
parser.add_argument("--output-dir", default="artifacts/plots")
args = parser.parse_args()
try:
import matplotlib.pyplot as plt
except Exception as exc:
raise RuntimeError(
"matplotlib is required for plotting. Install with: pip install matplotlib"
) from exc
data = json.loads(Path(args.input).read_text(encoding="utf-8"))
results = data.get("results", [])
rewards = [float(ep.get("reward", 0.0)) for ep in results]
discovered = [1 if (ep.get("reward_breakdown") or {}).get("fraud_discovered") else 0 for ep in results]
code_ok = [1 if (ep.get("reward_breakdown") or {}).get("correct_violation_code") else 0 for ep in results]
syco_ok = [1 if (ep.get("reward_breakdown") or {}).get("sycophancy_resisted") else 0 for ep in results]
calls = [int(ep.get("steps_taken", 0)) for ep in results]
def rolling_pct(values, window=10):
out = []
for idx in range(len(values)):
start = max(0, idx - window + 1)
section = values[start : idx + 1]
out.append((sum(section) / max(1, len(section))) * 100.0)
return out
out_dir = Path(args.output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
# 1. reward curve
plt.figure(figsize=(9, 4))
plt.plot(rewards, color="#1f6f78")
plt.title("Reward Curve")
plt.xlabel("Episode")
plt.ylabel("Reward")
plt.tight_layout()
plt.savefig(out_dir / "reward_curve.png", dpi=150)
plt.close()
# 2. fraud discovery rate (rolling)
plt.figure(figsize=(9, 4))
plt.plot(rolling_pct(discovered), color="#2a7f4f")
plt.title("Fraud Discovery Rate (Rolling 10)")
plt.xlabel("Episode")
plt.ylabel("%")
plt.ylim(0, 100)
plt.tight_layout()
plt.savefig(out_dir / "fraud_discovery_rate.png", dpi=150)
plt.close()
# 3. AML code accuracy (rolling)
plt.figure(figsize=(9, 4))
plt.plot(rolling_pct(code_ok), color="#c85c2d")
plt.title("AML Code Accuracy (Rolling 10)")
plt.xlabel("Episode")
plt.ylabel("%")
plt.ylim(0, 100)
plt.tight_layout()
plt.savefig(out_dir / "aml_code_accuracy.png", dpi=150)
plt.close()
# 4. sycophancy resistance (rolling)
plt.figure(figsize=(9, 4))
plt.plot(rolling_pct(syco_ok), color="#5a3e36")
plt.title("Sycophancy Resistance (Rolling 10)")
plt.xlabel("Episode")
plt.ylabel("%")
plt.ylim(0, 100)
plt.tight_layout()
plt.savefig(out_dir / "sycophancy_resistance.png", dpi=150)
plt.close()
# 5. detection calls
plt.figure(figsize=(9, 4))
plt.plot(calls, color="#444")
plt.title("Detection Calls Per Episode")
plt.xlabel("Episode")
plt.ylabel("Calls")
plt.tight_layout()
plt.savefig(out_dir / "detection_calls.png", dpi=150)
plt.close()
eval_path = Path(args.eval_input)
if eval_path.exists():
eval_data = json.loads(eval_path.read_text(encoding="utf-8"))
eval_results = eval_data.get("results", [])
eval_discovery = [1 if (ep.get("reward_breakdown") or {}).get("fraud_discovered") else 0 for ep in eval_results]
train_rate = (sum(discovered) / max(1, len(discovered))) * 100.0
eval_rate = (sum(eval_discovery) / max(1, len(eval_discovery))) * 100.0
plt.figure(figsize=(7, 4))
plt.bar(["Train", "Level4 Held-Out"], [train_rate, eval_rate], color=["#1f6f78", "#c85c2d"])
plt.title("Generalization: Train vs Held-Out Discovery Rate")
plt.ylabel("%")
plt.ylim(0, 100)
plt.tight_layout()
plt.savefig(out_dir / "generalization_train_vs_heldout.png", dpi=150)
plt.close()
print(f"Saved plots to: {out_dir}")
if __name__ == "__main__":
main()
|