Upload scripts/make_peer_compare.py with huggingface_hub

scripts/make_peer_compare.py

@@ -34,6 +34,12 @@ def parse(path):
 def med(x): 
     x = sorted(x); return x[len(x)//2] if x else 0
 
+def ce_at(d, lo, hi):
+    """Mean valid CE over a token window - the only fair quality comparison when the
+    two runs stopped at different token counts."""
+    v = [c for t, c in zip(d['vtokens'], d['vce']) if lo <= t <= hi]
+    return sum(v)/len(v) if v else None
+
 def main():
     a = parse(D / 'baseline_peer_pipeline.log')   # v20
     b = parse(D / 'v21_peer21.log')               # v21
@@ -53,6 +59,13 @@ def main():
                  experts_per_layer=495616, peer_layers=3, pools='1 shared',
                  ctrl_params=598145536, h_times_k=8),
         speedup=B/A if A else 0)
+    # same-token quality: compare both runs over the window they BOTH covered
+    hi = min(max(a['vtokens'] or [0]), max(b['vtokens'] or [0]))
+    lo = 0.5 * hi
+    if hi > 0:
+        M['same_token_window'] = {'lo_tokens': lo, 'hi_tokens': hi,
+                                  'v20_mean_valid_ce': ce_at(a, lo, hi),
+                                  'v21_mean_valid_ce': ce_at(b, lo, hi)}
     (D / 'compare_metrics.json').write_text(json.dumps(M, indent=2))
     print(json.dumps(M, indent=2))