Upload verified HELM artifacts [f33adc54b7854c37aac97272220550d6]

CHECKSUMS.json

@@ -1,14 +1,14 @@
 {
   "files": [
     {
-      "bytes": 5329,
+      "bytes": 7017,
       "path": "README.md",
-      "sha256": "32b1167c757ab886fce422d0941f7424d4abb402764e8b4eaedb30730e59d433"
+      "sha256": "cd999d957215a4b63b327a3c51ead89005cce40c7ba809d6719e9a07644debe1"
     },
     {
-      "bytes": 803,
+      "bytes": 1116,
       "path": "RESULTS.json",
-      "sha256": "3010cb9e9e942eb98e39cbb0e07a7e0182dc4a0fc5f84d20f56963793245388a"
+      "sha256": "1b9a45d43111da04ed964618d90fe8a6843956e84a3ba36e6b4e2e2a6b7e5b28"
     },
     {
       "bytes": 18426,
@@ -20,6 +20,11 @@
       "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_corpc420b_perm_cos_mb8_w8.json",
       "sha256": "55c353bad3bfe8447b68d2747242c6d8ee31c4f31989b7823c9907405f54cb57"
     },
+    {
+      "bytes": 18620,
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8.json",
+      "sha256": "755b813b7ee64fe16f6e35dea6e43e7fce7099032a4cb894e1643ccf24531a70"
+    },
     {
       "bytes": 4871,
       "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc1_s7_st152580_ev5000_corpc420b_perm_cos_mb8_w8.json",
@@ -40,6 +45,16 @@
       "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_corpc420b_perm_cos_mb8_w8_last_finite_saved_step2000.pt",
       "sha256": "690718ffba3f1a29ab829e562a14e036b580cdde098cf0709aba36152c87fdb5"
     },
+    {
+      "bytes": 3886302822,
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8.pt",
+      "sha256": "4f5e534dde21f5f23837c99f4e8f74d50bff4b010d9699ef3d653eb305ee8ed0"
+    },
+    {
+      "bytes": 1449,
+      "path": "compact_logs/policy.log",
+      "sha256": "1faa4a9ed0eb9addd94892d10012c21ecf36d5c7c5ecbdb910fc5584fc4bebfe"
+    },
     {
       "bytes": 29581,
       "path": "compact_logs/scaled.log",
@@ -61,9 +76,14 @@
       "sha256": "82fa6f81bbb83a76001690984c6ad3ff79cbe05ef468fcbb07de8f2576e7b412"
     },
     {
-      "bytes": 2010,
+      "bytes": 3751,
       "path": "eval_curves/all_ready_runs.csv",
-      "sha256": "4f00c51138b07d3d9690d47d3a97b4f952337e564d58bc61a7c91bc6071401ff"
+      "sha256": "61dd379291199632a17bdc0f176f857bb4bc0bc6e2034065961e4764060bb719"
+    },
+    {
+      "bytes": 1421,
+      "path": "eval_curves/policy.csv",
+      "sha256": "7ea8a4e32f8ef10de961dc6dc0b81034c3a63c6e9ffbdc3f9c8fac4138ac2387"
     },
     {
       "bytes": 1420,
@@ -90,6 +110,11 @@
       "path": "eval_curves/unit99.csv",
       "sha256": "306c59b6d98c9252f37717701f8b378da28941b84a176f2dd457c543b6385488"
     },
+    {
+      "bytes": 707,
+      "path": "geometry_policy.json",
+      "sha256": "273339407f0368717a5e28948d1c49b49e2f1692eb713bd5e5db8dab9f788b17"
+    },
     {
       "bytes": 204,
       "path": "health/original_unit_before_step2280.csv",

README.md

@@ -22,6 +22,7 @@ quantized in these runs.
 | Run | Observed outcome | C4 PPL | WikiText PPL | Available state |
 | --- | --- | ---: | ---: | --- |
 | Scaled init, seed 1234 | Finished 152,580 steps | 19.198618161946133 | 52.69278098379149 | Native final model |
+| Published init + FP32 geometry, seed 1234 | Finished 152,580 steps | 20.370028561152527 | 50.02364864030602 | Native final `_pol` model |
 | Published init, seed 1234 | Nonfinite first flagged at 2,280; detected at 2,300 | unavailable | unavailable | Last saved finite model at step 2,000 |
 | Published init, seed 7 | Finite through requested 10,000-step cap | 36.28621540996052 | 84.48696841726898 | Native step-10,000 training resume |
 | Published init, seed 99 | Finite through requested 10,000-step cap | 37.023797483960436 | 88.28929618013538 | Native step-10,000 training resume |
@@ -32,10 +33,19 @@ The original seed-1234 failure happened before its first scheduled evaluation.
 Its JSON intentionally retains `NaN` scores and an empty evaluation curve.
 The step-2,000 model is not a completed 20B checkpoint.
 
-The additional published-init run with `cinner` and
-`lorentzian_centroid` protected in FP32 (`_pol`) is pending and will be added
-after completion. Its geometry precision profile must be selected explicitly
-by downstream code; checkpoint tensors do not serialize method wrappers.
+The published-init `_pol` run also completed the full budget. It protects
+`cinner` and `lorentzian_centroid` in FP32 while the rest of the BF16 recipe,
+initialization, schedule and budget remain unchanged. See
+`geometry_policy.json` and the runtime selection below. Checkpoint tensors
+do not serialize the method wrappers.
+
+The original full-horizon attempt's final verifier crashed before final
+artifact upload. These final weights come from a verified continuation of
+the saved step-152,000 state: the last 580 updates were repeated with the
+same model, policy, seed, optimizer/RNG, data and full learning-rate schedule.
+This is a new continuation result; it is not claimed to recover identical
+original node-local final bytes.
+
 The forthcoming quantized 2B experiments are not present in this repository.
 
 ## Recipe and provenance
@@ -98,6 +108,37 @@ has not been qualified here. Learned normalizer weights are already
 included; do not rescale them after loading. This is the native research
 checkpoint format; the example above verifies and reads its payload.
 
+
+### Selecting the `_pol` geometry precision
+
+For the `_pol` model, select the same two protected methods in downstream
+code. The following is the method wrapper used by the pinned trainer; it
+does not construct a model or change the learned weights:
+
+```python
+import torch
+from helm.hypercore.manifolds import Lorentz
+
+def protect_fp32(cls, name):
+    original = getattr(cls, name)
+    def wrapped(self, *args, **kwargs):
+        with torch.autocast(device_type="cuda", enabled=False):
+            args = tuple(x.float() if torch.is_tensor(x) else x for x in args)
+            kwargs = {k: (x.float() if torch.is_tensor(x) else x)
+                      for k, x in kwargs.items()}
+            return original(self, *args, **kwargs)
+    setattr(cls, name, wrapped)
+
+protect_fp32(Lorentz, "cinner")
+protect_fp32(Lorentz, "lorentzian_centroid")
+```
+
+Apply this policy explicitly before using the `_pol` model. The remaining
+training operations used BF16 autocast, with FP32 parameters. Follow the
+pinned recipe for model/distributed initialization; this snippet is not a
+standalone model loader.
+
+
 Files under `resumes/` have the original training-state payload with `model`,
 `opt`, sampler/generator state, CPU/CUDA RNG state and evaluation history.
 They are for restoration through the original trainer, not the two-key model

RESULTS.json

@@ -1,10 +1,24 @@
 {
-  "pending_run": "unit_fp32_geometry_policy",
+  "pending_run": null,
+  "policy": {
+    "c4_ppl": 20.370028561152527,
+    "policy": true,
+    "recovery": {
+      "continuation_updates": 580,
+      "original_final_bytes_recovered": false,
+      "restored_step": 152000,
+      "same_recipe": true
+    },
+    "status": "finished",
+    "steps": 152580,
+    "wikitext_ppl": 50.02364864030602
+  },
   "ready_runs": [
     "scaled",
     "unit1234",
     "unit7",
-    "unit99"
+    "unit99",
+    "policy"
   ],
   "scaled": {
     "c4_ppl": 19.198618161946133,

cells.d/unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8.json

@@ -0,0 +1,1698 @@
+{
+ "unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8": {
+  "init": "unit",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c420b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 1,
+  "seed": 1234,
+  "steps": 152580,
+  "eval": 5000,
+  "tokens": 19998965760,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 8,
+  "compile": false,
+  "world": 8,
+  "R_init": 30.1513,
+  "u_init": 30.1513,
+  "radii": {
+   "layers.0.attn_norm": 30.15,
+   "layers.0.attn.kv_norm": 16.032,
+   "layers.0.ffn_norm": 30.15,
+   "layers.1.attn_norm": 30.15,
+   "layers.1.attn.kv_norm": 16.031,
+   "layers.1.ffn_norm": 30.15,
+   "layers.2.attn_norm": 30.15,
+   "layers.2.attn.kv_norm": 16.031,
+   "layers.2.ffn_norm": 30.15,
+   "layers.3.attn_norm": 30.15,
+   "layers.3.attn.kv_norm": 16.031,
+   "layers.3.ffn_norm": 30.15,
+   "layers.4.attn_norm": 30.15,
+   "layers.4.attn.kv_norm": 16.031,
+   "layers.4.ffn_norm": 30.15,
+   "layers.5.attn_norm": 30.15,
+   "layers.5.attn.kv_norm": 16.03,
+   "layers.5.ffn_norm": 30.15,
+   "layers.6.attn_norm": 30.15,
+   "layers.6.attn.kv_norm": 16.032,
+   "layers.6.ffn_norm": 30.15,
+   "layers.7.attn_norm": 30.15,
+   "layers.7.attn.kv_norm": 16.031,
+   "layers.7.ffn_norm": 30.15,
+   "layers.8.attn_norm": 30.15,
+   "layers.8.attn.kv_norm": 16.034,
+   "layers.8.ffn_norm": 30.15,
+   "layers.9.attn_norm": 30.15,
+   "layers.9.attn.kv_norm": 16.028,
+   "layers.9.ffn_norm": 30.15,
+   "layers.10.attn_norm": 30.15,
+   "layers.10.attn.kv_norm": 16.03,
+   "layers.10.ffn_norm": 30.15,
+   "layers.11.attn_norm": 30.15,
+   "layers.11.attn.kv_norm": 16.033,
+   "layers.11.ffn_norm": 30.15,
+   "layers.12.attn_norm": 30.15,
+   "layers.12.attn.kv_norm": 16.029,
+   "layers.12.ffn_norm": 30.15,
+   "layers.13.attn_norm": 30.15,
+   "layers.13.attn.kv_norm": 16.033,
+   "layers.13.ffn_norm": 30.15,
+   "layers.14.attn_norm": 30.15,
+   "layers.14.attn.kv_norm": 16.032,
+   "layers.14.ffn_norm": 30.15,
+   "layers.15.attn_norm": 30.15,
+   "layers.15.attn.kv_norm": 16.031,
+   "layers.15.ffn_norm": 30.15,
+   "norm": 30.151
+  },
+  "eval_touches_state": false,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 3367.4,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": true,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [
+   [
+    5000,
+    49.96224627206456
+   ],
+   [
+    10000,
+    38.34757071766789
+   ],
+   [
+    15000,
+    34.19195513906091
+   ],
+   [
+    20000,
+    32.168524269995935
+   ],
+   [
+    25000,
+    30.849758512410368
+   ],
+   [
+    30000,
+    29.979062376304086
+   ],
+   [
+    35000,
+    29.11726802688634
+   ],
+   [
+    40000,
+    28.58433864688657
+   ],
+   [
+    45000,
+    28.00962249511226
+   ],
+   [
+    50000,
+    27.579803871894065
+   ],
+   [
+    55000,
+    27.180486538834845
+   ],
+   [
+    60000,
+    26.82660577049195
+   ],
+   [
+    65000,
+    26.256727775702622
+   ],
+   [
+    70000,
+    25.831258275309533
+   ],
+   [
+    75000,
+    25.428781915034556
+   ],
+   [
+    80000,
+    25.041509583243812
+   ],
+   [
+    85000,
+    24.611292915690886
+   ],
+   [
+    90000,
+    24.228492172949245
+   ],
+   [
+    95000,
+    23.80994974806001
+   ],
+   [
+    100000,
+    23.38686133823178
+   ],
+   [
+    105000,
+    22.954160491577337
+   ],
+   [
+    110000,
+    22.50397573777943
+   ],
+   [
+    115000,
+    22.17635856096328
+   ],
+   [
+    120000,
+    21.843981902662684
+   ],
+   [
+    125000,
+    21.427204836865872
+   ],
+   [
+    130000,
+    21.156095309325533
+   ],
+   [
+    135000,
+    20.890162268957365
+   ],
+   [
+    140000,
+    20.667042690067046
+   ],
+   [
+    145000,
+    20.497996371069945
+   ],
+   [
+    150000,
+    20.394912540783352
+   ],
+   [
+    152580,
+    20.370028561152527
+   ]
+  ],
+  "ood_traj": [
+   [
+    5000,
+    124.93939687264222
+   ],
+   [
+    10000,
+    88.14201261882307
+   ],
+   [
+    15000,
+    77.78597054694117
+   ],
+   [
+    20000,
+    73.07426804069382
+   ],
+   [
+    25000,
+    70.93661797748476
+   ],
+   [
+    30000,
+    66.15964946320693
+   ],
+   [
+    35000,
+    66.73714176781131
+   ],
+   [
+    40000,
+    65.63382597489617
+   ],
+   [
+    45000,
+    63.010533551917774
+   ],
+   [
+    50000,
+    64.17758334317438
+   ],
+   [
+    55000,
+    62.46369541840954
+   ],
+   [
+    60000,
+    63.131650199754795
+   ],
+   [
+    65000,
+    60.2085794511229
+   ],
+   [
+    70000,
+    59.216968578191036
+   ],
+   [
+    75000,
+    58.26023149826963
+   ],
+   [
+    80000,
+    56.6621200346212
+   ],
+   [
+    85000,
+    57.07487729825752
+   ],
+   [
+    90000,
+    56.606062648114815
+   ],
+   [
+    95000,
+    56.30148377354903
+   ],
+   [
+    100000,
+    53.300426090221876
+   ],
+   [
+    105000,
+    53.61922907480571
+   ],
+   [
+    110000,
+    53.02736039799391
+   ],
+   [
+    115000,
+    52.0023925595677
+   ],
+   [
+    120000,
+    51.849058306007166
+   ],
+   [
+    125000,
+    51.25451432552149
+   ],
+   [
+    130000,
+    50.076957473304404
+   ],
+   [
+    135000,
+    49.28759955513546
+   ],
+   [
+    140000,
+    49.14775593251116
+   ],
+   [
+    145000,
+    49.62698778818433
+   ],
+   [
+    150000,
+    49.61545958685712
+   ],
+   [
+    152580,
+    50.02364864030602
+   ]
+  ],
+  "health": [
+   [
+    1,
+    1000,
+    6.75,
+    1.5516,
+    3.7819,
+    0.848
+   ],
+   [
+    1001,
+    2000,
+    4.8919,
+    0.9703,
+    1.7478,
+    0.391
+   ],
+   [
+    2001,
+    3000,
+    4.3409,
+    0.6633,
+    1.1186,
+    0.012
+   ],
+   [
+    3001,
+    4000,
+    4.1076,
+    0.4807,
+    0.7174,
+    0.0
+   ],
+   [
+    4001,
+    5000,
+    3.9558,
+    0.3996,
+    1.1396,
+    0.001
+   ],
+   [
+    5001,
+    6000,
+    3.871,
+    0.3681,
+    0.7807,
+    0.0
+   ],
+   [
+    6001,
+    7000,
+    3.7998,
+    0.3474,
+    0.6282,
+    0.0
+   ],
+   [
+    7001,
+    8000,
+    3.7507,
+    0.3329,
+    0.6729,
+    0.0
+   ],
+   [
+    8001,
+    9000,
+    3.6995,
+    0.3375,
+    1.8578,
+    0.003
+   ],
+   [
+    9001,
+    10000,
+    3.6645,
+    0.3504,
+    2.755,
+    0.007
+   ],
+   [
+    10001,
+    11000,
+    3.6362,
+    0.3474,
+    1.5414,
+    0.003
+   ],
+   [
+    11001,
+    12000,
+    3.6128,
+    0.3637,
+    1.6909,
+    0.008
+   ],
+   [
+    12001,
+    13000,
+    3.5782,
+    0.3606,
+    1.9604,
+    0.003
+   ],
+   [
+    13001,
+    14000,
+    3.5521,
+    0.3803,
+    3.2083,
+    0.01
+   ],
+   [
+    14001,
+    15000,
+    3.5423,
+    0.3852,
+    13.4772,
+    0.007
+   ],
+   [
+    15001,
+    16000,
+    3.5162,
+    0.3679,
+    2.353,
+    0.004
+   ],
+   [
+    16001,
+    17000,
+    3.5077,
+    0.3747,
+    1.7935,
+    0.003
+   ],
+   [
+    17001,
+    18000,
+    3.498,
+    0.3833,
+    2.0943,
+    0.008
+   ],
+   [
+    18001,
+    19000,
+    3.4732,
+    0.4175,
+    19.1287,
+    0.009
+   ],
+   [
+    19001,
+    20000,
+    3.4705,
+    0.381,
+    1.1418,
+    0.002
+   ],
+   [
+    20001,
+    21000,
+    3.4554,
+    0.3833,
+    2.1549,
+    0.003
+   ],
+   [
+    21001,
+    22000,
+    3.4492,
+    0.3934,
+    2.5727,
+    0.003
+   ],
+   [
+    22001,
+    23000,
+    3.4434,
+    0.3993,
+    1.7161,
+    0.006
+   ],
+   [
+    23001,
+    24000,
+    3.435,
+    0.4001,
+    1.8822,
+    0.005
+   ],
+   [
+    24001,
+    25000,
+    3.4244,
+    0.4147,
+    2.0988,
+    0.01
+   ],
+   [
+    25001,
+    26000,
+    3.4251,
+    0.4276,
+    3.7341,
+    0.013
+   ],
+   [
+    26001,
+    27000,
+    3.4117,
+    0.4135,
+    1.4897,
+    0.003
+   ],
+   [
+    27001,
+    28000,
+    3.4058,
+    0.4289,
+    2.1636,
+    0.004
+   ],
+   [
+    28001,
+    29000,
+    3.4117,
+    0.4328,
+    7.041,
+    0.006
+   ],
+   [
+    29001,
+    30000,
+    3.3956,
+    0.456,
+    7.5003,
+    0.007
+   ],
+   [
+    30001,
+    31000,
+    3.3889,
+    0.4602,
+    4.5034,
+    0.009
+   ],
+   [
+    31001,
+    32000,
+    3.384,
+    0.4394,
+    2.5379,
+    0.002
+   ],
+   [
+    32001,
+    33000,
+    3.3827,
+    0.4474,
+    1.7969,
+    0.005
+   ],
+   [
+    33001,
+    34000,
+    3.3823,
+    0.4613,
+    4.1479,
+    0.003
+   ],
+   [
+    34001,
+    35000,
+    3.3745,
+    0.4695,
+    4.6218,
+    0.013
+   ],
+   [
+    35001,
+    36000,
+    3.3783,
+    0.4547,
+    2.4859,
+    0.005
+   ],
+   [
+    36001,
+    37000,
+    3.3607,
+    0.4468,
+    0.8158,
+    0.0
+   ],
+   [
+    37001,
+    38000,
+    3.3576,
+    0.4536,
+    1.1436,
+    0.001
+   ],
+   [
+    38001,
+    39000,
+    3.3536,
+    0.4772,
+    13.9413,
+    0.003
+   ],
+   [
+    39001,
+    40000,
+    3.347,
+    0.4618,
+    4.3856,
+    0.003
+   ],
+   [
+    40001,
+    41000,
+    3.3455,
+    0.4622,
+    2.9804,
+    0.003
+   ],
+   [
+    41001,
+    42000,
+    3.3413,
+    0.4775,
+    12.0567,
+    0.003
+   ],
+   [
+    42001,
+    43000,
+    3.3406,
+    0.4694,
+    3.1809,
+    0.003
+   ],
+   [
+    43001,
+    44000,
+    3.3365,
+    0.4631,
+    1.3014,
+    0.001
+   ],
+   [
+    44001,
+    45000,
+    3.339,
+    0.4586,
+    1.3852,
+    0.003
+   ],
+   [
+    45001,
+    46000,
+    3.3259,
+    0.4638,
+    0.9383,
+    0.0
+   ],
+   [
+    46001,
+    47000,
+    3.3402,
+    0.4732,
+    5.3822,
+    0.003
+   ],
+   [
+    47001,
+    48000,
+    3.3184,
+    0.4661,
+    1.4338,
+    0.004
+   ],
+   [
+    48001,
+    49000,
+    3.3213,
+    0.4804,
+    2.9003,
+    0.003
+   ],
+   [
+    49001,
+    50000,
+    3.3106,
+    0.47,
+    3.1181,
+    0.004
+   ],
+   [
+    50001,
+    51000,
+    3.3105,
+    0.4798,
+    2.2618,
+    0.003
+   ],
+   [
+    51001,
+    52000,
+    3.3093,
+    0.4759,
+    1.258,
+    0.002
+   ],
+   [
+    52001,
+    53000,
+    3.3038,
+    0.4855,
+    1.5581,
+    0.003
+   ],
+   [
+    53001,
+    54000,
+    3.3003,
+    0.5078,
+    5.932,
+    0.012
+   ],
+   [
+    54001,
+    55000,
+    3.2979,
+    0.4774,
+    1.0334,
+    0.001
+   ],
+   [
+    55001,
+    56000,
+    3.2925,
+    0.4857,
+    1.3528,
+    0.003
+   ],
+   [
+    56001,
+    57000,
+    3.2898,
+    0.4792,
+    0.9706,
+    0.0
+   ],
+   [
+    57001,
+    58000,
+    3.2958,
+    0.4909,
+    3.4119,
+    0.002
+   ],
+   [
+    58001,
+    59000,
+    3.2828,
+    0.4838,
+    3.3821,
+    0.004
+   ],
+   [
+    59001,
+    60000,
+    3.2891,
+    0.4866,
+    1.6911,
+    0.003
+   ],
+   [
+    60001,
+    61000,
+    3.281,
+    0.4943,
+    1.0564,
+    0.001
+   ],
+   [
+    61001,
+    62000,
+    3.2787,
+    0.4844,
+    1.101,
+    0.002
+   ],
+   [
+    62001,
+    63000,
+    3.2724,
+    0.4919,
+    1.1718,
+    0.003
+   ],
+   [
+    63001,
+    64000,
+    3.2716,
+    0.4961,
+    1.0835,
+    0.002
+   ],
+   [
+    64001,
+    65000,
+    3.2725,
+    0.5024,
+    1.1472,
+    0.003
+   ],
+   [
+    65001,
+    66000,
+    3.2644,
+    0.5002,
+    1.32,
+    0.003
+   ],
+   [
+    66001,
+    67000,
+    3.2546,
+    0.5054,
+    1.8638,
+    0.003
+   ],
+   [
+    67001,
+    68000,
+    3.2643,
+    0.5025,
+    1.5174,
+    0.002
+   ],
+   [
+    68001,
+    69000,
+    3.2537,
+    0.4994,
+    1.1268,
+    0.001
+   ],
+   [
+    69001,
+    70000,
+    3.2502,
+    0.508,
+    0.9664,
+    0.0
+   ],
+   [
+    70001,
+    71000,
+    3.2505,
+    0.5066,
+    1.7512,
+    0.002
+   ],
+   [
+    71001,
+    72000,
+    3.2401,
+    0.5136,
+    4.4171,
+    0.005
+   ],
+   [
+    72001,
+    73000,
+    3.2392,
+    0.5031,
+    1.6987,
+    0.002
+   ],
+   [
+    73001,
+    74000,
+    3.2317,
+    0.51,
+    1.0716,
+    0.002
+   ],
+   [
+    74001,
+    75000,
+    3.231,
+    0.5103,
+    2.582,
+    0.001
+   ],
+   [
+    75001,
+    76000,
+    3.232,
+    0.5148,
+    0.9173,
+    0.0
+   ],
+   [
+    76001,
+    77000,
+    3.2238,
+    0.5154,
+    1.2744,
+    0.003
+   ],
+   [
+    77001,
+    78000,
+    3.2204,
+    0.5152,
+    0.9492,
+    0.0
+   ],
+   [
+    78001,
+    79000,
+    3.2232,
+    0.5143,
+    0.9523,
+    0.0
+   ],
+   [
+    79001,
+    80000,
+    3.208,
+    0.524,
+    1.0375,
+    0.002
+   ],
+   [
+    80001,
+    81000,
+    3.2025,
+    0.5222,
+    1.1009,
+    0.001
+   ],
+   [
+    81001,
+    82000,
+    3.2016,
+    0.5267,
+    1.9161,
+    0.004
+   ],
+   [
+    82001,
+    83000,
+    3.201,
+    0.5297,
+    1.0324,
+    0.001
+   ],
+   [
+    83001,
+    84000,
+    3.2052,
+    0.5259,
+    1.08,
+    0.002
+   ],
+   [
+    84001,
+    85000,
+    3.2008,
+    0.5303,
+    0.9897,
+    0.0
+   ],
+   [
+    85001,
+    86000,
+    3.1845,
+    0.5331,
+    2.2315,
+    0.002
+   ],
+   [
+    86001,
+    87000,
+    3.2007,
+    0.5338,
+    1.1827,
+    0.004
+   ],
+   [
+    87001,
+    88000,
+    3.1903,
+    0.5435,
+    1.8051,
+    0.005
+   ],
+   [
+    88001,
+    89000,
+    3.1851,
+    0.551,
+    1.2944,
+    0.005
+   ],
+   [
+    89001,
+    90000,
+    3.1809,
+    0.5475,
+    1.0767,
+    0.003
+   ],
+   [
+    90001,
+    91000,
+    3.1758,
+    0.5429,
+    1.0042,
+    0.001
+   ],
+   [
+    91001,
+    92000,
+    3.1751,
+    0.5433,
+    0.9719,
+    0.0
+   ],
+   [
+    92001,
+    93000,
+    3.1724,
+    0.5518,
+    1.1168,
+    0.003
+   ],
+   [
+    93001,
+    94000,
+    3.172,
+    0.5495,
+    1.2489,
+    0.005
+   ],
+   [
+    94001,
+    95000,
+    3.1696,
+    0.5597,
+    2.0796,
+    0.004
+   ],
+   [
+    95001,
+    96000,
+    3.1649,
+    0.5568,
+    1.8868,
+    0.004
+   ],
+   [
+    96001,
+    97000,
+    3.1668,
+    0.5651,
+    1.276,
+    0.002
+   ],
+   [
+    97001,
+    98000,
+    3.1559,
+    0.5635,
+    1.0723,
+    0.002
+   ],
+   [
+    98001,
+    99000,
+    3.1514,
+    0.5739,
+    1.0947,
+    0.004
+   ],
+   [
+    99001,
+    100000,
+    3.1453,
+    0.5714,
+    4.2261,
+    0.004
+   ],
+   [
+    100001,
+    101000,
+    3.1389,
+    0.5646,
+    1.0091,
+    0.001
+   ],
+   [
+    101001,
+    102000,
+    3.14,
+    0.5731,
+    1.0388,
+    0.001
+   ],
+   [
+    102001,
+    103000,
+    3.1407,
+    0.5861,
+    2.0673,
+    0.008
+   ],
+   [
+    103001,
+    104000,
+    3.1462,
+    0.58,
+    1.0356,
+    0.002
+   ],
+   [
+    104001,
+    105000,
+    3.1478,
+    0.597,
+    1.2768,
+    0.003
+   ],
+   [
+    105001,
+    106000,
+    3.1246,
+    0.5888,
+    1.2234,
+    0.002
+   ],
+   [
+    106001,
+    107000,
+    3.1244,
+    0.595,
+    1.1152,
+    0.003
+   ],
+   [
+    107001,
+    108000,
+    3.1148,
+    0.593,
+    1.1308,
+    0.003
+   ],
+   [
+    108001,
+    109000,
+    3.1168,
+    0.5922,
+    1.2022,
+    0.006
+   ],
+   [
+    109001,
+    110000,
+    3.1097,
+    0.6044,
+    1.1267,
+    0.002
+   ],
+   [
+    110001,
+    111000,
+    3.112,
+    0.6037,
+    1.2551,
+    0.004
+   ],
+   [
+    111001,
+    112000,
+    3.1074,
+    0.6224,
+    2.6109,
+    0.007
+   ],
+   [
+    112001,
+    113000,
+    3.1079,
+    0.6124,
+    1.1426,
+    0.005
+   ],
+   [
+    113001,
+    114000,
+    3.1021,
+    0.6133,
+    1.1242,
+    0.002
+   ],
+   [
+    114001,
+    115000,
+    3.1019,
+    0.6189,
+    1.1472,
+    0.007
+   ],
+   [
+    115001,
+    116000,
+    3.0894,
+    0.632,
+    1.2458,
+    0.001
+   ],
+   [
+    116001,
+    117000,
+    3.0808,
+    0.6284,
+    1.0943,
+    0.005
+   ],
+   [
+    117001,
+    118000,
+    3.0829,
+    0.624,
+    1.2099,
+    0.007
+   ],
+   [
+    118001,
+    119000,
+    3.0788,
+    0.6348,
+    1.3084,
+    0.003
+   ],
+   [
+    119001,
+    120000,
+    3.0737,
+    0.6517,
+    1.3869,
+    0.015
+   ],
+   [
+    120001,
+    121000,
+    3.0644,
+    0.66,
+    1.2275,
+    0.016
+   ],
+   [
+    121001,
+    122000,
+    3.0761,
+    0.6596,
+    1.4109,
+    0.007
+   ],
+   [
+    122001,
+    123000,
+    3.0777,
+    0.6606,
+    1.1979,
+    0.008
+   ],
+   [
+    123001,
+    124000,
+    3.064,
+    0.6674,
+    1.3809,
+    0.01
+   ],
+   [
+    124001,
+    125000,
+    3.0634,
+    0.6647,
+    1.2196,
+    0.009
+   ],
+   [
+    125001,
+    126000,
+    3.0651,
+    0.6686,
+    1.1487,
+    0.007
+   ],
+   [
+    126001,
+    127000,
+    3.052,
+    0.6756,
+    1.0928,
+    0.006
+   ],
+   [
+    127001,
+    128000,
+    3.0532,
+    0.6925,
+    1.269,
+    0.026
+   ],
+   [
+    128001,
+    129000,
+    3.0591,
+    0.6918,
+    1.2248,
+    0.019
+   ],
+   [
+    129001,
+    130000,
+    3.0449,
+    0.6853,
+    1.3284,
+    0.018
+   ],
+   [
+    130001,
+    131000,
+    3.0455,
+    0.6961,
+    1.2193,
+    0.016
+   ],
+   [
+    131001,
+    132000,
+    3.0486,
+    0.7047,
+    1.1984,
+    0.022
+   ],
+   [
+    132001,
+    133000,
+    3.045,
+    0.703,
+    5.4767,
+    0.012
+   ],
+   [
+    133001,
+    134000,
+    3.0319,
+    0.6959,
+    1.2449,
+    0.011
+   ],
+   [
+    134001,
+    135000,
+    3.0362,
+    0.7219,
+    1.2381,
+    0.019
+   ],
+   [
+    135001,
+    136000,
+    3.0287,
+    0.7057,
+    1.917,
+    0.015
+   ],
+   [
+    136001,
+    137000,
+    3.0349,
+    0.7172,
+    1.1884,
+    0.02
+   ],
+   [
+    137001,
+    138000,
+    3.0239,
+    0.7244,
+    1.193,
+    0.017
+   ],
+   [
+    138001,
+    139000,
+    3.0242,
+    0.7231,
+    1.1733,
+    0.025
+   ],
+   [
+    139001,
+    140000,
+    3.0266,
+    0.7424,
+    1.237,
+    0.029
+   ],
+   [
+    140001,
+    141000,
+    3.0157,
+    0.743,
+    1.6834,
+    0.032
+   ],
+   [
+    141001,
+    142000,
+    3.0283,
+    0.7309,
+    1.2567,
+    0.015
+   ],
+   [
+    142001,
+    143000,
+    3.0166,
+    0.7392,
+    1.2105,
+    0.029
+   ],
+   [
+    143001,
+    144000,
+    3.0174,
+    0.7513,
+    1.3392,
+    0.041
+   ],
+   [
+    144001,
+    145000,
+    3.0116,
+    0.762,
+    1.8557,
+    0.042
+   ],
+   [
+    145001,
+    146000,
+    3.0224,
+    0.7601,
+    1.2927,
+    0.037
+   ],
+   [
+    146001,
+    147000,
+    3.0024,
+    0.757,
+    1.3755,
+    0.029
+   ],
+   [
+    147001,
+    148000,
+    3.0058,
+    0.7706,
+    1.5253,
+    0.046
+   ],
+   [
+    148001,
+    149000,
+    3.0164,
+    0.7953,
+    1.4331,
+    0.07
+   ],
+   [
+    149001,
+    150000,
+    3.0048,
+    0.7819,
+    1.3301,
+    0.048
+   ],
+   [
+    150001,
+    151000,
+    3.0085,
+    0.7837,
+    1.373,
+    0.051
+   ],
+   [
+    151001,
+    152000,
+    3.0068,
+    0.8049,
+    1.3925,
+    0.068
+   ],
+   [
+    152001,
+    152580,
+    3.0116,
+    0.8048,
+    1.27,
+    0.069
+   ]
+  ],
+  "status": "finished",
+  "ppl": 20.370028561152527,
+  "ood_ppl": 50.02364864030602,
+  "trained": true,
+  "ckpt": "/mnt/local/helm-prcv-a1-84deb097/data/ckpt_unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8.pt"
+ }
+}

checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f5e534dde21f5f23837c99f4e8f74d50bff4b010d9699ef3d653eb305ee8ed0
+size 3886302822

compact_logs/policy.log

@@ -0,0 +1,15 @@
+# Selected scientific lines; this is a derived compact log, not the full raw worker log.
+2026-10-20 07:18:13 [policy] cinner and lorentzian_centroid held in fp32
+2026-10-20 07:18:13 [cfg] unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8
+2026-10-20 07:18:13 [env] {'torch': '2.11.0+cu128', 'gpu': 'NVIDIA B200', 'tf32_matmul': False, 'f32_mode': 'highest', 'python': '3.12.3'}
+2026-10-20 07:18:20 [model] 967.0M parameters
+2026-10-20 07:18:20 [data] corpus c420b (c4_20b_tokens.i32)
+2026-10-20 07:18:20 [data] 19999.70M training tokens; this cell processes 19999.0M, 1.00 of the corpus
+2026-10-20 07:18:20 [ood] 256 sequences from wikitext_train_tokens.pt
+2026-10-20 07:18:21 [radius] max over 49 normalizers 30.151; u = R/sqrt(c) = 30.151
+2026-10-20 07:18:21 [sampler] permutation of 9765477 windows; this cell uses 9765120 of them (1.000 passes)
+2026-10-20 07:18:21 [sched] warmup 3051 steps to 0.001, cosine to 0.0001 at step 152580
+2026-10-20 07:18:30 [resume] from step 152000, 3370 min so far
+2026-10-20 07:31:19 [unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8] step 152580  ppl      20.4  ood      50.0  (3367m train)
+2026-10-20 07:31:23 [health] steps 152001-152580: loss 3.0116  grad-norm mean 0.805 max 1.270  clipped 7%
+2026-10-20 07:31:24 [done] unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8: ppl 20.4, ood 50.0, 3367 min, trained

eval_curves/all_ready_runs.csv

@@ -34,3 +34,34 @@ unit7,7,5000,46.67910942924407,106.56456822620376
 unit7,7,10000,36.28621540996052,84.48696841726898
 unit99,99,5000,47.14196413877905,111.48137111560875
 unit99,99,10000,37.023797483960436,88.28929618013538
+policy,1234,5000,49.96224627206456,124.93939687264222
+policy,1234,10000,38.34757071766789,88.14201261882307
+policy,1234,15000,34.19195513906091,77.78597054694117
+policy,1234,20000,32.168524269995935,73.07426804069382
+policy,1234,25000,30.849758512410368,70.93661797748476
+policy,1234,30000,29.979062376304086,66.15964946320693
+policy,1234,35000,29.11726802688634,66.73714176781131
+policy,1234,40000,28.58433864688657,65.63382597489617
+policy,1234,45000,28.00962249511226,63.010533551917774
+policy,1234,50000,27.579803871894065,64.17758334317438
+policy,1234,55000,27.180486538834845,62.46369541840954
+policy,1234,60000,26.82660577049195,63.131650199754795
+policy,1234,65000,26.256727775702622,60.2085794511229
+policy,1234,70000,25.831258275309533,59.216968578191036
+policy,1234,75000,25.428781915034556,58.26023149826963
+policy,1234,80000,25.041509583243812,56.6621200346212
+policy,1234,85000,24.611292915690886,57.07487729825752
+policy,1234,90000,24.228492172949245,56.606062648114815
+policy,1234,95000,23.80994974806001,56.30148377354903
+policy,1234,100000,23.38686133823178,53.300426090221876
+policy,1234,105000,22.954160491577337,53.61922907480571
+policy,1234,110000,22.50397573777943,53.02736039799391
+policy,1234,115000,22.17635856096328,52.0023925595677
+policy,1234,120000,21.843981902662684,51.849058306007166
+policy,1234,125000,21.427204836865872,51.25451432552149
+policy,1234,130000,21.156095309325533,50.076957473304404
+policy,1234,135000,20.890162268957365,49.28759955513546
+policy,1234,140000,20.667042690067046,49.14775593251116
+policy,1234,145000,20.497996371069945,49.62698778818433
+policy,1234,150000,20.394912540783352,49.61545958685712
+policy,1234,152580,20.370028561152527,50.02364864030602

eval_curves/policy.csv

@@ -0,0 +1,32 @@
+optimizer_step,c4_heldout_ppl,wikitext_heldout_ppl
+5000,49.96224627206456,124.93939687264222
+10000,38.34757071766789,88.14201261882307
+15000,34.19195513906091,77.78597054694117
+20000,32.168524269995935,73.07426804069382
+25000,30.849758512410368,70.93661797748476
+30000,29.979062376304086,66.15964946320693
+35000,29.11726802688634,66.73714176781131
+40000,28.58433864688657,65.63382597489617
+45000,28.00962249511226,63.010533551917774
+50000,27.579803871894065,64.17758334317438
+55000,27.180486538834845,62.46369541840954
+60000,26.82660577049195,63.131650199754795
+65000,26.256727775702622,60.2085794511229
+70000,25.831258275309533,59.216968578191036
+75000,25.428781915034556,58.26023149826963
+80000,25.041509583243812,56.6621200346212
+85000,24.611292915690886,57.07487729825752
+90000,24.228492172949245,56.606062648114815
+95000,23.80994974806001,56.30148377354903
+100000,23.38686133823178,53.300426090221876
+105000,22.954160491577337,53.61922907480571
+110000,22.50397573777943,53.02736039799391
+115000,22.17635856096328,52.0023925595677
+120000,21.843981902662684,51.849058306007166
+125000,21.427204836865872,51.25451432552149
+130000,21.156095309325533,50.076957473304404
+135000,20.890162268957365,49.28759955513546
+140000,20.667042690067046,49.14775593251116
+145000,20.497996371069945,49.62698778818433
+150000,20.394912540783352,49.61545958685712
+152580,20.370028561152527,50.02364864030602

geometry_policy.json

@@ -0,0 +1,20 @@
+{
+  "activation": "Apply the pinned trainer's R_POLICY=1 method wrappers before downstream model use.",
+  "autocast_disabled_inside_protected_operations": true,
+  "cell_key": "unit_bf16_d910_c1_lr0.001_acc1_s1234_st152580_ev5000_pol_corpc420b_perm_cos_mb8_w8",
+  "compile": false,
+  "curvature_c": 1.0,
+  "initialization": "published",
+  "parameter_precision": "fp32",
+  "planned_steps": 152580,
+  "policy": true,
+  "protected_operation_precision": "fp32",
+  "protected_operations": [
+    "Lorentz.cinner",
+    "Lorentz.lorentzian_centroid"
+  ],
+  "remaining_training_recipe": "unchanged BF16 20B recipe",
+  "runtime_policy_serialized_in_checkpoint": false,
+  "seed": 1234,
+  "training_precision": "bf16"
+}