Upload verified HELM artifacts [2d0602a5e5934c3dbf76eb922bdd1ead]

CHECKPOINTS.json

@@ -0,0 +1,59 @@
+{
+  "checkpoints": [
+    {
+      "actual_step": 19500,
+      "bytes": 3885189944,
+      "crc32c": "l4FVdw==",
+      "final_checkpoint": false,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": false,
+      "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32_last_finite_saved_step19500.pt",
+      "run": "scaled4",
+      "sha256": "d4e743e7c8e1fc74adc169daafabad4a5543b21c4e3af813c1afe32d2a64a379",
+      "status": "verified_last_finite_export"
+    },
+    {
+      "actual_step": 37500,
+      "bytes": 3885189944,
+      "crc32c": "SjiFrA==",
+      "final_checkpoint": false,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": false,
+      "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32_last_finite_saved_step37500.pt",
+      "run": "scaled8",
+      "sha256": "42ff7e2ea0ad417a592c5c2756331111dd51ec0c900438c79d9eed5a35ce63d0",
+      "status": "verified_last_finite_export"
+    },
+    {
+      "actual_step": 21000,
+      "bytes": 3885189944,
+      "crc32c": "6z4blA==",
+      "final_checkpoint": false,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": false,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8_last_finite_saved_step21000.pt",
+      "run": "unit_bf16_s7",
+      "sha256": "95cb6389f3a95cc91aca0931dc4cee44691a8c498f5f29fd6525ab0aed04dff4",
+      "status": "verified_last_finite_export"
+    },
+    {
+      "actual_step": 61000,
+      "bytes": 3886231570,
+      "crc32c": "OXOu4g==",
+      "final_checkpoint": true,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": true,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.pt",
+      "run": "unit_bf16_s99",
+      "sha256": "0490250edd59c85f2d8b53cba3238f42b39ab839ed8f8aa0452815f5f3b586df",
+      "status": "verified_native_final"
+    }
+  ],
+  "optimizer_resumes_included": false,
+  "planned_step_in_native_key_is_not_evidence_of_full_budget_completion": true,
+  "schema": "helm.c42b.public-checkpoints.v1"
+}

CHECKSUMS.json

@@ -0,0 +1,216 @@
+{
+  "files": [
+    {
+      "bytes": 8025,
+      "path": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+      "sha256": "6ef01b6eb73a5a58a9a7a58d674b9052b33f8c6f43d13ae68586854312a4312e"
+    },
+    {
+      "bytes": 414,
+      "path": "eval_curves/scaled4.csv",
+      "sha256": "e8047389ca837ac4171a71da9b7d12a1b9568a711e71ef805b5a419627d31ee8"
+    },
+    {
+      "bytes": 2057,
+      "path": "health/scaled4.csv",
+      "sha256": "361dabfb6054d36a30d00749d9ca1217a981d95f10df34e315062f282293efff"
+    },
+    {
+      "bytes": 8475,
+      "path": "scientific_logs/scaled4.log",
+      "sha256": "84e33dc05a9267b3eff7ae59991c3aff626886cc94b826bca85e2f50b5c9e1c4"
+    },
+    {
+      "bytes": 3732,
+      "path": "profiles/scaled4.json",
+      "sha256": "6523bcb12b25c6adbd38f251dedddd7d0840f4592efc7bf8826a5f9a36cca2a5"
+    },
+    {
+      "bytes": 3885189944,
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32_last_finite_saved_step19500.pt",
+      "sha256": "d4e743e7c8e1fc74adc169daafabad4a5543b21c4e3af813c1afe32d2a64a379"
+    },
+    {
+      "bytes": 11515,
+      "path": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+      "sha256": "566d1325afbb580f0d8485054e6b56e95eec2b96f2a7393a4c3a6a7cb5bff177"
+    },
+    {
+      "bytes": 806,
+      "path": "eval_curves/scaled8.csv",
+      "sha256": "19d50a1454953096509a05961b3d6e7a3e583288de338185e5d74a74c5593390"
+    },
+    {
+      "bytes": 3346,
+      "path": "health/scaled8.csv",
+      "sha256": "185a1b7957e6025eea46274010635a905d3719317d831450f0f2c8661ac4903e"
+    },
+    {
+      "bytes": 13678,
+      "path": "scientific_logs/scaled8.log",
+      "sha256": "57f9f0ba62a18a72eeab324458e22faf41bb5f6a7c8fb1e7997d276d27aad552"
+    },
+    {
+      "bytes": 3734,
+      "path": "profiles/scaled8.json",
+      "sha256": "36a6f469b9de51011d4224853b595c11fecf71e5928bb0d66748c8277bc2f12a"
+    },
+    {
+      "bytes": 3885189944,
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32_last_finite_saved_step37500.pt",
+      "sha256": "42ff7e2ea0ad417a592c5c2756331111dd51ec0c900438c79d9eed5a35ce63d0"
+    },
+    {
+      "bytes": 8582,
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.json",
+      "sha256": "b8560c64449f735ab4d712e6a4df7aca7910bb856ed44e22bf2bf75768d549eb"
+    },
+    {
+      "bytes": 457,
+      "path": "eval_curves/unit_bf16_s7.csv",
+      "sha256": "95316fe251685246f4efe8f50c5465655e63f11a4d1a7e722d16360e64679489"
+    },
+    {
+      "bytes": 2322,
+      "path": "health/unit_bf16_s7.csv",
+      "sha256": "b886521fb2574d07991a35cb252e64169e5dff9265dca5a8f524f6f63ba79e1e"
+    },
+    {
+      "bytes": 7578,
+      "path": "scientific_logs/unit_bf16_s7.log",
+      "sha256": "ae7183f19ce5ef485bd3f7c7e720c6a1ab5c9a460a9764ab7f23fab7c614a6e5"
+    },
+    {
+      "bytes": 3808,
+      "path": "profiles/unit_bf16_s7.json",
+      "sha256": "c2bf5b8a3325ba31699f4def296f52c929d70cf1337c334200e8718b36d9ea4c"
+    },
+    {
+      "bytes": 3885189944,
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8_last_finite_saved_step21000.pt",
+      "sha256": "95cb6389f3a95cc91aca0931dc4cee44691a8c498f5f29fd6525ab0aed04dff4"
+    },
+    {
+      "bytes": 16519,
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.json",
+      "sha256": "d396be378862680c3c53f90e0d7b29f77fbf178d1364d5af364b9a8f91202428"
+    },
+    {
+      "bytes": 1378,
+      "path": "eval_curves/unit_bf16_s99.csv",
+      "sha256": "4e63e21807f173088d3ac8d825661ff514d94f6369f387cc2986a2fbb6584682"
+    },
+    {
+      "bytes": 5391,
+      "path": "health/unit_bf16_s99.csv",
+      "sha256": "55e12010c5aa2549f0ef78b9667ab3fb5db707909a02c7c6881089cd0cab6b62"
+    },
+    {
+      "bytes": 18993,
+      "path": "scientific_logs/unit_bf16_s99.log",
+      "sha256": "fe32e36ced5e421023ca38bb2e3eed64dda25b98dc15a12c1054192d85185508"
+    },
+    {
+      "bytes": 3836,
+      "path": "profiles/unit_bf16_s99.json",
+      "sha256": "fef847b3142666a22cb0047760c751af615a349673ea4ec761c663b24dcc24e2"
+    },
+    {
+      "bytes": 3886231570,
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.pt",
+      "sha256": "0490250edd59c85f2d8b53cba3238f42b39ab839ed8f8aa0452815f5f3b586df"
+    },
+    {
+      "bytes": 3803,
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+      "sha256": "c8024707c0bcc82ac8f0d43e78fbc458c4f096f1cbbb97b04a711d161980872a"
+    },
+    {
+      "bytes": 26,
+      "path": "eval_curves/unit4.csv",
+      "sha256": "94f2a495c39b5e11638210689d7887eb6d2b469af20e9c62771aa34b169cc87b"
+    },
+    {
+      "bytes": 169,
+      "path": "health/unit4.csv",
+      "sha256": "2889cc03404cfc1fd2861815272c443eebd9edbf99ab98987b7f3641132c353b"
+    },
+    {
+      "bytes": 2088,
+      "path": "scientific_logs/unit4.log",
+      "sha256": "59a59df95671799b19ea68e5ff475b3ff93fc4f3f364bf9d3c7521788d80860a"
+    },
+    {
+      "bytes": 3735,
+      "path": "profiles/unit4.json",
+      "sha256": "a0f9da3bad5629261adb84bb6ed497f24953cb05ff6693523ce0b2ff9300fdca"
+    },
+    {
+      "bytes": 3799,
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+      "sha256": "c593f530ac19449f8adb4464c72383b51048ab95f629f2af5a3edff8ee2a6cf8"
+    },
+    {
+      "bytes": 26,
+      "path": "eval_curves/unit8.csv",
+      "sha256": "94f2a495c39b5e11638210689d7887eb6d2b469af20e9c62771aa34b169cc87b"
+    },
+    {
+      "bytes": 164,
+      "path": "health/unit8.csv",
+      "sha256": "5e0c21319fca760b643b2d6051de0f838185f47f15fbad1884a9f2fdd3efd1c0"
+    },
+    {
+      "bytes": 2091,
+      "path": "scientific_logs/unit8.log",
+      "sha256": "0a574595492ac05acbf5897f2d6dbf11f3236321a6073c844ef5d1fbb954f01e"
+    },
+    {
+      "bytes": 3736,
+      "path": "profiles/unit8.json",
+      "sha256": "a9373d5ae4165e3221a7218c21957603ce7a242242cf23b399e79189738bb209"
+    },
+    {
+      "bytes": 4014,
+      "path": "eval_curves/all_runs.csv",
+      "sha256": "b2581bdaa4fa8866b95ced9b000749f42e732c0fc0a76a28ce5d57f15b4577f0"
+    },
+    {
+      "bytes": 5635,
+      "path": "RESULTS.json",
+      "sha256": "b35bf886ffc7f4b4c32e29b41370da1955799b1a0b9f060d6c01a64c78e57820"
+    },
+    {
+      "bytes": 1106,
+      "path": "RESULTS.md",
+      "sha256": "1ec5ed8749fac40ab38f27090262c3ae26b7f95d59a7d3ad6e216e72baf46e98"
+    },
+    {
+      "bytes": 2713,
+      "path": "CHECKPOINTS.json",
+      "sha256": "d49659db9f62c33527506f496167d386e9bc37a5f3c13d380de5a72ad8f03362"
+    },
+    {
+      "bytes": 2344,
+      "path": "EXPORTS.json",
+      "sha256": "32d75b67576654d8f7819e2fe0448b99fb11f6998595398ce4617f1cdc5236c0"
+    },
+    {
+      "bytes": 2866,
+      "path": "README.md",
+      "sha256": "cf401e18c4e72e910d06d11a05100cfe1e6929f551aaeed53865f9b6d94f35f0"
+    },
+    {
+      "bytes": 2265,
+      "path": "LOADING.md",
+      "sha256": "d907c380e7572ca0de677c16cf765dfbeadade4bd4845f3c26d44b637266180a"
+    },
+    {
+      "bytes": 17033,
+      "path": "SMALL_MANIFEST.json",
+      "sha256": "63513ab4fe0c9c78c67a0260889bd83b1c4ba00c2fc631bc2aabe389a8b0602b"
+    }
+  ],
+  "schema": "helm.c42b.public-checksums.v1",
+  "self_included": false
+}

EXPORTS.json

@@ -0,0 +1,65 @@
+{
+  "model_only_exports": [
+    {
+      "buffer_casts": [
+        {
+          "elements": 4194304,
+          "from": "torch.bool",
+          "name": "attn_mask",
+          "to": "torch.float32"
+        }
+      ],
+      "bytes": 3885189944,
+      "export_sha256": "d4e743e7c8e1fc74adc169daafabad4a5543b21c4e3af813c1afe32d2a64a379",
+      "final_checkpoint": false,
+      "full_budget_completed": false,
+      "parameter_bytes_unchanged": true,
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32_last_finite_saved_step19500.pt",
+      "run": "scaled4",
+      "source_resume_sha256": "715c2fc7e21a4d69bde6b6b4399c7c0f23aedb8bf8622fb3b606288e265fa463",
+      "source_step": 19500
+    },
+    {
+      "buffer_casts": [
+        {
+          "elements": 4194304,
+          "from": "torch.bool",
+          "name": "attn_mask",
+          "to": "torch.float32"
+        }
+      ],
+      "bytes": 3885189944,
+      "export_sha256": "42ff7e2ea0ad417a592c5c2756331111dd51ec0c900438c79d9eed5a35ce63d0",
+      "final_checkpoint": false,
+      "full_budget_completed": false,
+      "parameter_bytes_unchanged": true,
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32_last_finite_saved_step37500.pt",
+      "run": "scaled8",
+      "source_resume_sha256": "51515fdce794860dd28e603c2d38f53caffc0b0f6a083433811fb710cf7feb19",
+      "source_step": 37500
+    },
+    {
+      "buffer_casts": [
+        {
+          "elements": 4194304,
+          "from": "torch.bool",
+          "name": "attn_mask",
+          "to": "torch.float32"
+        }
+      ],
+      "bytes": 3885189944,
+      "export_sha256": "95cb6389f3a95cc91aca0931dc4cee44691a8c498f5f29fd6525ab0aed04dff4",
+      "final_checkpoint": false,
+      "full_budget_completed": false,
+      "parameter_bytes_unchanged": true,
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8_last_finite_saved_step21000.pt",
+      "run": "unit_bf16_s7",
+      "source_resume_sha256": "a5d9c2cb411714db03459aec6bbd09afb1d4a7cdd59b20acf2e4cad2b5b9d7f8",
+      "source_step": 21000
+    }
+  ],
+  "native_bool_mask_cast_only": true,
+  "native_seed99_final_reused": true,
+  "optimizer_resumes_included": false,
+  "schema": "helm.c42b.public-exports.v1"
+}

LOADING.md

@@ -0,0 +1,45 @@
+# Loading the native model-only checkpoints
+
+Use the pinned HELM/model configuration and evaluator supplied with PR2
+`5edb20302017712ac97bf39374f66db80f94bc51` (HELM `7501deca8f413848bfef804be64ce874b72a3cd7`). Checkpoint state is FP32 and has exactly two keys:
+`key` and `state_dict`. The training environment used Python 3.12.3,
+PyTorch 2.11.0+cu128 and CUDA 12.8.
+
+```python
+import torch
+payload = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
+assert set(payload) == {"key", "state_dict"}
+```
+
+These are custom HELM native archives. Load them through the supplied HELM
+evaluator; they are not transformers save_pretrained/AutoModel repositories.
+Construct the matching width-910, 16-layer HELM with curvature1 and the
+configuration used by the supplied evaluator, then load state_dict strictly.
+Training used world8; preserve the corresponding model configuration when
+adapting the evaluator. Evaluate with the native BF16 autocast context while
+retaining FP32 parameter storage.
+
+Follow the original init-dependent construction/radius recipe before loading
+the learned state. Scaled runs recorded R_init=u_init=1; published native BF16
+seeds7/99 recorded R_init=u_init=30.1496. Each profiles/*.json and unchanged
+native result retains the measured normalizer radii. Do not reapply initializer
+scaling to learned weights after loading them.
+
+For Scaled4, install the supplied PR2 hooks once in a fresh evaluation process:
+
+```python
+from helm.hypercore.manifolds import Lorentz
+import lowp_emul
+lowp_emul.install(Lorentz, fmt="e2m1", block=32, ops="both")
+```
+
+For Scaled8, use fmt="e4m3". Install the hooks before model construction.
+Native BF16 seeds7/99 use neither lowp hooks nor the FP32 geometry policy.
+The bit-width suffix identifies operand emulation, not quantized weight storage.
+Run each profile in a fresh process so different wrappers do not stack.
+
+CHECKPOINTS.json distinguishes the seed99 full-budget final checkpoint from
+the three exports of earlier finite saved states. The latter are NOT FINAL
+and must be reported with their actual saved step. Optimizer/RNG resumes are
+not included here. The native scalar final PPL is NaN for failed runs; the
+last finite evaluation in RESULTS.json is a separate earlier observation.

README.md

@@ -0,0 +1,53 @@
+---
+library_name: pytorch
+pipeline_tag: text-generation
+tags:
+- helm
+- hyperbolic
+- bf16
+- fp4
+- fp8
+---
+# HELM 1B: C4 2B attention operand experiments
+
+Six eager HELM runs use the same width-910, 16-layer model and the 61,000-step
+2B recipe. Scaled4/Scaled8 and published Unit4/Unit8 emulate E2M1/E4M3 rounding
+of the operands to both attention geometry operations, in blocks of 32.
+The rest of training is BF16 with FP32 parameters; these are not packed
+4-bit/8-bit model weights. Native BF16 Unit seeds 7/99 have no rounding hooks
+and no FP32 geometry policy.
+
+| Run | Native outcome | Actual terminal step | Final C4 / WikiText PPL |
+| --- | --- | ---: | --- |
+| scaled4 | nonfinite | 19600 | No final PPL; see last finite evaluation in RESULTS.json |
+| scaled8 | nonfinite | 37800 | No final PPL; see last finite evaluation in RESULTS.json |
+| unit_bf16_s7 | nonfinite | 21300 | No final PPL; see last finite evaluation in RESULTS.json |
+| unit_bf16_s99 | finished | 61000 | 42.14531938222052 / 150.1752538901007 |
+| unit4 | nonfinite | 100 | No final PPL; see last finite evaluation in RESULTS.json |
+| unit8 | nonfinite | 500 | No final PPL; see last finite evaluation in RESULTS.json |
+
+Only BF16 seed99 completed all 61,000 steps. Its final C4/WikiText PPL is
+**42.14531938222052 / 150.1752538901007**. Other runs ended through the native
+loss/pre-clip-gradient-norm nonfinite guard. Their JSONs retain intentional
+NaNs and original bytes. Early Unit4/Unit8 have no evaluation or saved weights.
+The last finite model exports for Scaled4, Scaled8 and BF16 seed7 are explicitly
+**NOT FINAL**, from saved steps **19,500 / 37,500 / 21,000**. Their native keys
+encode the planned 61,000-step horizon; the suffix and CHECKPOINTS.json give
+the actual saved step.
+
+All six native cells.d JSONs, per-run evaluation/health CSVs, the combined
+68-point curve and compact scientific logs are included. Full operator logs
+and optimizer resumes stay outside this model package. The original 20B
+comparison is a separate package.
+
+See RESULTS.json for last finite evaluation scores and profiles/ for each run.
+Lowp checkpoints require the corresponding E2M1/E4M3 hooks for downstream
+evaluation; loading their keys alone does not install those hooks.
+See [LOADING.md](LOADING.md) and [CHECKPOINTS.json](CHECKPOINTS.json).
+
+The pinned scientific code is [PR2 at 5edb20302017712ac97bf39374f66db80f94bc51](https://github.com/kaisaraka/helm-scale-repro/tree/5edb20302017712ac97bf39374f66db80f94bc51),
+with HELM revision `7501deca8f413848bfef804be64ce874b72a3cd7`. The source repository is private; use the provided
+version when reproducing its model configuration and attention wrappers.
+The native C4 tensor matches SHA256
+`03c0c8663b10d51b0117ae97264e085e95c9c90ef332f2765ed4087844dddb7c`;
+WikiText matches `2759990ff21d21050c8e223d855cd785b8a5c2b2d8f02fafbb6b8d4023ae3609`.

RESULTS.json

@@ -0,0 +1,143 @@
+{
+  "health_reporting": "Native aggregate intervals; the last interval can span nonfinite detection. These are not per-step diagnostics.",
+  "paired_evaluation_points": 68,
+  "runs": [
+    {
+      "actual_terminal_step": 19600,
+      "attention_operand_format": "e2m1",
+      "evaluation_points": 9,
+      "final_c4_ppl": null,
+      "final_wikitext_ppl": null,
+      "first_nonfinite_step": 19576,
+      "full_budget_completed": false,
+      "initialization": "scaled",
+      "last_finite_evaluation": {
+        "c4_ppl": 1283.73413189331,
+        "step": 18000,
+        "wikitext_ppl": 2968.7661825600626
+      },
+      "last_finite_saved_step": 19500,
+      "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+      "native_json": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+      "native_json_sha256": "6ef01b6eb73a5a58a9a7a58d674b9052b33f8c6f43d13ae68586854312a4312e",
+      "planned_steps": 61000,
+      "planned_tokens": 1998848000,
+      "run": "scaled4",
+      "seed": 1234,
+      "status": "nonfinite"
+    },
+    {
+      "actual_terminal_step": 37800,
+      "attention_operand_format": "e4m3",
+      "evaluation_points": 18,
+      "final_c4_ppl": null,
+      "final_wikitext_ppl": null,
+      "first_nonfinite_step": 37785,
+      "full_budget_completed": false,
+      "initialization": "scaled",
+      "last_finite_evaluation": {
+        "c4_ppl": 64.18778187722458,
+        "step": 36000,
+        "wikitext_ppl": 178.46032642706157
+      },
+      "last_finite_saved_step": 37500,
+      "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+      "native_json": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+      "native_json_sha256": "566d1325afbb580f0d8485054e6b56e95eec2b96f2a7393a4c3a6a7cb5bff177",
+      "planned_steps": 61000,
+      "planned_tokens": 1998848000,
+      "run": "scaled8",
+      "seed": 1234,
+      "status": "nonfinite"
+    },
+    {
+      "actual_terminal_step": 21300,
+      "attention_operand_format": null,
+      "evaluation_points": 10,
+      "final_c4_ppl": null,
+      "final_wikitext_ppl": null,
+      "first_nonfinite_step": 21249,
+      "full_budget_completed": false,
+      "initialization": "unit",
+      "last_finite_evaluation": {
+        "c4_ppl": 1971.1374237795667,
+        "step": 20000,
+        "wikitext_ppl": 3768.617090639044
+      },
+      "last_finite_saved_step": 21000,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+      "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.json",
+      "native_json_sha256": "b8560c64449f735ab4d712e6a4df7aca7910bb856ed44e22bf2bf75768d549eb",
+      "planned_steps": 61000,
+      "planned_tokens": 1998848000,
+      "run": "unit_bf16_s7",
+      "seed": 7,
+      "status": "nonfinite"
+    },
+    {
+      "actual_terminal_step": 61000,
+      "attention_operand_format": null,
+      "evaluation_points": 31,
+      "final_c4_ppl": 42.14531938222052,
+      "final_wikitext_ppl": 150.1752538901007,
+      "first_nonfinite_step": null,
+      "full_budget_completed": true,
+      "initialization": "unit",
+      "last_finite_evaluation": {
+        "c4_ppl": 42.14531938222052,
+        "step": 61000,
+        "wikitext_ppl": 150.1752538901007
+      },
+      "last_finite_saved_step": null,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+      "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.json",
+      "native_json_sha256": "d396be378862680c3c53f90e0d7b29f77fbf178d1364d5af364b9a8f91202428",
+      "planned_steps": 61000,
+      "planned_tokens": 1998848000,
+      "run": "unit_bf16_s99",
+      "seed": 99,
+      "status": "finished"
+    },
+    {
+      "actual_terminal_step": 100,
+      "attention_operand_format": "e2m1",
+      "evaluation_points": 0,
+      "final_c4_ppl": null,
+      "final_wikitext_ppl": null,
+      "first_nonfinite_step": 94,
+      "full_budget_completed": false,
+      "initialization": "unit",
+      "last_finite_evaluation": null,
+      "last_finite_saved_step": null,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+      "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+      "native_json_sha256": "c8024707c0bcc82ac8f0d43e78fbc458c4f096f1cbbb97b04a711d161980872a",
+      "planned_steps": 61000,
+      "planned_tokens": 1998848000,
+      "run": "unit4",
+      "seed": 1234,
+      "status": "nonfinite"
+    },
+    {
+      "actual_terminal_step": 500,
+      "attention_operand_format": "e4m3",
+      "evaluation_points": 0,
+      "final_c4_ppl": null,
+      "final_wikitext_ppl": null,
+      "first_nonfinite_step": 470,
+      "full_budget_completed": false,
+      "initialization": "unit",
+      "last_finite_evaluation": null,
+      "last_finite_saved_step": null,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+      "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+      "native_json_sha256": "c593f530ac19449f8adb4464c72383b51048ab95f629f2af5a3edff8ee2a6cf8",
+      "planned_steps": 61000,
+      "planned_tokens": 1998848000,
+      "run": "unit8",
+      "seed": 1234,
+      "status": "nonfinite"
+    }
+  ],
+  "schema": "helm.c42b.public-results.v1"
+}

RESULTS.md

@@ -0,0 +1,15 @@
+# Verified 2B results
+
+| Run | Native outcome | First nonfinite / detected | Saved model step | Last finite evaluation C4 / WikiText |
+| --- | --- | --- | --- | --- |
+| scaled4 | nonfinite | 19576 / 19600 | 19500 | 18000: 1283.73413189331 / 2968.7661825600626 |
+| scaled8 | nonfinite | 37785 / 37800 | 37500 | 36000: 64.18778187722458 / 178.46032642706157 |
+| unit_bf16_s7 | nonfinite | 21249 / 21300 | 21000 | 20000: 1971.1374237795667 / 3768.617090639044 |
+| unit_bf16_s99 | finished | Finished 61,000 | 61000 | 61000: 42.14531938222052 / 150.1752538901007 |
+| unit4 | nonfinite | 94 / 100 | None | No evaluation |
+| unit8 | nonfinite | 470 / 500 | None | No evaluation |
+
+Only native BF16 seed99 completed the full budget; its final C4/WikiText PPL is **42.14531938222052 / 150.1752538901007**.
+All other listed evaluations precede a nonfinite exit and are not final full-budget PPL.
+The three partial model exports are from saved steps19,500/37,500/21,000, not the intervening last finite optimizer update.
+Native JSONs retain NaNs byte-for-byte; health intervals retain native aggregation boundaries.

SMALL_MANIFEST.json

@@ -0,0 +1,411 @@
+{
+  "checkpoint_reservations": [
+    {
+      "actual_step": 19500,
+      "bytes": 3885189944,
+      "crc32c": "l4FVdw==",
+      "final_checkpoint": false,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": false,
+      "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32_last_finite_saved_step19500.pt",
+      "run": "scaled4",
+      "sha256": "d4e743e7c8e1fc74adc169daafabad4a5543b21c4e3af813c1afe32d2a64a379",
+      "status": "verified_last_finite_export"
+    },
+    {
+      "actual_step": 37500,
+      "bytes": 3885189944,
+      "crc32c": "SjiFrA==",
+      "final_checkpoint": false,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": false,
+      "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+      "path": "checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32_last_finite_saved_step37500.pt",
+      "run": "scaled8",
+      "sha256": "42ff7e2ea0ad417a592c5c2756331111dd51ec0c900438c79d9eed5a35ce63d0",
+      "status": "verified_last_finite_export"
+    },
+    {
+      "actual_step": 21000,
+      "bytes": 3885189944,
+      "crc32c": "6z4blA==",
+      "final_checkpoint": false,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": false,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8_last_finite_saved_step21000.pt",
+      "run": "unit_bf16_s7",
+      "sha256": "95cb6389f3a95cc91aca0931dc4cee44691a8c498f5f29fd6525ab0aed04dff4",
+      "status": "verified_last_finite_export"
+    },
+    {
+      "actual_step": 61000,
+      "bytes": 3886231570,
+      "crc32c": "OXOu4g==",
+      "final_checkpoint": true,
+      "format": "native {key,state_dict}; model-only; FP32 state",
+      "full_budget_completed": true,
+      "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+      "path": "checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.pt",
+      "run": "unit_bf16_s99",
+      "sha256": "0490250edd59c85f2d8b53cba3238f42b39ab839ed8f8aa0452815f5f3b586df",
+      "status": "verified_native_final"
+    }
+  ],
+  "files": [
+    {
+      "bytes": 8025,
+      "crc32c": "mtuxfQ==",
+      "path": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+      "provenance": {
+        "kind": "unchanged_native_result",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_sha256": "6ef01b6eb73a5a58a9a7a58d674b9052b33f8c6f43d13ae68586854312a4312e"
+      },
+      "sha256": "6ef01b6eb73a5a58a9a7a58d674b9052b33f8c6f43d13ae68586854312a4312e"
+    },
+    {
+      "bytes": 414,
+      "crc32c": "g7JDug==",
+      "path": "eval_curves/scaled4.csv",
+      "provenance": {
+        "kind": "unchanged_collected_evaluation.csv",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_sha256": "e8047389ca837ac4171a71da9b7d12a1b9568a711e71ef805b5a419627d31ee8"
+      },
+      "sha256": "e8047389ca837ac4171a71da9b7d12a1b9568a711e71ef805b5a419627d31ee8"
+    },
+    {
+      "bytes": 2057,
+      "crc32c": "adoLIQ==",
+      "path": "health/scaled4.csv",
+      "provenance": {
+        "kind": "unchanged_collected_health.csv",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_sha256": "361dabfb6054d36a30d00749d9ca1217a981d95f10df34e315062f282293efff"
+      },
+      "sha256": "361dabfb6054d36a30d00749d9ca1217a981d95f10df34e315062f282293efff"
+    },
+    {
+      "bytes": 8475,
+      "crc32c": "mxniLw==",
+      "path": "scientific_logs/scaled4.log",
+      "provenance": {
+        "kind": "filtered_scientific_lines",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_log_sha256": "51637063c8a97a428b9b7ed9c973690e383643912dbdfd0e4b37dbef265db0c4",
+        "worker_paths_redacted": true
+      },
+      "sha256": "84e33dc05a9267b3eff7ae59991c3aff626886cc94b826bca85e2f50b5c9e1c4"
+    },
+    {
+      "bytes": 3732,
+      "crc32c": "E/XXzw==",
+      "path": "profiles/scaled4.json",
+      "sha256": "6523bcb12b25c6adbd38f251dedddd7d0840f4592efc7bf8826a5f9a36cca2a5"
+    },
+    {
+      "bytes": 11515,
+      "crc32c": "GWHkcQ==",
+      "path": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+      "provenance": {
+        "kind": "unchanged_native_result",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_sha256": "566d1325afbb580f0d8485054e6b56e95eec2b96f2a7393a4c3a6a7cb5bff177"
+      },
+      "sha256": "566d1325afbb580f0d8485054e6b56e95eec2b96f2a7393a4c3a6a7cb5bff177"
+    },
+    {
+      "bytes": 806,
+      "crc32c": "bx5ruw==",
+      "path": "eval_curves/scaled8.csv",
+      "provenance": {
+        "kind": "unchanged_collected_evaluation.csv",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_sha256": "19d50a1454953096509a05961b3d6e7a3e583288de338185e5d74a74c5593390"
+      },
+      "sha256": "19d50a1454953096509a05961b3d6e7a3e583288de338185e5d74a74c5593390"
+    },
+    {
+      "bytes": 3346,
+      "crc32c": "V8W18w==",
+      "path": "health/scaled8.csv",
+      "provenance": {
+        "kind": "unchanged_collected_health.csv",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_sha256": "185a1b7957e6025eea46274010635a905d3719317d831450f0f2c8661ac4903e"
+      },
+      "sha256": "185a1b7957e6025eea46274010635a905d3719317d831450f0f2c8661ac4903e"
+    },
+    {
+      "bytes": 13678,
+      "crc32c": "Ep/V4A==",
+      "path": "scientific_logs/scaled8.log",
+      "provenance": {
+        "kind": "filtered_scientific_lines",
+        "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_log_sha256": "1b40aee120a8e9fccc609fe9cc2c2ac53df9a10664436ff438a07e7f3186d3d1",
+        "worker_paths_redacted": true
+      },
+      "sha256": "57f9f0ba62a18a72eeab324458e22faf41bb5f6a7c8fb1e7997d276d27aad552"
+    },
+    {
+      "bytes": 3734,
+      "crc32c": "mwyTtw==",
+      "path": "profiles/scaled8.json",
+      "sha256": "36a6f469b9de51011d4224853b595c11fecf71e5928bb0d66748c8277bc2f12a"
+    },
+    {
+      "bytes": 8582,
+      "crc32c": "bBU+rw==",
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.json",
+      "provenance": {
+        "kind": "unchanged_native_result",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_sha256": "b8560c64449f735ab4d712e6a4df7aca7910bb856ed44e22bf2bf75768d549eb"
+      },
+      "sha256": "b8560c64449f735ab4d712e6a4df7aca7910bb856ed44e22bf2bf75768d549eb"
+    },
+    {
+      "bytes": 457,
+      "crc32c": "REEpmQ==",
+      "path": "eval_curves/unit_bf16_s7.csv",
+      "provenance": {
+        "kind": "unchanged_collected_evaluation.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_sha256": "95316fe251685246f4efe8f50c5465655e63f11a4d1a7e722d16360e64679489"
+      },
+      "sha256": "95316fe251685246f4efe8f50c5465655e63f11a4d1a7e722d16360e64679489"
+    },
+    {
+      "bytes": 2322,
+      "crc32c": "6g7KZA==",
+      "path": "health/unit_bf16_s7.csv",
+      "provenance": {
+        "kind": "unchanged_collected_health.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_sha256": "b886521fb2574d07991a35cb252e64169e5dff9265dca5a8f524f6f63ba79e1e"
+      },
+      "sha256": "b886521fb2574d07991a35cb252e64169e5dff9265dca5a8f524f6f63ba79e1e"
+    },
+    {
+      "bytes": 7578,
+      "crc32c": "brmIDQ==",
+      "path": "scientific_logs/unit_bf16_s7.log",
+      "provenance": {
+        "kind": "filtered_scientific_lines",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_log_sha256": "d074b5d5c5cbb56db594af1751c8776c3ff31bead3acbdc79c1af0696e859e52",
+        "worker_paths_redacted": true
+      },
+      "sha256": "ae7183f19ce5ef485bd3f7c7e720c6a1ab5c9a460a9764ab7f23fab7c614a6e5"
+    },
+    {
+      "bytes": 3808,
+      "crc32c": "eBlgMw==",
+      "path": "profiles/unit_bf16_s7.json",
+      "sha256": "c2bf5b8a3325ba31699f4def296f52c929d70cf1337c334200e8718b36d9ea4c"
+    },
+    {
+      "bytes": 16519,
+      "crc32c": "NET/3g==",
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.json",
+      "provenance": {
+        "kind": "unchanged_native_result",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_sha256": "d396be378862680c3c53f90e0d7b29f77fbf178d1364d5af364b9a8f91202428"
+      },
+      "sha256": "d396be378862680c3c53f90e0d7b29f77fbf178d1364d5af364b9a8f91202428"
+    },
+    {
+      "bytes": 1378,
+      "crc32c": "+GZjPg==",
+      "path": "eval_curves/unit_bf16_s99.csv",
+      "provenance": {
+        "kind": "unchanged_collected_evaluation.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_sha256": "4e63e21807f173088d3ac8d825661ff514d94f6369f387cc2986a2fbb6584682"
+      },
+      "sha256": "4e63e21807f173088d3ac8d825661ff514d94f6369f387cc2986a2fbb6584682"
+    },
+    {
+      "bytes": 5391,
+      "crc32c": "1DBPCA==",
+      "path": "health/unit_bf16_s99.csv",
+      "provenance": {
+        "kind": "unchanged_collected_health.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_sha256": "55e12010c5aa2549f0ef78b9667ab3fb5db707909a02c7c6881089cd0cab6b62"
+      },
+      "sha256": "55e12010c5aa2549f0ef78b9667ab3fb5db707909a02c7c6881089cd0cab6b62"
+    },
+    {
+      "bytes": 18993,
+      "crc32c": "nWHwyw==",
+      "path": "scientific_logs/unit_bf16_s99.log",
+      "provenance": {
+        "kind": "filtered_scientific_lines",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+        "source_log_sha256": "b0099dedbf217bf7de5148002026967a51729a885bcfe9b00dacc9ed27bb3db1",
+        "worker_paths_redacted": true
+      },
+      "sha256": "fe32e36ced5e421023ca38bb2e3eed64dda25b98dc15a12c1054192d85185508"
+    },
+    {
+      "bytes": 3836,
+      "crc32c": "uLLs3Q==",
+      "path": "profiles/unit_bf16_s99.json",
+      "sha256": "fef847b3142666a22cb0047760c751af615a349673ea4ec761c663b24dcc24e2"
+    },
+    {
+      "bytes": 3803,
+      "crc32c": "oE0hAw==",
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+      "provenance": {
+        "kind": "unchanged_native_result",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_sha256": "c8024707c0bcc82ac8f0d43e78fbc458c4f096f1cbbb97b04a711d161980872a"
+      },
+      "sha256": "c8024707c0bcc82ac8f0d43e78fbc458c4f096f1cbbb97b04a711d161980872a"
+    },
+    {
+      "bytes": 26,
+      "crc32c": "8X6IOQ==",
+      "path": "eval_curves/unit4.csv",
+      "provenance": {
+        "kind": "unchanged_collected_evaluation.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_sha256": "94f2a495c39b5e11638210689d7887eb6d2b469af20e9c62771aa34b169cc87b"
+      },
+      "sha256": "94f2a495c39b5e11638210689d7887eb6d2b469af20e9c62771aa34b169cc87b"
+    },
+    {
+      "bytes": 169,
+      "crc32c": "jSnvUA==",
+      "path": "health/unit4.csv",
+      "provenance": {
+        "kind": "unchanged_collected_health.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_sha256": "2889cc03404cfc1fd2861815272c443eebd9edbf99ab98987b7f3641132c353b"
+      },
+      "sha256": "2889cc03404cfc1fd2861815272c443eebd9edbf99ab98987b7f3641132c353b"
+    },
+    {
+      "bytes": 2088,
+      "crc32c": "6uiZLw==",
+      "path": "scientific_logs/unit4.log",
+      "provenance": {
+        "kind": "filtered_scientific_lines",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+        "source_log_sha256": "e3c6f336ce2fc0ad07e0b99ed6f86a1bccec5db39f2734640ee461900aa9cf7e",
+        "worker_paths_redacted": true
+      },
+      "sha256": "59a59df95671799b19ea68e5ff475b3ff93fc4f3f364bf9d3c7521788d80860a"
+    },
+    {
+      "bytes": 3735,
+      "crc32c": "NDuhyQ==",
+      "path": "profiles/unit4.json",
+      "sha256": "a0f9da3bad5629261adb84bb6ed497f24953cb05ff6693523ce0b2ff9300fdca"
+    },
+    {
+      "bytes": 3799,
+      "crc32c": "aIGbpg==",
+      "path": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+      "provenance": {
+        "kind": "unchanged_native_result",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_sha256": "c593f530ac19449f8adb4464c72383b51048ab95f629f2af5a3edff8ee2a6cf8"
+      },
+      "sha256": "c593f530ac19449f8adb4464c72383b51048ab95f629f2af5a3edff8ee2a6cf8"
+    },
+    {
+      "bytes": 26,
+      "crc32c": "8X6IOQ==",
+      "path": "eval_curves/unit8.csv",
+      "provenance": {
+        "kind": "unchanged_collected_evaluation.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_sha256": "94f2a495c39b5e11638210689d7887eb6d2b469af20e9c62771aa34b169cc87b"
+      },
+      "sha256": "94f2a495c39b5e11638210689d7887eb6d2b469af20e9c62771aa34b169cc87b"
+    },
+    {
+      "bytes": 164,
+      "crc32c": "rWRk1A==",
+      "path": "health/unit8.csv",
+      "provenance": {
+        "kind": "unchanged_collected_health.csv",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_sha256": "5e0c21319fca760b643b2d6051de0f838185f47f15fbad1884a9f2fdd3efd1c0"
+      },
+      "sha256": "5e0c21319fca760b643b2d6051de0f838185f47f15fbad1884a9f2fdd3efd1c0"
+    },
+    {
+      "bytes": 2091,
+      "crc32c": "6phL6Q==",
+      "path": "scientific_logs/unit8.log",
+      "provenance": {
+        "kind": "filtered_scientific_lines",
+        "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+        "source_log_sha256": "e2be31c2ddd18cbdac8f94f416916b8babad5c7c67198ddd68fa388965441c39",
+        "worker_paths_redacted": true
+      },
+      "sha256": "0a574595492ac05acbf5897f2d6dbf11f3236321a6073c844ef5d1fbb954f01e"
+    },
+    {
+      "bytes": 3736,
+      "crc32c": "jQAnpQ==",
+      "path": "profiles/unit8.json",
+      "sha256": "a9373d5ae4165e3221a7218c21957603ce7a242242cf23b399e79189738bb209"
+    },
+    {
+      "bytes": 4014,
+      "crc32c": "5EydAw==",
+      "path": "eval_curves/all_runs.csv",
+      "sha256": "b2581bdaa4fa8866b95ced9b000749f42e732c0fc0a76a28ce5d57f15b4577f0"
+    },
+    {
+      "bytes": 5635,
+      "crc32c": "VDlFLw==",
+      "path": "RESULTS.json",
+      "sha256": "b35bf886ffc7f4b4c32e29b41370da1955799b1a0b9f060d6c01a64c78e57820"
+    },
+    {
+      "bytes": 1106,
+      "crc32c": "DK/sEA==",
+      "path": "RESULTS.md",
+      "sha256": "1ec5ed8749fac40ab38f27090262c3ae26b7f95d59a7d3ad6e216e72baf46e98"
+    },
+    {
+      "bytes": 2713,
+      "crc32c": "UpiQUg==",
+      "path": "CHECKPOINTS.json",
+      "sha256": "d49659db9f62c33527506f496167d386e9bc37a5f3c13d380de5a72ad8f03362"
+    },
+    {
+      "bytes": 2344,
+      "crc32c": "pootRA==",
+      "path": "EXPORTS.json",
+      "sha256": "32d75b67576654d8f7819e2fe0448b99fb11f6998595398ce4617f1cdc5236c0"
+    },
+    {
+      "bytes": 2866,
+      "crc32c": "14OlYA==",
+      "path": "README.md",
+      "sha256": "cf401e18c4e72e910d06d11a05100cfe1e6929f551aaeed53865f9b6d94f35f0"
+    },
+    {
+      "bytes": 2265,
+      "crc32c": "eADd7w==",
+      "path": "LOADING.md",
+      "sha256": "d907c380e7572ca0de677c16cf765dfbeadade4bd4845f3c26d44b637266180a"
+    }
+  ],
+  "native_json_bytes_preserved": true,
+  "operational_metadata_included": false,
+  "paired_evaluation_points": 68,
+  "schema": "helm.c42b.public-small-manifest.v1",
+  "scope": "All listed small files; this manifest itself is excluded."
+}

cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json

@@ -0,0 +1,622 @@
+{
+ "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32": {
+  "init": "scaled",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c42b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 2,
+  "seed": 1234,
+  "steps": 61000,
+  "eval": 2000,
+  "tokens": 1998848000,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 1,
+  "compile": false,
+  "world": 8,
+  "R_init": 1.0,
+  "u_init": 1.0,
+  "radii": {
+   "layers.0.attn_norm": 1.0,
+   "layers.0.attn.kv_norm": 1.0,
+   "layers.0.ffn_norm": 1.0,
+   "layers.1.attn_norm": 1.0,
+   "layers.1.attn.kv_norm": 1.0,
+   "layers.1.ffn_norm": 1.0,
+   "layers.2.attn_norm": 1.0,
+   "layers.2.attn.kv_norm": 1.0,
+   "layers.2.ffn_norm": 1.0,
+   "layers.3.attn_norm": 1.0,
+   "layers.3.attn.kv_norm": 1.0,
+   "layers.3.ffn_norm": 1.0,
+   "layers.4.attn_norm": 1.0,
+   "layers.4.attn.kv_norm": 1.0,
+   "layers.4.ffn_norm": 1.0,
+   "layers.5.attn_norm": 1.0,
+   "layers.5.attn.kv_norm": 1.0,
+   "layers.5.ffn_norm": 1.0,
+   "layers.6.attn_norm": 1.0,
+   "layers.6.attn.kv_norm": 1.0,
+   "layers.6.ffn_norm": 1.0,
+   "layers.7.attn_norm": 1.0,
+   "layers.7.attn.kv_norm": 1.0,
+   "layers.7.ffn_norm": 1.0,
+   "layers.8.attn_norm": 1.0,
+   "layers.8.attn.kv_norm": 1.0,
+   "layers.8.ffn_norm": 1.0,
+   "layers.9.attn_norm": 1.0,
+   "layers.9.attn.kv_norm": 1.0,
+   "layers.9.ffn_norm": 1.0,
+   "layers.10.attn_norm": 1.0,
+   "layers.10.attn.kv_norm": 1.0,
+   "layers.10.ffn_norm": 1.0,
+   "layers.11.attn_norm": 1.0,
+   "layers.11.attn.kv_norm": 1.0,
+   "layers.11.ffn_norm": 1.0,
+   "layers.12.attn_norm": 1.0,
+   "layers.12.attn.kv_norm": 1.0,
+   "layers.12.ffn_norm": 1.0,
+   "layers.13.attn_norm": 1.0,
+   "layers.13.attn.kv_norm": 1.0,
+   "layers.13.ffn_norm": 1.0,
+   "layers.14.attn_norm": 1.0,
+   "layers.14.attn.kv_norm": 1.0,
+   "layers.14.ffn_norm": 1.0,
+   "layers.15.attn_norm": 1.0,
+   "layers.15.attn.kv_norm": 1.0,
+   "layers.15.ffn_norm": 1.0,
+   "norm": 1.0
+  },
+  "eval_touches_state": false,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 305.3,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": false,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [
+   [
+    2000,
+    202.94915885729733
+   ],
+   [
+    4000,
+    423.6878515622484
+   ],
+   [
+    6000,
+    1489.0393986409879
+   ],
+   [
+    8000,
+    2921.744164318744
+   ],
+   [
+    10000,
+    2477.426440174055
+   ],
+   [
+    12000,
+    1390.7013621661292
+   ],
+   [
+    14000,
+    819.5750432927489
+   ],
+   [
+    16000,
+    1377.5248366693936
+   ],
+   [
+    18000,
+    1283.73413189331
+   ]
+  ],
+  "ood_traj": [
+   [
+    2000,
+    727.6774965555242
+   ],
+   [
+    4000,
+    1275.309291092224
+   ],
+   [
+    6000,
+    2830.876521664318
+   ],
+   [
+    8000,
+    6316.541965586585
+   ],
+   [
+    10000,
+    4583.318111351122
+   ],
+   [
+    12000,
+    3173.9007569039645
+   ],
+   [
+    14000,
+    2152.868470762587
+   ],
+   [
+    16000,
+    2871.021391179499
+   ],
+   [
+    18000,
+    2968.7661825600626
+   ]
+  ],
+  "health": [
+   [
+    1,
+    500,
+    9.7027,
+    0.6383,
+    4.5736,
+    0.096
+   ],
+   [
+    501,
+    1000,
+    6.4858,
+    0.9984,
+    2.8632,
+    0.4
+   ],
+   [
+    1001,
+    1500,
+    5.772,
+    0.9442,
+    2.5858,
+    0.35
+   ],
+   [
+    1501,
+    2000,
+    5.4105,
+    0.826,
+    12.2292,
+    0.138
+   ],
+   [
+    2001,
+    2500,
+    5.6805,
+    511.6543,
+    45749.1328,
+    0.962
+   ],
+   [
+    2501,
+    3000,
+    5.9166,
+    176220.096,
+    74396832.0,
+    1.0
+   ],
+   [
+    3001,
+    3500,
+    6.0195,
+    1298834.048,
+    402409728.0,
+    1.0
+   ],
+   [
+    3501,
+    4000,
+    6.5126,
+    52333150.208,
+    3462804224.0,
+    1.0
+   ],
+   [
+    4001,
+    4500,
+    6.8108,
+    97818525.696,
+    38629756928.0,
+    1.0
+   ],
+   [
+    4501,
+    5000,
+    6.4227,
+    542141.824,
+    67260152.0,
+    1.0
+   ],
+   [
+    5001,
+    5500,
+    6.3261,
+    475369.824,
+    37219236.0,
+    1.0
+   ],
+   [
+    5501,
+    6000,
+    6.5516,
+    76535504.896,
+    10348393472.0,
+    1.0
+   ],
+   [
+    6001,
+    6500,
+    6.7598,
+    15476954.112,
+    2227032576.0,
+    1.0
+   ],
+   [
+    6501,
+    7000,
+    6.2951,
+    19520233.472,
+    4598260224.0,
+    1.0
+   ],
+   [
+    7001,
+    7500,
+    6.2324,
+    401158111.232,
+    78418305024.0,
+    1.0
+   ],
+   [
+    7501,
+    8000,
+    7.0267,
+    142769815.552,
+    21067493376.0,
+    1.0
+   ],
+   [
+    8001,
+    8500,
+    7.8348,
+    44360120926.208,
+    21826317058048.0,
+    1.0
+   ],
+   [
+    8501,
+    9000,
+    7.5718,
+    4380696.064,
+    403556960.0,
+    1.0
+   ],
+   [
+    9001,
+    9500,
+    7.5329,
+    6421007.36,
+    2302962944.0,
+    1.0
+   ],
+   [
+    9501,
+    10000,
+    7.7485,
+    1609620914.176,
+    104182497280.0,
+    1.0
+   ],
+   [
+    10001,
+    10500,
+    7.8401,
+    9454984626.176,
+    3724470124544.0,
+    1.0
+   ],
+   [
+    10501,
+    11000,
+    7.5425,
+    1184740016.128,
+    467213385728.0,
+    1.0
+   ],
+   [
+    11001,
+    11500,
+    7.2397,
+    192504086.528,
+    29852839936.0,
+    1.0
+   ],
+   [
+    11501,
+    12000,
+    7.2025,
+    981352120.32,
+    118586875904.0,
+    1.0
+   ],
+   [
+    12001,
+    12500,
+    7.0402,
+    44174032.896,
+    8867602432.0,
+    1.0
+   ],
+   [
+    12501,
+    13000,
+    6.8783,
+    1848472.064,
+    434353472.0,
+    1.0
+   ],
+   [
+    13001,
+    13500,
+    6.946,
+    621718.016,
+    103816272.0,
+    1.0
+   ],
+   [
+    13501,
+    14000,
+    6.8234,
+    5290834.944,
+    960473664.0,
+    1.0
+   ],
+   [
+    14001,
+    14500,
+    6.5601,
+    1377414.912,
+    465742496.0,
+    1.0
+   ],
+   [
+    14501,
+    15000,
+    6.4952,
+    11370491805.696,
+    5675421270016.0,
+    1.0
+   ],
+   [
+    15001,
+    15500,
+    6.9291,
+    18377663840.256,
+    3771178418176.0,
+    1.0
+   ],
+   [
+    15501,
+    16000,
+    7.0357,
+    7124707035840.512,
+    3476155403337728.0,
+    1.0
+   ],
+   [
+    16001,
+    16500,
+    7.2548,
+    668064915914.752,
+    253003755945984.0,
+    1.0
+   ],
+   [
+    16501,
+    17000,
+    7.2793,
+    17794508783.616,
+    3162229178368.0,
+    1.0
+   ],
+   [
+    17001,
+    17500,
+    7.3164,
+    2411633311.744,
+    1038175633408.0,
+    1.0
+   ],
+   [
+    17501,
+    18000,
+    7.1763,
+    3796626571.264,
+    923216904192.0,
+    1.0
+   ],
+   [
+    18001,
+    18500,
+    7.1747,
+    6283812405.248,
+    2384119988224.0,
+    1.0
+   ],
+   [
+    18501,
+    19000,
+    6.9052,
+    9985686.528,
+    2451557120.0,
+    1.0
+   ],
+   [
+    19001,
+    19500,
+    6.514,
+    341344.224,
+    33471126.0,
+    1.0
+   ],
+   [
+    19501,
+    19600,
+    6.3348,
+    26860.6017,
+    1098950.125,
+    1.0
+   ]
+  ],
+  "status": "nonfinite",
+  "nonfinite_step": 19576,
+  "nonfinite_seen_at": 19600,
+  "prenan_resume": "/mnt/local/helm-lp2-a1-80803345/data/prenan_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.pt",
+  "prenan_step": 19500,
+  "ppl": NaN,
+  "ood_ppl": NaN,
+  "trained": false,
+  "ckpt": null
+ }
+}

cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json

@@ -0,0 +1,982 @@
+{
+ "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32": {
+  "init": "scaled",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c42b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 2,
+  "seed": 1234,
+  "steps": 61000,
+  "eval": 2000,
+  "tokens": 1998848000,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 1,
+  "compile": false,
+  "world": 8,
+  "R_init": 1.0,
+  "u_init": 1.0,
+  "radii": {
+   "layers.0.attn_norm": 1.0,
+   "layers.0.attn.kv_norm": 1.0,
+   "layers.0.ffn_norm": 1.0,
+   "layers.1.attn_norm": 1.0,
+   "layers.1.attn.kv_norm": 1.0,
+   "layers.1.ffn_norm": 1.0,
+   "layers.2.attn_norm": 1.0,
+   "layers.2.attn.kv_norm": 1.0,
+   "layers.2.ffn_norm": 1.0,
+   "layers.3.attn_norm": 1.0,
+   "layers.3.attn.kv_norm": 1.0,
+   "layers.3.ffn_norm": 1.0,
+   "layers.4.attn_norm": 1.0,
+   "layers.4.attn.kv_norm": 1.0,
+   "layers.4.ffn_norm": 1.0,
+   "layers.5.attn_norm": 1.0,
+   "layers.5.attn.kv_norm": 1.0,
+   "layers.5.ffn_norm": 1.0,
+   "layers.6.attn_norm": 1.0,
+   "layers.6.attn.kv_norm": 1.0,
+   "layers.6.ffn_norm": 1.0,
+   "layers.7.attn_norm": 1.0,
+   "layers.7.attn.kv_norm": 1.0,
+   "layers.7.ffn_norm": 1.0,
+   "layers.8.attn_norm": 1.0,
+   "layers.8.attn.kv_norm": 1.0,
+   "layers.8.ffn_norm": 1.0,
+   "layers.9.attn_norm": 1.0,
+   "layers.9.attn.kv_norm": 1.0,
+   "layers.9.ffn_norm": 1.0,
+   "layers.10.attn_norm": 1.0,
+   "layers.10.attn.kv_norm": 1.0,
+   "layers.10.ffn_norm": 1.0,
+   "layers.11.attn_norm": 1.0,
+   "layers.11.attn.kv_norm": 1.0,
+   "layers.11.ffn_norm": 1.0,
+   "layers.12.attn_norm": 1.0,
+   "layers.12.attn.kv_norm": 1.0,
+   "layers.12.ffn_norm": 1.0,
+   "layers.13.attn_norm": 1.0,
+   "layers.13.attn.kv_norm": 1.0,
+   "layers.13.ffn_norm": 1.0,
+   "layers.14.attn_norm": 1.0,
+   "layers.14.attn.kv_norm": 1.0,
+   "layers.14.ffn_norm": 1.0,
+   "layers.15.attn_norm": 1.0,
+   "layers.15.attn.kv_norm": 1.0,
+   "layers.15.ffn_norm": 1.0,
+   "norm": 1.0
+  },
+  "eval_touches_state": false,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 598.6,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": false,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [
+   [
+    2000,
+    174.7353397930056
+   ],
+   [
+    4000,
+    96.74375678060908
+   ],
+   [
+    6000,
+    71.96221656823293
+   ],
+   [
+    8000,
+    63.241492720062254
+   ],
+   [
+    10000,
+    58.61788591360284
+   ],
+   [
+    12000,
+    184.65813325491646
+   ],
+   [
+    14000,
+    137.8900502438572
+   ],
+   [
+    16000,
+    169.85524170158178
+   ],
+   [
+    18000,
+    100.05913976244142
+   ],
+   [
+    20000,
+    133.7793884974217
+   ],
+   [
+    22000,
+    88.13256767692252
+   ],
+   [
+    24000,
+    109.31591745755064
+   ],
+   [
+    26000,
+    103.20969940209199
+   ],
+   [
+    28000,
+    78.43288412727317
+   ],
+   [
+    30000,
+    74.88167041491353
+   ],
+   [
+    32000,
+    70.9270138520994
+   ],
+   [
+    34000,
+    72.69512954617576
+   ],
+   [
+    36000,
+    64.18778187722458
+   ]
+  ],
+  "ood_traj": [
+   [
+    2000,
+    610.8903868767173
+   ],
+   [
+    4000,
+    278.5533283405427
+   ],
+   [
+    6000,
+    210.89463260614258
+   ],
+   [
+    8000,
+    181.14849829538917
+   ],
+   [
+    10000,
+    178.9281156617443
+   ],
+   [
+    12000,
+    604.537755576149
+   ],
+   [
+    14000,
+    454.58166719125
+   ],
+   [
+    16000,
+    513.6227455341899
+   ],
+   [
+    18000,
+    253.7676409450834
+   ],
+   [
+    20000,
+    429.05551043005937
+   ],
+   [
+    22000,
+    227.04865686304538
+   ],
+   [
+    24000,
+    302.6282972733793
+   ],
+   [
+    26000,
+    249.42930528236712
+   ],
+   [
+    28000,
+    189.27421096393644
+   ],
+   [
+    30000,
+    195.60225877785737
+   ],
+   [
+    32000,
+    182.039545371768
+   ],
+   [
+    34000,
+    194.83785455268676
+   ],
+   [
+    36000,
+    178.46032642706157
+   ]
+  ],
+  "health": [
+   [
+    1,
+    500,
+    9.7014,
+    0.6114,
+    3.1427,
+    0.082
+   ],
+   [
+    501,
+    1000,
+    6.4802,
+    0.9902,
+    2.7411,
+    0.384
+   ],
+   [
+    1001,
+    1500,
+    5.6738,
+    0.9769,
+    2.2324,
+    0.378
+   ],
+   [
+    1501,
+    2000,
+    5.2693,
+    0.7479,
+    1.7225,
+    0.1
+   ],
+   [
+    2001,
+    2500,
+    5.0838,
+    0.6596,
+    1.2531,
+    0.02
+   ],
+   [
+    2501,
+    3000,
+    4.9109,
+    0.6128,
+    0.9605,
+    0.0
+   ],
+   [
+    3001,
+    3500,
+    4.7996,
+    0.5739,
+    0.9984,
+    0.0
+   ],
+   [
+    3501,
+    4000,
+    4.685,
+    0.5463,
+    1.3612,
+    0.004
+   ],
+   [
+    4001,
+    4500,
+    4.5061,
+    0.5196,
+    0.7655,
+    0.0
+   ],
+   [
+    4501,
+    5000,
+    4.3982,
+    0.5005,
+    1.1143,
+    0.002
+   ],
+   [
+    5001,
+    5500,
+    4.3312,
+    0.4785,
+    0.9606,
+    0.0
+   ],
+   [
+    5501,
+    6000,
+    4.2797,
+    0.465,
+    0.8798,
+    0.0
+   ],
+   [
+    6001,
+    6500,
+    4.251,
+    0.4564,
+    1.5455,
+    0.002
+   ],
+   [
+    6501,
+    7000,
+    4.1993,
+    0.4535,
+    1.8914,
+    0.002
+   ],
+   [
+    7001,
+    7500,
+    4.1884,
+    0.4514,
+    1.9766,
+    0.002
+   ],
+   [
+    7501,
+    8000,
+    4.1425,
+    0.4524,
+    1.3019,
+    0.002
+   ],
+   [
+    8001,
+    8500,
+    4.116,
+    0.4551,
+    1.0741,
+    0.002
+   ],
+   [
+    8501,
+    9000,
+    4.112,
+    0.473,
+    9.4198,
+    0.004
+   ],
+   [
+    9001,
+    9500,
+    4.0714,
+    0.4581,
+    1.1425,
+    0.004
+   ],
+   [
+    9501,
+    10000,
+    4.0591,
+    0.4666,
+    2.6143,
+    0.006
+   ],
+   [
+    10001,
+    10500,
+    4.0487,
+    0.5566,
+    32.4937,
+    0.016
+   ],
+   [
+    10501,
+    11000,
+    4.046,
+    1.5361,
+    308.9725,
+    0.102
+   ],
+   [
+    11001,
+    11500,
+    4.1673,
+    10.7338,
+    1052.016,
+    0.528
+   ],
+   [
+    11501,
+    12000,
+    5.0044,
+    461.9405,
+    41910.043,
+    1.0
+   ],
+   [
+    12001,
+    12500,
+    5.3022,
+    4680.4765,
+    841680.4375,
+    0.998
+   ],
+   [
+    12501,
+    13000,
+    5.9042,
+    1014261.056,
+    270517312.0,
+    1.0
+   ],
+   [
+    13001,
+    13500,
+    5.7568,
+    9639.093,
+    1885640.625,
+    0.994
+   ],
+   [
+    13501,
+    14000,
+    5.1947,
+    1278.0512,
+    536246.0625,
+    0.97
+   ],
+   [
+    14001,
+    14500,
+    4.821,
+    191.1503,
+    15655.7461,
+    0.998
+   ],
+   [
+    14501,
+    15000,
+    4.665,
+    611.5712,
+    38539.7383,
+    1.0
+   ],
+   [
+    15001,
+    15500,
+    4.6244,
+    851.9009,
+    19325.9492,
+    1.0
+   ],
+   [
+    15501,
+    16000,
+    5.1716,
+    220322.144,
+    69922816.0,
+    1.0
+   ],
+   [
+    16001,
+    16500,
+    5.0968,
+    45862.9,
+    2904747.0,
+    1.0
+   ],
+   [
+    16501,
+    17000,
+    4.6144,
+    2933.3897,
+    565594.6875,
+    1.0
+   ],
+   [
+    17001,
+    17500,
+    4.5892,
+    1432.0439,
+    154115.0938,
+    1.0
+   ],
+   [
+    17501,
+    18000,
+    4.5908,
+    1931.2683,
+    117779.0391,
+    1.0
+   ],
+   [
+    18001,
+    18500,
+    4.6279,
+    7236.754,
+    314153.25,
+    1.0
+   ],
+   [
+    18501,
+    19000,
+    4.5665,
+    7100.8715,
+    991551.6875,
+    1.0
+   ],
+   [
+    19001,
+    19500,
+    4.6523,
+    4891.302,
+    240722.5312,
+    1.0
+   ],
+   [
+    19501,
+    20000,
+    4.6961,
+    26726.336,
+    3621682.25,
+    1.0
+   ],
+   [
+    20001,
+    20500,
+    4.592,
+    2473.4257,
+    126145.2734,
+    1.0
+   ],
+   [
+    20501,
+    21000,
+    4.5865,
+    7740.638,
+    278614.9062,
+    1.0
+   ],
+   [
+    21001,
+    21500,
+    4.7668,
+    96743.416,
+    11756970.0,
+    1.0
+   ],
+   [
+    21501,
+    22000,
+    4.5791,
+    5707.7255,
+    316491.7188,
+    1.0
+   ],
+   [
+    22001,
+    22500,
+    4.504,
+    17389.226,
+    2030029.0,
+    1.0
+   ],
+   [
+    22501,
+    23000,
+    4.4457,
+    6945.707,
+    1344003.75,
+    1.0
+   ],
+   [
+    23001,
+    23500,
+    4.6888,
+    98770.912,
+    42497904.0,
+    1.0
+   ],
+   [
+    23501,
+    24000,
+    4.7402,
+    70396.968,
+    11657271.0,
+    1.0
+   ],
+   [
+    24001,
+    24500,
+    4.7429,
+    67652.272,
+    9392578.0,
+    1.0
+   ],
+   [
+    24501,
+    25000,
+    4.6507,
+    37201.272,
+    2190922.5,
+    1.0
+   ],
+   [
+    25001,
+    25500,
+    4.6586,
+    137704.464,
+    9793316.0,
+    1.0
+   ],
+   [
+    25501,
+    26000,
+    4.59,
+    44320.172,
+    7772814.0,
+    1.0
+   ],
+   [
+    26001,
+    26500,
+    4.4964,
+    8890.997,
+    737653.0,
+    1.0
+   ],
+   [
+    26501,
+    27000,
+    4.5432,
+    15652.788,
+    3759203.25,
+    1.0
+   ],
+   [
+    27001,
+    27500,
+    4.5528,
+    16914.748,
+    1196604.375,
+    1.0
+   ],
+   [
+    27501,
+    28000,
+    4.4514,
+    11789.215,
+    2854084.0,
+    1.0
+   ],
+   [
+    28001,
+    28500,
+    4.3895,
+    1443.7495,
+    95537.0391,
+    1.0
+   ],
+   [
+    28501,
+    29000,
+    4.4212,
+    1729.4299,
+    85023.1328,
+    1.0
+   ],
+   [
+    29001,
+    29500,
+    4.3516,
+    394.4305,
+    33271.6328,
+    1.0
+   ],
+   [
+    29501,
+    30000,
+    4.3481,
+    685.3107,
+    41030.5547,
+    0.998
+   ],
+   [
+    30001,
+    30500,
+    4.3093,
+    355.514,
+    61418.9766,
+    0.99
+   ],
+   [
+    30501,
+    31000,
+    4.3154,
+    550.7336,
+    178044.5,
+    0.992
+   ],
+   [
+    31001,
+    31500,
+    4.2759,
+    244.7275,
+    14119.3076,
+    0.978
+   ],
+   [
+    31501,
+    32000,
+    4.264,
+    960.2903,
+    178336.3438,
+    0.992
+   ],
+   [
+    32001,
+    32500,
+    4.2232,
+    2247.0422,
+    881971.25,
+    0.996
+   ],
+   [
+    32501,
+    33000,
+    4.2433,
+    5435.5505,
+    709869.625,
+    1.0
+   ],
+   [
+    33001,
+    33500,
+    4.2045,
+    17925.334,
+    2200165.25,
+    1.0
+   ],
+   [
+    33501,
+    34000,
+    4.2324,
+    650.6419,
+    26031.8379,
+    1.0
+   ],
+   [
+    34001,
+    34500,
+    4.2475,
+    322.4877,
+    28030.752,
+    0.976
+   ],
+   [
+    34501,
+    35000,
+    4.2041,
+    2849.214,
+    1364074.625,
+    0.942
+   ],
+   [
+    35001,
+    35500,
+    4.2382,
+    251.1124,
+    25658.3789,
+    0.984
+   ],
+   [
+    35501,
+    36000,
+    4.2298,
+    121.1901,
+    5135.478,
+    1.0
+   ],
+   [
+    36001,
+    36500,
+    4.2358,
+    1492.3666,
+    140415.7344,
+    1.0
+   ],
+   [
+    36501,
+    37000,
+    4.3222,
+    1445.7544,
+    349021.25,
+    1.0
+   ],
+   [
+    37001,
+    37500,
+    4.1971,
+    325.8608,
+    35151.1172,
+    0.976
+   ],
+   [
+    37501,
+    37800,
+    4.1911,
+    98.3187,
+    5155.6104,
+    0.954
+   ]
+  ],
+  "status": "nonfinite",
+  "nonfinite_step": 37785,
+  "nonfinite_seen_at": 37800,
+  "prenan_resume": "/mnt/local/helm-lp2-a1-d6e68f80/data/prenan_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.pt",
+  "prenan_step": 37500,
+  "ppl": NaN,
+  "ood_ppl": NaN,
+  "trained": false,
+  "ckpt": null
+ }
+}

cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json

@@ -0,0 +1,236 @@
+{
+ "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32": {
+  "init": "unit",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c42b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 2,
+  "seed": 1234,
+  "steps": 61000,
+  "eval": 2000,
+  "tokens": 1998848000,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 1,
+  "compile": false,
+  "world": 8,
+  "R_init": 30.1496,
+  "u_init": 30.1496,
+  "radii": {
+   "layers.0.attn_norm": 30.15,
+   "layers.0.attn.kv_norm": 16.031,
+   "layers.0.ffn_norm": 30.15,
+   "layers.1.attn_norm": 30.15,
+   "layers.1.attn.kv_norm": 16.031,
+   "layers.1.ffn_norm": 30.15,
+   "layers.2.attn_norm": 30.15,
+   "layers.2.attn.kv_norm": 16.031,
+   "layers.2.ffn_norm": 30.15,
+   "layers.3.attn_norm": 30.15,
+   "layers.3.attn.kv_norm": 16.031,
+   "layers.3.ffn_norm": 30.15,
+   "layers.4.attn_norm": 30.15,
+   "layers.4.attn.kv_norm": 16.031,
+   "layers.4.ffn_norm": 30.15,
+   "layers.5.attn_norm": 30.15,
+   "layers.5.attn.kv_norm": 16.031,
+   "layers.5.ffn_norm": 30.15,
+   "layers.6.attn_norm": 30.15,
+   "layers.6.attn.kv_norm": 16.031,
+   "layers.6.ffn_norm": 30.15,
+   "layers.7.attn_norm": 30.15,
+   "layers.7.attn.kv_norm": 16.031,
+   "layers.7.ffn_norm": 30.15,
+   "layers.8.attn_norm": 30.15,
+   "layers.8.attn.kv_norm": 16.031,
+   "layers.8.ffn_norm": 30.15,
+   "layers.9.attn_norm": 30.15,
+   "layers.9.attn.kv_norm": 16.031,
+   "layers.9.ffn_norm": 30.15,
+   "layers.10.attn_norm": 30.15,
+   "layers.10.attn.kv_norm": 16.031,
+   "layers.10.ffn_norm": 30.15,
+   "layers.11.attn_norm": 30.15,
+   "layers.11.attn.kv_norm": 16.031,
+   "layers.11.ffn_norm": 30.15,
+   "layers.12.attn_norm": 30.15,
+   "layers.12.attn.kv_norm": 16.031,
+   "layers.12.ffn_norm": 30.15,
+   "layers.13.attn_norm": 30.15,
+   "layers.13.attn.kv_norm": 16.031,
+   "layers.13.ffn_norm": 30.15,
+   "layers.14.attn_norm": 30.15,
+   "layers.14.attn.kv_norm": 16.031,
+   "layers.14.ffn_norm": 30.15,
+   "layers.15.attn_norm": 30.15,
+   "layers.15.attn.kv_norm": 16.031,
+   "layers.15.ffn_norm": 30.15,
+   "norm": 30.15
+  },
+  "eval_touches_state": null,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 1.7,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": false,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [],
+  "ood_traj": [],
+  "health": [
+   [
+    1,
+    100,
+    12.0917,
+    1.6661621597263521e+18,
+    1.7868244826848231e+19,
+    1.0
+   ]
+  ],
+  "status": "nonfinite",
+  "nonfinite_step": 94,
+  "nonfinite_seen_at": 100,
+  "prenan_resume": null,
+  "prenan_step": null,
+  "ppl": NaN,
+  "ood_ppl": NaN,
+  "trained": false,
+  "ckpt": null
+ }
+}

cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json

@@ -0,0 +1,236 @@
+{
+ "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32": {
+  "init": "unit",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c42b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 2,
+  "seed": 1234,
+  "steps": 61000,
+  "eval": 2000,
+  "tokens": 1998848000,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 1,
+  "compile": false,
+  "world": 8,
+  "R_init": 30.1496,
+  "u_init": 30.1496,
+  "radii": {
+   "layers.0.attn_norm": 30.15,
+   "layers.0.attn.kv_norm": 16.031,
+   "layers.0.ffn_norm": 30.15,
+   "layers.1.attn_norm": 30.15,
+   "layers.1.attn.kv_norm": 16.031,
+   "layers.1.ffn_norm": 30.15,
+   "layers.2.attn_norm": 30.15,
+   "layers.2.attn.kv_norm": 16.031,
+   "layers.2.ffn_norm": 30.15,
+   "layers.3.attn_norm": 30.15,
+   "layers.3.attn.kv_norm": 16.031,
+   "layers.3.ffn_norm": 30.15,
+   "layers.4.attn_norm": 30.15,
+   "layers.4.attn.kv_norm": 16.031,
+   "layers.4.ffn_norm": 30.15,
+   "layers.5.attn_norm": 30.15,
+   "layers.5.attn.kv_norm": 16.031,
+   "layers.5.ffn_norm": 30.15,
+   "layers.6.attn_norm": 30.15,
+   "layers.6.attn.kv_norm": 16.031,
+   "layers.6.ffn_norm": 30.15,
+   "layers.7.attn_norm": 30.15,
+   "layers.7.attn.kv_norm": 16.031,
+   "layers.7.ffn_norm": 30.15,
+   "layers.8.attn_norm": 30.15,
+   "layers.8.attn.kv_norm": 16.031,
+   "layers.8.ffn_norm": 30.15,
+   "layers.9.attn_norm": 30.15,
+   "layers.9.attn.kv_norm": 16.031,
+   "layers.9.ffn_norm": 30.15,
+   "layers.10.attn_norm": 30.15,
+   "layers.10.attn.kv_norm": 16.031,
+   "layers.10.ffn_norm": 30.15,
+   "layers.11.attn_norm": 30.15,
+   "layers.11.attn.kv_norm": 16.031,
+   "layers.11.ffn_norm": 30.15,
+   "layers.12.attn_norm": 30.15,
+   "layers.12.attn.kv_norm": 16.031,
+   "layers.12.ffn_norm": 30.15,
+   "layers.13.attn_norm": 30.15,
+   "layers.13.attn.kv_norm": 16.031,
+   "layers.13.ffn_norm": 30.15,
+   "layers.14.attn_norm": 30.15,
+   "layers.14.attn.kv_norm": 16.031,
+   "layers.14.ffn_norm": 30.15,
+   "layers.15.attn_norm": 30.15,
+   "layers.15.attn.kv_norm": 16.031,
+   "layers.15.ffn_norm": 30.15,
+   "norm": 30.15
+  },
+  "eval_touches_state": null,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 7.4,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": false,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [],
+  "ood_traj": [],
+  "health": [
+   [
+    1,
+    500,
+    12.0859,
+    6.8915436589917096e+16,
+    4.65222941019e+18,
+    1.0
+   ]
+  ],
+  "status": "nonfinite",
+  "nonfinite_step": 470,
+  "nonfinite_seen_at": 500,
+  "prenan_resume": null,
+  "prenan_step": null,
+  "ppl": NaN,
+  "ood_ppl": NaN,
+  "trained": false,
+  "ckpt": null
+ }
+}

cells.d/unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.json

@@ -0,0 +1,654 @@
+{
+ "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8": {
+  "init": "unit",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c42b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 2,
+  "seed": 7,
+  "steps": 61000,
+  "eval": 2000,
+  "tokens": 1998848000,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 1,
+  "compile": false,
+  "world": 8,
+  "R_init": 30.1496,
+  "u_init": 30.1496,
+  "radii": {
+   "layers.0.attn_norm": 30.15,
+   "layers.0.attn.kv_norm": 16.032,
+   "layers.0.ffn_norm": 30.15,
+   "layers.1.attn_norm": 30.15,
+   "layers.1.attn.kv_norm": 16.031,
+   "layers.1.ffn_norm": 30.15,
+   "layers.2.attn_norm": 30.15,
+   "layers.2.attn.kv_norm": 16.031,
+   "layers.2.ffn_norm": 30.15,
+   "layers.3.attn_norm": 30.15,
+   "layers.3.attn.kv_norm": 16.031,
+   "layers.3.ffn_norm": 30.15,
+   "layers.4.attn_norm": 30.15,
+   "layers.4.attn.kv_norm": 16.031,
+   "layers.4.ffn_norm": 30.15,
+   "layers.5.attn_norm": 30.15,
+   "layers.5.attn.kv_norm": 16.031,
+   "layers.5.ffn_norm": 30.15,
+   "layers.6.attn_norm": 30.15,
+   "layers.6.attn.kv_norm": 16.031,
+   "layers.6.ffn_norm": 30.15,
+   "layers.7.attn_norm": 30.15,
+   "layers.7.attn.kv_norm": 16.031,
+   "layers.7.ffn_norm": 30.15,
+   "layers.8.attn_norm": 30.15,
+   "layers.8.attn.kv_norm": 16.031,
+   "layers.8.ffn_norm": 30.15,
+   "layers.9.attn_norm": 30.15,
+   "layers.9.attn.kv_norm": 16.031,
+   "layers.9.ffn_norm": 30.15,
+   "layers.10.attn_norm": 30.15,
+   "layers.10.attn.kv_norm": 16.031,
+   "layers.10.ffn_norm": 30.15,
+   "layers.11.attn_norm": 30.15,
+   "layers.11.attn.kv_norm": 16.031,
+   "layers.11.ffn_norm": 30.15,
+   "layers.12.attn_norm": 30.15,
+   "layers.12.attn.kv_norm": 16.031,
+   "layers.12.ffn_norm": 30.15,
+   "layers.13.attn_norm": 30.15,
+   "layers.13.attn.kv_norm": 16.031,
+   "layers.13.ffn_norm": 30.15,
+   "layers.14.attn_norm": 30.15,
+   "layers.14.attn.kv_norm": 16.031,
+   "layers.14.ffn_norm": 30.15,
+   "layers.15.attn_norm": 30.15,
+   "layers.15.attn.kv_norm": 16.031,
+   "layers.15.ffn_norm": 30.15,
+   "norm": 30.15
+  },
+  "eval_touches_state": false,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 263.5,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": false,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [
+   [
+    2000,
+    3081.6974153686674
+   ],
+   [
+    4000,
+    3725.402220121194
+   ],
+   [
+    6000,
+    6292.427285379758
+   ],
+   [
+    8000,
+    7222.959259311071
+   ],
+   [
+    10000,
+    3740.618074705774
+   ],
+   [
+    12000,
+    2544.7708270565704
+   ],
+   [
+    14000,
+    2546.072196279018
+   ],
+   [
+    16000,
+    2555.342611791562
+   ],
+   [
+    18000,
+    2444.166069195049
+   ],
+   [
+    20000,
+    1971.1374237795667
+   ]
+  ],
+  "ood_traj": [
+   [
+    2000,
+    5205.354479250944
+   ],
+   [
+    4000,
+    6150.853129856762
+   ],
+   [
+    6000,
+    9973.111690775377
+   ],
+   [
+    8000,
+    11965.958219630595
+   ],
+   [
+    10000,
+    7218.157307897914
+   ],
+   [
+    12000,
+    4938.8878274173785
+   ],
+   [
+    14000,
+    4870.624762853716
+   ],
+   [
+    16000,
+    4668.330447319325
+   ],
+   [
+    18000,
+    4608.339614505096
+   ],
+   [
+    20000,
+    3768.617090639044
+   ]
+  ],
+  "health": [
+   [
+    1,
+    500,
+    8.2741,
+    279988968357.888,
+    48261033361408.0,
+    0.79
+   ],
+   [
+    501,
+    1000,
+    7.8433,
+    5975770930171871.0,
+    1.3047096345771377e+18,
+    1.0
+   ],
+   [
+    1001,
+    1500,
+    7.88,
+    2290687066058522.5,
+    7.318069019011973e+17,
+    1.0
+   ],
+   [
+    1501,
+    2000,
+    7.9409,
+    3525511466696311.0,
+    4.866156027487191e+17,
+    1.0
+   ],
+   [
+    2001,
+    2500,
+    7.9134,
+    73541376389677.06,
+    3.347726217510912e+16,
+    1.0
+   ],
+   [
+    2501,
+    3000,
+    7.8792,
+    4096278887661.568,
+    672137216000000.0,
+    1.0
+   ],
+   [
+    3001,
+    3500,
+    7.9836,
+    89853490381193.22,
+    1.4693937818107904e+16,
+    1.0
+   ],
+   [
+    3501,
+    4000,
+    8.1615,
+    18353447522795.52,
+    6111865163743232.0,
+    1.0
+   ],
+   [
+    4001,
+    4500,
+    8.3271,
+    13516694798467.072,
+    3735046905135104.0,
+    1.0
+   ],
+   [
+    4501,
+    5000,
+    8.6039,
+    2092514911715.328,
+    214209128300544.0,
+    1.0
+   ],
+   [
+    5001,
+    5500,
+    8.7694,
+    1035891418071.04,
+    238156959973376.0,
+    1.0
+   ],
+   [
+    5501,
+    6000,
+    8.6709,
+    209231848407.04,
+    44125311205376.0,
+    1.0
+   ],
+   [
+    6001,
+    6500,
+    8.7985,
+    134661242093.568,
+    12161647640576.0,
+    1.0
+   ],
+   [
+    6501,
+    7000,
+    8.928,
+    156913174052.864,
+    35240881946624.0,
+    1.0
+   ],
+   [
+    7001,
+    7500,
+    8.958,
+    1351027328548.864,
+    595335684554752.0,
+    1.0
+   ],
+   [
+    7501,
+    8000,
+    8.8752,
+    46685258514.432,
+    9556080984064.0,
+    1.0
+   ],
+   [
+    8001,
+    8500,
+    8.9012,
+    1064509120708.608,
+    226382458126336.0,
+    1.0
+   ],
+   [
+    8501,
+    9000,
+    8.8973,
+    302028072943.616,
+    146165672706048.0,
+    1.0
+   ],
+   [
+    9001,
+    9500,
+    8.831,
+    12558347010.048,
+    1865004744704.0,
+    1.0
+   ],
+   [
+    9501,
+    10000,
+    8.5071,
+    1132090359.808,
+    304874225664.0,
+    1.0
+   ],
+   [
+    10001,
+    10500,
+    7.9842,
+    17828536.32,
+    2609804032.0,
+    1.0
+   ],
+   [
+    10501,
+    11000,
+    7.8464,
+    2342629.376,
+    242555216.0,
+    1.0
+   ],
+   [
+    11001,
+    11500,
+    7.8282,
+    728957.952,
+    23701910.0,
+    1.0
+   ],
+   [
+    11501,
+    12000,
+    7.8193,
+    586352.832,
+    128272416.0,
+    1.0
+   ],
+   [
+    12001,
+    12500,
+    7.8078,
+    923206.848,
+    135035104.0,
+    1.0
+   ],
+   [
+    12501,
+    13000,
+    7.8106,
+    5223426.048,
+    2275896320.0,
+    1.0
+   ],
+   [
+    13001,
+    13500,
+    7.8208,
+    489591.456,
+    76937816.0,
+    1.0
+   ],
+   [
+    13501,
+    14000,
+    7.8182,
+    4563990.528,
+    1242059392.0,
+    1.0
+   ],
+   [
+    14001,
+    14500,
+    7.8242,
+    1906761.728,
+    546927040.0,
+    1.0
+   ],
+   [
+    14501,
+    15000,
+    7.8063,
+    528363.68,
+    64105964.0,
+    1.0
+   ],
+   [
+    15001,
+    15500,
+    7.8116,
+    2287449.856,
+    744769856.0,
+    1.0
+   ],
+   [
+    15501,
+    16000,
+    7.8181,
+    89636.696,
+    5071824.5,
+    1.0
+   ],
+   [
+    16001,
+    16500,
+    7.8105,
+    302070.016,
+    61053604.0,
+    1.0
+   ],
+   [
+    16501,
+    17000,
+    7.789,
+    213601.184,
+    30829950.0,
+    1.0
+   ],
+   [
+    17001,
+    17500,
+    7.7641,
+    530482.144,
+    144564000.0,
+    1.0
+   ],
+   [
+    17501,
+    18000,
+    7.7736,
+    35051.048,
+    3545721.25,
+    1.0
+   ],
+   [
+    18001,
+    18500,
+    7.7783,
+    79231.312,
+    19613498.0,
+    1.0
+   ],
+   [
+    18501,
+    19000,
+    7.6963,
+    94032.744,
+    8377379.0,
+    1.0
+   ],
+   [
+    19001,
+    19500,
+    7.6139,
+    16569.631,
+    932935.875,
+    1.0
+   ],
+   [
+    19501,
+    20000,
+    7.5621,
+    7749.368,
+    1046033.375,
+    1.0
+   ],
+   [
+    20001,
+    20500,
+    7.4969,
+    6382.7535,
+    902437.75,
+    1.0
+   ],
+   [
+    20501,
+    21000,
+    7.3791,
+    3381.0467,
+    425152.4062,
+    1.0
+   ],
+   [
+    21001,
+    21300,
+    7.3324,
+    2397.1069,
+    89643.5859,
+    1.0
+   ]
+  ],
+  "status": "nonfinite",
+  "nonfinite_step": 21249,
+  "nonfinite_seen_at": 21300,
+  "prenan_resume": "/mnt/local/helm-lp2-a1-454cd478/data/prenan_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.pt",
+  "prenan_step": 21000,
+  "ppl": NaN,
+  "ood_ppl": NaN,
+  "trained": false,
+  "ckpt": null
+ }
+}

cells.d/unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.json

@@ -0,0 +1,1450 @@
+{
+ "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8": {
+  "init": "unit",
+  "gain": null,
+  "prec": "bf16",
+  "dim": 910,
+  "corpus": "c42b",
+  "ood_file": "wikitext_train_tokens.pt",
+  "c": 1.0,
+  "K": 1.0,
+  "lr": 0.001,
+  "acc": 2,
+  "seed": 99,
+  "steps": 61000,
+  "eval": 2000,
+  "tokens": 1998848000,
+  "corpus_fraction": 1.0,
+  "params": 966957719,
+  "mb": 1,
+  "compile": false,
+  "world": 8,
+  "R_init": 30.1496,
+  "u_init": 30.1496,
+  "radii": {
+   "layers.0.attn_norm": 30.15,
+   "layers.0.attn.kv_norm": 16.031,
+   "layers.0.ffn_norm": 30.15,
+   "layers.1.attn_norm": 30.15,
+   "layers.1.attn.kv_norm": 16.031,
+   "layers.1.ffn_norm": 30.15,
+   "layers.2.attn_norm": 30.15,
+   "layers.2.attn.kv_norm": 16.031,
+   "layers.2.ffn_norm": 30.15,
+   "layers.3.attn_norm": 30.15,
+   "layers.3.attn.kv_norm": 16.031,
+   "layers.3.ffn_norm": 30.15,
+   "layers.4.attn_norm": 30.15,
+   "layers.4.attn.kv_norm": 16.031,
+   "layers.4.ffn_norm": 30.15,
+   "layers.5.attn_norm": 30.15,
+   "layers.5.attn.kv_norm": 16.031,
+   "layers.5.ffn_norm": 30.15,
+   "layers.6.attn_norm": 30.15,
+   "layers.6.attn.kv_norm": 16.031,
+   "layers.6.ffn_norm": 30.15,
+   "layers.7.attn_norm": 30.15,
+   "layers.7.attn.kv_norm": 16.031,
+   "layers.7.ffn_norm": 30.15,
+   "layers.8.attn_norm": 30.15,
+   "layers.8.attn.kv_norm": 16.031,
+   "layers.8.ffn_norm": 30.15,
+   "layers.9.attn_norm": 30.15,
+   "layers.9.attn.kv_norm": 16.031,
+   "layers.9.ffn_norm": 30.15,
+   "layers.10.attn_norm": 30.15,
+   "layers.10.attn.kv_norm": 16.031,
+   "layers.10.ffn_norm": 30.15,
+   "layers.11.attn_norm": 30.15,
+   "layers.11.attn.kv_norm": 16.031,
+   "layers.11.ffn_norm": 30.15,
+   "layers.12.attn_norm": 30.15,
+   "layers.12.attn.kv_norm": 16.031,
+   "layers.12.ffn_norm": 30.15,
+   "layers.13.attn_norm": 30.15,
+   "layers.13.attn.kv_norm": 16.031,
+   "layers.13.ffn_norm": 30.15,
+   "layers.14.attn_norm": 30.15,
+   "layers.14.attn.kv_norm": 16.031,
+   "layers.14.ffn_norm": 30.15,
+   "layers.15.attn_norm": 30.15,
+   "layers.15.attn.kv_norm": 16.031,
+   "layers.15.ffn_norm": 30.15,
+   "norm": 30.15
+  },
+  "eval_touches_state": false,
+  "env": {
+   "torch": "2.11.0+cu128",
+   "gpu": "NVIDIA B200",
+   "tf32_matmul": false,
+   "f32_mode": "highest",
+   "python": "3.12.3"
+  },
+  "train_min": 828.5,
+  "sampler": "perm",
+  "sched": "cos",
+  "zloss": 0.0,
+  "train_curv": false,
+  "policy": false,
+  "f32_mode": "highest",
+  "curv_init": [
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0,
+   1.0
+  ],
+  "curv_final": null,
+  "curv_traj": [],
+  "traj": [
+   [
+    2000,
+    3147.398573896505
+   ],
+   [
+    4000,
+    2707.576220762702
+   ],
+   [
+    6000,
+    2575.1728912633325
+   ],
+   [
+    8000,
+    2622.6838109655505
+   ],
+   [
+    10000,
+    2502.9961945528967
+   ],
+   [
+    12000,
+    2555.6282095415113
+   ],
+   [
+    14000,
+    2508.2693488862164
+   ],
+   [
+    16000,
+    2484.914327511012
+   ],
+   [
+    18000,
+    2008.3956480861134
+   ],
+   [
+    20000,
+    1696.8711998096192
+   ],
+   [
+    22000,
+    747.8862353569422
+   ],
+   [
+    24000,
+    225.288699211105
+   ],
+   [
+    26000,
+    130.1189965612892
+   ],
+   [
+    28000,
+    95.71685234767584
+   ],
+   [
+    30000,
+    79.44686243495329
+   ],
+   [
+    32000,
+    72.3136327300778
+   ],
+   [
+    34000,
+    65.51809922995677
+   ],
+   [
+    36000,
+    60.97274644377895
+   ],
+   [
+    38000,
+    57.76574385030062
+   ],
+   [
+    40000,
+    54.881623367777976
+   ],
+   [
+    42000,
+    52.39814165898491
+   ],
+   [
+    44000,
+    50.373673803312734
+   ],
+   [
+    46000,
+    48.942898812706986
+   ],
+   [
+    48000,
+    47.28694778444997
+   ],
+   [
+    50000,
+    46.0191269043681
+   ],
+   [
+    52000,
+    44.916454874907835
+   ],
+   [
+    54000,
+    44.17769508380568
+   ],
+   [
+    56000,
+    43.48701997213694
+   ],
+   [
+    58000,
+    42.92737757988304
+   ],
+   [
+    60000,
+    42.254358168398994
+   ],
+   [
+    61000,
+    42.14531938222052
+   ]
+  ],
+  "ood_traj": [
+   [
+    2000,
+    5423.754433548017
+   ],
+   [
+    4000,
+    4847.083836922551
+   ],
+   [
+    6000,
+    4918.846833759445
+   ],
+   [
+    8000,
+    4914.688281620374
+   ],
+   [
+    10000,
+    4768.259514711109
+   ],
+   [
+    12000,
+    4784.532402448754
+   ],
+   [
+    14000,
+    4704.528607576495
+   ],
+   [
+    16000,
+    4599.509893326248
+   ],
+   [
+    18000,
+    4011.7327462565904
+   ],
+   [
+    20000,
+    3996.417619164261
+   ],
+   [
+    22000,
+    1892.978732361764
+   ],
+   [
+    24000,
+    811.6196224355541
+   ],
+   [
+    26000,
+    420.9505620899285
+   ],
+   [
+    28000,
+    273.45603981582235
+   ],
+   [
+    30000,
+    226.86532284399632
+   ],
+   [
+    32000,
+    208.0626240782148
+   ],
+   [
+    34000,
+    184.70113508475671
+   ],
+   [
+    36000,
+    167.96732176969124
+   ],
+   [
+    38000,
+    159.04639087743783
+   ],
+   [
+    40000,
+    164.44964338345233
+   ],
+   [
+    42000,
+    154.59696545128406
+   ],
+   [
+    44000,
+    153.76564483058056
+   ],
+   [
+    46000,
+    150.02471746335945
+   ],
+   [
+    48000,
+    154.86071208598747
+   ],
+   [
+    50000,
+    159.17233857984223
+   ],
+   [
+    52000,
+    155.1561176257511
+   ],
+   [
+    54000,
+    153.79407774390504
+   ],
+   [
+    56000,
+    156.58927100395852
+   ],
+   [
+    58000,
+    158.2432593686224
+   ],
+   [
+    60000,
+    141.4304989335991
+   ],
+   [
+    61000,
+    150.1752538901007
+   ]
+  ],
+  "health": [
+   [
+    1,
+    500,
+    8.2677,
+    4029060218.88,
+    1159352942592.0,
+    0.786
+   ],
+   [
+    501,
+    1000,
+    7.8374,
+    2.3309431004572156e+16,
+    7.506468687163949e+18,
+    1.0
+   ],
+   [
+    1001,
+    1500,
+    7.8911,
+    895374512256712.8,
+    2.4823254831490662e+17,
+    1.0
+   ],
+   [
+    1501,
+    2000,
+    7.9687,
+    577116679708868.6,
+    2.185821556876247e+17,
+    1.0
+   ],
+   [
+    2001,
+    2500,
+    8.0919,
+    87295339140218.88,
+    3.83244269739049e+16,
+    1.0
+   ],
+   [
+    2501,
+    3000,
+    7.968,
+    39968789562916.87,
+    1.9767367936507904e+16,
+    1.0
+   ],
+   [
+    3001,
+    3500,
+    7.9204,
+    200710566182.912,
+    38664566472704.0,
+    1.0
+   ],
+   [
+    3501,
+    4000,
+    7.8349,
+    358843416.576,
+    27402854400.0,
+    1.0
+   ],
+   [
+    4001,
+    4500,
+    7.8499,
+    138364409872.384,
+    50494122754048.0,
+    1.0
+   ],
+   [
+    4501,
+    5000,
+    7.8386,
+    2303797231.616,
+    319101075456.0,
+    1.0
+   ],
+   [
+    5001,
+    5500,
+    7.8155,
+    1248364724.224,
+    219623407616.0,
+    1.0
+   ],
+   [
+    5501,
+    6000,
+    7.8476,
+    45163974.656,
+    11680719872.0,
+    1.0
+   ],
+   [
+    6001,
+    6500,
+    7.8398,
+    205707673.6,
+    14566643712.0,
+    1.0
+   ],
+   [
+    6501,
+    7000,
+    7.9865,
+    6965250.56,
+    1029098112.0,
+    1.0
+   ],
+   [
+    7001,
+    7500,
+    7.8599,
+    66527158.272,
+    14402017280.0,
+    1.0
+   ],
+   [
+    7501,
+    8000,
+    7.8567,
+    646149505.024,
+    156670820352.0,
+    1.0
+   ],
+   [
+    8001,
+    8500,
+    7.8375,
+    987057.536,
+    129652760.0,
+    1.0
+   ],
+   [
+    8501,
+    9000,
+    7.7812,
+    127671582.72,
+    56667426816.0,
+    1.0
+   ],
+   [
+    9001,
+    9500,
+    7.7905,
+    50559336.448,
+    4162972160.0,
+    1.0
+   ],
+   [
+    9501,
+    10000,
+    7.7827,
+    41094246.4,
+    4887258624.0,
+    1.0
+   ],
+   [
+    10001,
+    10500,
+    7.8093,
+    53485305.856,
+    23333797888.0,
+    1.0
+   ],
+   [
+    10501,
+    11000,
+    7.8024,
+    29450575.872,
+    6264079360.0,
+    1.0
+   ],
+   [
+    11001,
+    11500,
+    7.7992,
+    12417133.568,
+    1991587200.0,
+    1.0
+   ],
+   [
+    11501,
+    12000,
+    7.8328,
+    19736858.624,
+    8977612800.0,
+    1.0
+   ],
+   [
+    12001,
+    12500,
+    7.83,
+    305598.4,
+    23749328.0,
+    1.0
+   ],
+   [
+    12501,
+    13000,
+    7.8058,
+    68349.136,
+    15220312.0,
+    1.0
+   ],
+   [
+    13001,
+    13500,
+    7.7986,
+    243661.376,
+    89982584.0,
+    1.0
+   ],
+   [
+    13501,
+    14000,
+    7.8029,
+    353022.144,
+    33967996.0,
+    1.0
+   ],
+   [
+    14001,
+    14500,
+    7.7932,
+    5561296.896,
+    703914240.0,
+    1.0
+   ],
+   [
+    14501,
+    15000,
+    7.8114,
+    1621969.024,
+    163940480.0,
+    1.0
+   ],
+   [
+    15001,
+    15500,
+    7.8004,
+    316363.776,
+    105838704.0,
+    1.0
+   ],
+   [
+    15501,
+    16000,
+    7.8113,
+    39209.184,
+    7385705.5,
+    1.0
+   ],
+   [
+    16001,
+    16500,
+    7.7727,
+    74813.624,
+    15877264.0,
+    1.0
+   ],
+   [
+    16501,
+    17000,
+    7.7546,
+    151585.264,
+    35481012.0,
+    1.0
+   ],
+   [
+    17001,
+    17500,
+    7.7075,
+    10426.528,
+    324472.8125,
+    1.0
+   ],
+   [
+    17501,
+    18000,
+    7.6165,
+    5841.039,
+    308463.125,
+    1.0
+   ],
+   [
+    18001,
+    18500,
+    7.549,
+    17413.684,
+    3653565.5,
+    1.0
+   ],
+   [
+    18501,
+    19000,
+    7.4955,
+    7608.817,
+    988159.5625,
+    1.0
+   ],
+   [
+    19001,
+    19500,
+    7.4081,
+    6861.078,
+    1417659.5,
+    1.0
+   ],
+   [
+    19501,
+    20000,
+    7.3676,
+    5175.8085,
+    933903.625,
+    1.0
+   ],
+   [
+    20001,
+    20500,
+    7.426,
+    1747.8492,
+    125891.2656,
+    1.0
+   ],
+   [
+    20501,
+    21000,
+    7.268,
+    7808.9935,
+    2127733.0,
+    1.0
+   ],
+   [
+    21001,
+    21500,
+    7.101,
+    1423.578,
+    210228.375,
+    1.0
+   ],
+   [
+    21501,
+    22000,
+    6.8099,
+    211.1428,
+    11816.293,
+    0.996
+   ],
+   [
+    22001,
+    22500,
+    6.4401,
+    27.3203,
+    920.8558,
+    0.992
+   ],
+   [
+    22501,
+    23000,
+    6.0178,
+    6.441,
+    289.1143,
+    0.972
+   ],
+   [
+    23001,
+    23500,
+    5.6746,
+    4.7993,
+    30.3995,
+    0.956
+   ],
+   [
+    23501,
+    24000,
+    5.4995,
+    3.6647,
+    13.0331,
+    0.928
+   ],
+   [
+    24001,
+    24500,
+    5.326,
+    3.3736,
+    14.5345,
+    0.878
+   ],
+   [
+    24501,
+    25000,
+    5.1846,
+    3.1234,
+    79.6111,
+    0.864
+   ],
+   [
+    25001,
+    25500,
+    5.0677,
+    2.5231,
+    11.0337,
+    0.83
+   ],
+   [
+    25501,
+    26000,
+    4.9072,
+    2.1976,
+    8.0747,
+    0.838
+   ],
+   [
+    26001,
+    26500,
+    4.8066,
+    2.1376,
+    10.0006,
+    0.818
+   ],
+   [
+    26501,
+    27000,
+    4.6753,
+    2.1215,
+    21.268,
+    0.856
+   ],
+   [
+    27001,
+    27500,
+    4.6294,
+    2.8368,
+    41.3304,
+    0.86
+   ],
+   [
+    27501,
+    28000,
+    4.5755,
+    2.1243,
+    22.4167,
+    0.802
+   ],
+   [
+    28001,
+    28500,
+    4.5268,
+    1.8335,
+    11.5507,
+    0.796
+   ],
+   [
+    28501,
+    29000,
+    4.462,
+    1.7914,
+    27.395,
+    0.764
+   ],
+   [
+    29001,
+    29500,
+    4.4037,
+    1.6559,
+    13.4478,
+    0.76
+   ],
+   [
+    29501,
+    30000,
+    4.4009,
+    1.5815,
+    9.0915,
+    0.718
+   ],
+   [
+    30001,
+    30500,
+    4.3442,
+    1.4774,
+    6.9297,
+    0.7
+   ],
+   [
+    30501,
+    31000,
+    4.3286,
+    1.8579,
+    48.4739,
+    0.758
+   ],
+   [
+    31001,
+    31500,
+    4.2859,
+    1.4787,
+    19.6523,
+    0.722
+   ],
+   [
+    31501,
+    32000,
+    4.2687,
+    3.6381,
+    521.2945,
+    0.746
+   ],
+   [
+    32001,
+    32500,
+    4.2599,
+    1.9122,
+    63.335,
+    0.66
+   ],
+   [
+    32501,
+    33000,
+    4.227,
+    1.7457,
+    116.627,
+    0.648
+   ],
+   [
+    33001,
+    33500,
+    4.2043,
+    1.3489,
+    3.9692,
+    0.688
+   ],
+   [
+    33501,
+    34000,
+    4.1551,
+    1.4007,
+    21.5781,
+    0.622
+   ],
+   [
+    34001,
+    34500,
+    4.1675,
+    1.2549,
+    7.3409,
+    0.604
+   ],
+   [
+    34501,
+    35000,
+    4.1445,
+    1.2692,
+    3.5421,
+    0.652
+   ],
+   [
+    35001,
+    35500,
+    4.1371,
+    1.292,
+    5.1026,
+    0.632
+   ],
+   [
+    35501,
+    36000,
+    4.0983,
+    1.4282,
+    9.7385,
+    0.664
+   ],
+   [
+    36001,
+    36500,
+    4.1223,
+    3.7575,
+    117.7014,
+    0.884
+   ],
+   [
+    36501,
+    37000,
+    4.0695,
+    4.904,
+    348.9586,
+    0.934
+   ],
+   [
+    37001,
+    37500,
+    4.0842,
+    2.9045,
+    229.8082,
+    0.726
+   ],
+   [
+    37501,
+    38000,
+    4.0639,
+    1.9381,
+    152.6303,
+    0.688
+   ],
+   [
+    38001,
+    38500,
+    4.0525,
+    1.378,
+    18.7875,
+    0.608
+   ],
+   [
+    38501,
+    39000,
+    4.0368,
+    1.4908,
+    51.6213,
+    0.622
+   ],
+   [
+    39001,
+    39500,
+    4.0236,
+    1.5879,
+    57.2907,
+    0.668
+   ],
+   [
+    39501,
+    40000,
+    3.9865,
+    1.1688,
+    6.6591,
+    0.594
+   ],
+   [
+    40001,
+    40500,
+    3.9875,
+    1.2421,
+    6.1756,
+    0.644
+   ],
+   [
+    40501,
+    41000,
+    3.9942,
+    1.2459,
+    14.735,
+    0.586
+   ],
+   [
+    41001,
+    41500,
+    3.9839,
+    1.2265,
+    7.9689,
+    0.65
+   ],
+   [
+    41501,
+    42000,
+    3.9453,
+    1.2154,
+    8.9331,
+    0.606
+   ],
+   [
+    42001,
+    42500,
+    3.9462,
+    1.1613,
+    12.8081,
+    0.578
+   ],
+   [
+    42501,
+    43000,
+    3.9449,
+    1.178,
+    3.5934,
+    0.636
+   ],
+   [
+    43001,
+    43500,
+    3.9106,
+    1.1271,
+    2.9759,
+    0.57
+   ],
+   [
+    43501,
+    44000,
+    3.9122,
+    1.1631,
+    6.6498,
+    0.576
+   ],
+   [
+    44001,
+    44500,
+    3.9059,
+    1.1006,
+    4.8825,
+    0.522
+   ],
+   [
+    44501,
+    45000,
+    3.9081,
+    1.1859,
+    9.6619,
+    0.63
+   ],
+   [
+    45001,
+    45500,
+    3.8967,
+    1.0946,
+    2.8455,
+    0.562
+   ],
+   [
+    45501,
+    46000,
+    3.8778,
+    1.1343,
+    10.1249,
+    0.566
+   ],
+   [
+    46001,
+    46500,
+    3.8677,
+    1.1478,
+    3.0913,
+    0.594
+   ],
+   [
+    46501,
+    47000,
+    3.8585,
+    1.141,
+    8.0959,
+    0.582
+   ],
+   [
+    47001,
+    47500,
+    3.856,
+    1.1271,
+    2.7162,
+    0.584
+   ],
+   [
+    47501,
+    48000,
+    3.8641,
+    1.1153,
+    3.8281,
+    0.596
+   ],
+   [
+    48001,
+    48500,
+    3.8704,
+    1.1974,
+    24.8528,
+    0.58
+   ],
+   [
+    48501,
+    49000,
+    3.8467,
+    1.15,
+    6.6223,
+    0.64
+   ],
+   [
+    49001,
+    49500,
+    3.8208,
+    1.1355,
+    10.5214,
+    0.61
+   ],
+   [
+    49501,
+    50000,
+    3.8136,
+    1.1372,
+    3.2393,
+    0.674
+   ],
+   [
+    50001,
+    50500,
+    3.8025,
+    1.1066,
+    6.0937,
+    0.572
+   ],
+   [
+    50501,
+    51000,
+    3.7829,
+    1.1579,
+    2.7934,
+    0.674
+   ],
+   [
+    51001,
+    51500,
+    3.7904,
+    1.1646,
+    5.5827,
+    0.658
+   ],
+   [
+    51501,
+    52000,
+    3.7827,
+    1.1351,
+    2.7482,
+    0.664
+   ],
+   [
+    52001,
+    52500,
+    3.7755,
+    1.1332,
+    5.5667,
+    0.664
+   ],
+   [
+    52501,
+    53000,
+    3.8042,
+    1.1419,
+    5.2309,
+    0.694
+   ],
+   [
+    53001,
+    53500,
+    3.7811,
+    1.1615,
+    5.0569,
+    0.704
+   ],
+   [
+    53501,
+    54000,
+    3.799,
+    1.1445,
+    2.9256,
+    0.674
+   ],
+   [
+    54001,
+    54500,
+    3.7553,
+    1.1206,
+    2.2743,
+    0.644
+   ],
+   [
+    54501,
+    55000,
+    3.7891,
+    1.1335,
+    4.4303,
+    0.682
+   ],
+   [
+    55001,
+    55500,
+    3.7658,
+    1.128,
+    3.0582,
+    0.686
+   ],
+   [
+    55501,
+    56000,
+    3.7724,
+    1.1893,
+    9.2161,
+    0.722
+   ],
+   [
+    56001,
+    56500,
+    3.7683,
+    1.1439,
+    2.5521,
+    0.732
+   ],
+   [
+    56501,
+    57000,
+    3.7704,
+    1.1785,
+    3.8908,
+    0.768
+   ],
+   [
+    57001,
+    57500,
+    3.7553,
+    1.149,
+    2.3439,
+    0.7
+   ],
+   [
+    57501,
+    58000,
+    3.7509,
+    1.1868,
+    7.4161,
+    0.75
+   ],
+   [
+    58001,
+    58500,
+    3.7416,
+    1.2314,
+    15.3189,
+    0.794
+   ],
+   [
+    58501,
+    59000,
+    3.7374,
+    1.1469,
+    2.4297,
+    0.774
+   ],
+   [
+    59001,
+    59500,
+    3.7469,
+    1.1641,
+    4.0544,
+    0.714
+   ],
+   [
+    59501,
+    60000,
+    3.7033,
+    1.18,
+    9.3027,
+    0.784
+   ],
+   [
+    60001,
+    60500,
+    3.7548,
+    1.1707,
+    3.158,
+    0.776
+   ],
+   [
+    60501,
+    61000,
+    3.7601,
+    1.2077,
+    10.4621,
+    0.814
+   ]
+  ],
+  "status": "finished",
+  "ppl": 42.14531938222052,
+  "ood_ppl": 150.1752538901007,
+  "trained": true,
+  "ckpt": "/mnt/local/helm-lp2-a1-93540a93/data/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.pt"
+ }
+}

checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32_last_finite_saved_step19500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4e743e7c8e1fc74adc169daafabad4a5543b21c4e3af813c1afe32d2a64a379
+size 3885189944

checkpoints/ckpt_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32_last_finite_saved_step37500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42ff7e2ea0ad417a592c5c2756331111dd51ec0c900438c79d9eed5a35ce63d0
+size 3885189944

checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8_last_finite_saved_step21000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95cb6389f3a95cc91aca0931dc4cee44691a8c498f5f29fd6525ab0aed04dff4
+size 3885189944

checkpoints/ckpt_unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0490250edd59c85f2d8b53cba3238f42b39ab839ed8f8aa0452815f5f3b586df
+size 3886231570

eval_curves/all_runs.csv

@@ -0,0 +1,69 @@
+run,seed,step,c4_ppl,wikitext_ppl
+scaled4,1234,2000,202.94915885729733,727.6774965555242
+scaled4,1234,4000,423.6878515622484,1275.309291092224
+scaled4,1234,6000,1489.0393986409879,2830.876521664318
+scaled4,1234,8000,2921.744164318744,6316.541965586585
+scaled4,1234,10000,2477.426440174055,4583.318111351122
+scaled4,1234,12000,1390.7013621661292,3173.9007569039645
+scaled4,1234,14000,819.5750432927489,2152.868470762587
+scaled4,1234,16000,1377.5248366693936,2871.021391179499
+scaled4,1234,18000,1283.73413189331,2968.7661825600626
+scaled8,1234,2000,174.7353397930056,610.8903868767173
+scaled8,1234,4000,96.74375678060908,278.5533283405427
+scaled8,1234,6000,71.96221656823293,210.89463260614258
+scaled8,1234,8000,63.241492720062254,181.14849829538917
+scaled8,1234,10000,58.61788591360284,178.9281156617443
+scaled8,1234,12000,184.65813325491646,604.537755576149
+scaled8,1234,14000,137.8900502438572,454.58166719125
+scaled8,1234,16000,169.85524170158178,513.6227455341899
+scaled8,1234,18000,100.05913976244142,253.7676409450834
+scaled8,1234,20000,133.7793884974217,429.05551043005937
+scaled8,1234,22000,88.13256767692252,227.04865686304538
+scaled8,1234,24000,109.31591745755064,302.6282972733793
+scaled8,1234,26000,103.20969940209199,249.42930528236712
+scaled8,1234,28000,78.43288412727317,189.27421096393644
+scaled8,1234,30000,74.88167041491353,195.60225877785737
+scaled8,1234,32000,70.9270138520994,182.039545371768
+scaled8,1234,34000,72.69512954617576,194.83785455268676
+scaled8,1234,36000,64.18778187722458,178.46032642706157
+unit_bf16_s7,7,2000,3081.6974153686674,5205.354479250944
+unit_bf16_s7,7,4000,3725.402220121194,6150.853129856762
+unit_bf16_s7,7,6000,6292.427285379758,9973.111690775377
+unit_bf16_s7,7,8000,7222.959259311071,11965.958219630595
+unit_bf16_s7,7,10000,3740.618074705774,7218.157307897914
+unit_bf16_s7,7,12000,2544.7708270565704,4938.8878274173785
+unit_bf16_s7,7,14000,2546.072196279018,4870.624762853716
+unit_bf16_s7,7,16000,2555.342611791562,4668.330447319325
+unit_bf16_s7,7,18000,2444.166069195049,4608.339614505096
+unit_bf16_s7,7,20000,1971.1374237795667,3768.617090639044
+unit_bf16_s99,99,2000,3147.398573896505,5423.754433548017
+unit_bf16_s99,99,4000,2707.576220762702,4847.083836922551
+unit_bf16_s99,99,6000,2575.1728912633325,4918.846833759445
+unit_bf16_s99,99,8000,2622.6838109655505,4914.688281620374
+unit_bf16_s99,99,10000,2502.9961945528967,4768.259514711109
+unit_bf16_s99,99,12000,2555.6282095415113,4784.532402448754
+unit_bf16_s99,99,14000,2508.2693488862164,4704.528607576495
+unit_bf16_s99,99,16000,2484.914327511012,4599.509893326248
+unit_bf16_s99,99,18000,2008.3956480861134,4011.7327462565904
+unit_bf16_s99,99,20000,1696.8711998096192,3996.417619164261
+unit_bf16_s99,99,22000,747.8862353569422,1892.978732361764
+unit_bf16_s99,99,24000,225.288699211105,811.6196224355541
+unit_bf16_s99,99,26000,130.1189965612892,420.9505620899285
+unit_bf16_s99,99,28000,95.71685234767584,273.45603981582235
+unit_bf16_s99,99,30000,79.44686243495329,226.86532284399632
+unit_bf16_s99,99,32000,72.3136327300778,208.0626240782148
+unit_bf16_s99,99,34000,65.51809922995677,184.70113508475671
+unit_bf16_s99,99,36000,60.97274644377895,167.96732176969124
+unit_bf16_s99,99,38000,57.76574385030062,159.04639087743783
+unit_bf16_s99,99,40000,54.881623367777976,164.44964338345233
+unit_bf16_s99,99,42000,52.39814165898491,154.59696545128406
+unit_bf16_s99,99,44000,50.373673803312734,153.76564483058056
+unit_bf16_s99,99,46000,48.942898812706986,150.02471746335945
+unit_bf16_s99,99,48000,47.28694778444997,154.86071208598747
+unit_bf16_s99,99,50000,46.0191269043681,159.17233857984223
+unit_bf16_s99,99,52000,44.916454874907835,155.1561176257511
+unit_bf16_s99,99,54000,44.17769508380568,153.79407774390504
+unit_bf16_s99,99,56000,43.48701997213694,156.58927100395852
+unit_bf16_s99,99,58000,42.92737757988304,158.2432593686224
+unit_bf16_s99,99,60000,42.254358168398994,141.4304989335991
+unit_bf16_s99,99,61000,42.14531938222052,150.1752538901007

eval_curves/scaled4.csv

@@ -0,0 +1,10 @@
+step,c4_ppl,wikitext_ppl
+2000,202.94915885729733,727.6774965555242
+4000,423.6878515622484,1275.309291092224
+6000,1489.0393986409879,2830.876521664318
+8000,2921.744164318744,6316.541965586585
+10000,2477.426440174055,4583.318111351122
+12000,1390.7013621661292,3173.9007569039645
+14000,819.5750432927489,2152.868470762587
+16000,1377.5248366693936,2871.021391179499
+18000,1283.73413189331,2968.7661825600626

eval_curves/scaled8.csv

@@ -0,0 +1,19 @@
+step,c4_ppl,wikitext_ppl
+2000,174.7353397930056,610.8903868767173
+4000,96.74375678060908,278.5533283405427
+6000,71.96221656823293,210.89463260614258
+8000,63.241492720062254,181.14849829538917
+10000,58.61788591360284,178.9281156617443
+12000,184.65813325491646,604.537755576149
+14000,137.8900502438572,454.58166719125
+16000,169.85524170158178,513.6227455341899
+18000,100.05913976244142,253.7676409450834
+20000,133.7793884974217,429.05551043005937
+22000,88.13256767692252,227.04865686304538
+24000,109.31591745755064,302.6282972733793
+26000,103.20969940209199,249.42930528236712
+28000,78.43288412727317,189.27421096393644
+30000,74.88167041491353,195.60225877785737
+32000,70.9270138520994,182.039545371768
+34000,72.69512954617576,194.83785455268676
+36000,64.18778187722458,178.46032642706157

eval_curves/unit4.csv

@@ -0,0 +1 @@
+step,c4_ppl,wikitext_ppl

eval_curves/unit8.csv

@@ -0,0 +1 @@
+step,c4_ppl,wikitext_ppl

eval_curves/unit_bf16_s7.csv

@@ -0,0 +1,11 @@
+step,c4_ppl,wikitext_ppl
+2000,3081.6974153686674,5205.354479250944
+4000,3725.402220121194,6150.853129856762
+6000,6292.427285379758,9973.111690775377
+8000,7222.959259311071,11965.958219630595
+10000,3740.618074705774,7218.157307897914
+12000,2544.7708270565704,4938.8878274173785
+14000,2546.072196279018,4870.624762853716
+16000,2555.342611791562,4668.330447319325
+18000,2444.166069195049,4608.339614505096
+20000,1971.1374237795667,3768.617090639044

eval_curves/unit_bf16_s99.csv

@@ -0,0 +1,32 @@
+step,c4_ppl,wikitext_ppl
+2000,3147.398573896505,5423.754433548017
+4000,2707.576220762702,4847.083836922551
+6000,2575.1728912633325,4918.846833759445
+8000,2622.6838109655505,4914.688281620374
+10000,2502.9961945528967,4768.259514711109
+12000,2555.6282095415113,4784.532402448754
+14000,2508.2693488862164,4704.528607576495
+16000,2484.914327511012,4599.509893326248
+18000,2008.3956480861134,4011.7327462565904
+20000,1696.8711998096192,3996.417619164261
+22000,747.8862353569422,1892.978732361764
+24000,225.288699211105,811.6196224355541
+26000,130.1189965612892,420.9505620899285
+28000,95.71685234767584,273.45603981582235
+30000,79.44686243495329,226.86532284399632
+32000,72.3136327300778,208.0626240782148
+34000,65.51809922995677,184.70113508475671
+36000,60.97274644377895,167.96732176969124
+38000,57.76574385030062,159.04639087743783
+40000,54.881623367777976,164.44964338345233
+42000,52.39814165898491,154.59696545128406
+44000,50.373673803312734,153.76564483058056
+46000,48.942898812706986,150.02471746335945
+48000,47.28694778444997,154.86071208598747
+50000,46.0191269043681,159.17233857984223
+52000,44.916454874907835,155.1561176257511
+54000,44.17769508380568,153.79407774390504
+56000,43.48701997213694,156.58927100395852
+58000,42.92737757988304,158.2432593686224
+60000,42.254358168398994,141.4304989335991
+61000,42.14531938222052,150.1752538901007

health/scaled4.csv

@@ -0,0 +1,41 @@
+from_step,through_step,loss_mean,preclip_gradient_norm_mean,preclip_gradient_norm_max,clipped_fraction
+1,500,9.7027,0.6383,4.5736,0.096
+501,1000,6.4858,0.9984,2.8632,0.4
+1001,1500,5.772,0.9442,2.5858,0.35
+1501,2000,5.4105,0.826,12.2292,0.138
+2001,2500,5.6805,511.6543,45749.1328,0.962
+2501,3000,5.9166,176220.096,74396832.0,1.0
+3001,3500,6.0195,1298834.048,402409728.0,1.0
+3501,4000,6.5126,52333150.208,3462804224.0,1.0
+4001,4500,6.8108,97818525.696,38629756928.0,1.0
+4501,5000,6.4227,542141.824,67260152.0,1.0
+5001,5500,6.3261,475369.824,37219236.0,1.0
+5501,6000,6.5516,76535504.896,10348393472.0,1.0
+6001,6500,6.7598,15476954.112,2227032576.0,1.0
+6501,7000,6.2951,19520233.472,4598260224.0,1.0
+7001,7500,6.2324,401158111.232,78418305024.0,1.0
+7501,8000,7.0267,142769815.552,21067493376.0,1.0
+8001,8500,7.8348,44360120926.208,21826317058048.0,1.0
+8501,9000,7.5718,4380696.064,403556960.0,1.0
+9001,9500,7.5329,6421007.36,2302962944.0,1.0
+9501,10000,7.7485,1609620914.176,104182497280.0,1.0
+10001,10500,7.8401,9454984626.176,3724470124544.0,1.0
+10501,11000,7.5425,1184740016.128,467213385728.0,1.0
+11001,11500,7.2397,192504086.528,29852839936.0,1.0
+11501,12000,7.2025,981352120.32,118586875904.0,1.0
+12001,12500,7.0402,44174032.896,8867602432.0,1.0
+12501,13000,6.8783,1848472.064,434353472.0,1.0
+13001,13500,6.946,621718.016,103816272.0,1.0
+13501,14000,6.8234,5290834.944,960473664.0,1.0
+14001,14500,6.5601,1377414.912,465742496.0,1.0
+14501,15000,6.4952,11370491805.696,5675421270016.0,1.0
+15001,15500,6.9291,18377663840.256,3771178418176.0,1.0
+15501,16000,7.0357,7124707035840.512,3476155403337728.0,1.0
+16001,16500,7.2548,668064915914.752,253003755945984.0,1.0
+16501,17000,7.2793,17794508783.616,3162229178368.0,1.0
+17001,17500,7.3164,2411633311.744,1038175633408.0,1.0
+17501,18000,7.1763,3796626571.264,923216904192.0,1.0
+18001,18500,7.1747,6283812405.248,2384119988224.0,1.0
+18501,19000,6.9052,9985686.528,2451557120.0,1.0
+19001,19500,6.514,341344.224,33471126.0,1.0
+19501,19600,6.3348,26860.6017,1098950.125,1.0

health/scaled8.csv

@@ -0,0 +1,77 @@
+from_step,through_step,loss_mean,preclip_gradient_norm_mean,preclip_gradient_norm_max,clipped_fraction
+1,500,9.7014,0.6114,3.1427,0.082
+501,1000,6.4802,0.9902,2.7411,0.384
+1001,1500,5.6738,0.9769,2.2324,0.378
+1501,2000,5.2693,0.7479,1.7225,0.1
+2001,2500,5.0838,0.6596,1.2531,0.02
+2501,3000,4.9109,0.6128,0.9605,0.0
+3001,3500,4.7996,0.5739,0.9984,0.0
+3501,4000,4.685,0.5463,1.3612,0.004
+4001,4500,4.5061,0.5196,0.7655,0.0
+4501,5000,4.3982,0.5005,1.1143,0.002
+5001,5500,4.3312,0.4785,0.9606,0.0
+5501,6000,4.2797,0.465,0.8798,0.0
+6001,6500,4.251,0.4564,1.5455,0.002
+6501,7000,4.1993,0.4535,1.8914,0.002
+7001,7500,4.1884,0.4514,1.9766,0.002
+7501,8000,4.1425,0.4524,1.3019,0.002
+8001,8500,4.116,0.4551,1.0741,0.002
+8501,9000,4.112,0.473,9.4198,0.004
+9001,9500,4.0714,0.4581,1.1425,0.004
+9501,10000,4.0591,0.4666,2.6143,0.006
+10001,10500,4.0487,0.5566,32.4937,0.016
+10501,11000,4.046,1.5361,308.9725,0.102
+11001,11500,4.1673,10.7338,1052.016,0.528
+11501,12000,5.0044,461.9405,41910.043,1.0
+12001,12500,5.3022,4680.4765,841680.4375,0.998
+12501,13000,5.9042,1014261.056,270517312.0,1.0
+13001,13500,5.7568,9639.093,1885640.625,0.994
+13501,14000,5.1947,1278.0512,536246.0625,0.97
+14001,14500,4.821,191.1503,15655.7461,0.998
+14501,15000,4.665,611.5712,38539.7383,1.0
+15001,15500,4.6244,851.9009,19325.9492,1.0
+15501,16000,5.1716,220322.144,69922816.0,1.0
+16001,16500,5.0968,45862.9,2904747.0,1.0
+16501,17000,4.6144,2933.3897,565594.6875,1.0
+17001,17500,4.5892,1432.0439,154115.0938,1.0
+17501,18000,4.5908,1931.2683,117779.0391,1.0
+18001,18500,4.6279,7236.754,314153.25,1.0
+18501,19000,4.5665,7100.8715,991551.6875,1.0
+19001,19500,4.6523,4891.302,240722.5312,1.0
+19501,20000,4.6961,26726.336,3621682.25,1.0
+20001,20500,4.592,2473.4257,126145.2734,1.0
+20501,21000,4.5865,7740.638,278614.9062,1.0
+21001,21500,4.7668,96743.416,11756970.0,1.0
+21501,22000,4.5791,5707.7255,316491.7188,1.0
+22001,22500,4.504,17389.226,2030029.0,1.0
+22501,23000,4.4457,6945.707,1344003.75,1.0
+23001,23500,4.6888,98770.912,42497904.0,1.0
+23501,24000,4.7402,70396.968,11657271.0,1.0
+24001,24500,4.7429,67652.272,9392578.0,1.0
+24501,25000,4.6507,37201.272,2190922.5,1.0
+25001,25500,4.6586,137704.464,9793316.0,1.0
+25501,26000,4.59,44320.172,7772814.0,1.0
+26001,26500,4.4964,8890.997,737653.0,1.0
+26501,27000,4.5432,15652.788,3759203.25,1.0
+27001,27500,4.5528,16914.748,1196604.375,1.0
+27501,28000,4.4514,11789.215,2854084.0,1.0
+28001,28500,4.3895,1443.7495,95537.0391,1.0
+28501,29000,4.4212,1729.4299,85023.1328,1.0
+29001,29500,4.3516,394.4305,33271.6328,1.0
+29501,30000,4.3481,685.3107,41030.5547,0.998
+30001,30500,4.3093,355.514,61418.9766,0.99
+30501,31000,4.3154,550.7336,178044.5,0.992
+31001,31500,4.2759,244.7275,14119.3076,0.978
+31501,32000,4.264,960.2903,178336.3438,0.992
+32001,32500,4.2232,2247.0422,881971.25,0.996
+32501,33000,4.2433,5435.5505,709869.625,1.0
+33001,33500,4.2045,17925.334,2200165.25,1.0
+33501,34000,4.2324,650.6419,26031.8379,1.0
+34001,34500,4.2475,322.4877,28030.752,0.976
+34501,35000,4.2041,2849.214,1364074.625,0.942
+35001,35500,4.2382,251.1124,25658.3789,0.984
+35501,36000,4.2298,121.1901,5135.478,1.0
+36001,36500,4.2358,1492.3666,140415.7344,1.0
+36501,37000,4.3222,1445.7544,349021.25,1.0
+37001,37500,4.1971,325.8608,35151.1172,0.976
+37501,37800,4.1911,98.3187,5155.6104,0.954

health/unit4.csv

@@ -0,0 +1,2 @@
+from_step,through_step,loss_mean,preclip_gradient_norm_mean,preclip_gradient_norm_max,clipped_fraction
+1,100,12.0917,1.6661621597263521e+18,1.7868244826848231e+19,1.0

health/unit8.csv

@@ -0,0 +1,2 @@
+from_step,through_step,loss_mean,preclip_gradient_norm_mean,preclip_gradient_norm_max,clipped_fraction
+1,500,12.0859,6.8915436589917096e+16,4.65222941019e+18,1.0

health/unit_bf16_s7.csv

@@ -0,0 +1,44 @@
+from_step,through_step,loss_mean,preclip_gradient_norm_mean,preclip_gradient_norm_max,clipped_fraction
+1,500,8.2741,279988968357.888,48261033361408.0,0.79
+501,1000,7.8433,5975770930171871.0,1.3047096345771377e+18,1.0
+1001,1500,7.88,2290687066058522.5,7.318069019011973e+17,1.0
+1501,2000,7.9409,3525511466696311.0,4.866156027487191e+17,1.0
+2001,2500,7.9134,73541376389677.06,3.347726217510912e+16,1.0
+2501,3000,7.8792,4096278887661.568,672137216000000.0,1.0
+3001,3500,7.9836,89853490381193.22,1.4693937818107904e+16,1.0
+3501,4000,8.1615,18353447522795.52,6111865163743232.0,1.0
+4001,4500,8.3271,13516694798467.072,3735046905135104.0,1.0
+4501,5000,8.6039,2092514911715.328,214209128300544.0,1.0
+5001,5500,8.7694,1035891418071.04,238156959973376.0,1.0
+5501,6000,8.6709,209231848407.04,44125311205376.0,1.0
+6001,6500,8.7985,134661242093.568,12161647640576.0,1.0
+6501,7000,8.928,156913174052.864,35240881946624.0,1.0
+7001,7500,8.958,1351027328548.864,595335684554752.0,1.0
+7501,8000,8.8752,46685258514.432,9556080984064.0,1.0
+8001,8500,8.9012,1064509120708.608,226382458126336.0,1.0
+8501,9000,8.8973,302028072943.616,146165672706048.0,1.0
+9001,9500,8.831,12558347010.048,1865004744704.0,1.0
+9501,10000,8.5071,1132090359.808,304874225664.0,1.0
+10001,10500,7.9842,17828536.32,2609804032.0,1.0
+10501,11000,7.8464,2342629.376,242555216.0,1.0
+11001,11500,7.8282,728957.952,23701910.0,1.0
+11501,12000,7.8193,586352.832,128272416.0,1.0
+12001,12500,7.8078,923206.848,135035104.0,1.0
+12501,13000,7.8106,5223426.048,2275896320.0,1.0
+13001,13500,7.8208,489591.456,76937816.0,1.0
+13501,14000,7.8182,4563990.528,1242059392.0,1.0
+14001,14500,7.8242,1906761.728,546927040.0,1.0
+14501,15000,7.8063,528363.68,64105964.0,1.0
+15001,15500,7.8116,2287449.856,744769856.0,1.0
+15501,16000,7.8181,89636.696,5071824.5,1.0
+16001,16500,7.8105,302070.016,61053604.0,1.0
+16501,17000,7.789,213601.184,30829950.0,1.0
+17001,17500,7.7641,530482.144,144564000.0,1.0
+17501,18000,7.7736,35051.048,3545721.25,1.0
+18001,18500,7.7783,79231.312,19613498.0,1.0
+18501,19000,7.6963,94032.744,8377379.0,1.0
+19001,19500,7.6139,16569.631,932935.875,1.0
+19501,20000,7.5621,7749.368,1046033.375,1.0
+20001,20500,7.4969,6382.7535,902437.75,1.0
+20501,21000,7.3791,3381.0467,425152.4062,1.0
+21001,21300,7.3324,2397.1069,89643.5859,1.0

health/unit_bf16_s99.csv

@@ -0,0 +1,123 @@
+from_step,through_step,loss_mean,preclip_gradient_norm_mean,preclip_gradient_norm_max,clipped_fraction
+1,500,8.2677,4029060218.88,1159352942592.0,0.786
+501,1000,7.8374,2.3309431004572156e+16,7.506468687163949e+18,1.0
+1001,1500,7.8911,895374512256712.8,2.4823254831490662e+17,1.0
+1501,2000,7.9687,577116679708868.6,2.185821556876247e+17,1.0
+2001,2500,8.0919,87295339140218.88,3.83244269739049e+16,1.0
+2501,3000,7.968,39968789562916.87,1.9767367936507904e+16,1.0
+3001,3500,7.9204,200710566182.912,38664566472704.0,1.0
+3501,4000,7.8349,358843416.576,27402854400.0,1.0
+4001,4500,7.8499,138364409872.384,50494122754048.0,1.0
+4501,5000,7.8386,2303797231.616,319101075456.0,1.0
+5001,5500,7.8155,1248364724.224,219623407616.0,1.0
+5501,6000,7.8476,45163974.656,11680719872.0,1.0
+6001,6500,7.8398,205707673.6,14566643712.0,1.0
+6501,7000,7.9865,6965250.56,1029098112.0,1.0
+7001,7500,7.8599,66527158.272,14402017280.0,1.0
+7501,8000,7.8567,646149505.024,156670820352.0,1.0
+8001,8500,7.8375,987057.536,129652760.0,1.0
+8501,9000,7.7812,127671582.72,56667426816.0,1.0
+9001,9500,7.7905,50559336.448,4162972160.0,1.0
+9501,10000,7.7827,41094246.4,4887258624.0,1.0
+10001,10500,7.8093,53485305.856,23333797888.0,1.0
+10501,11000,7.8024,29450575.872,6264079360.0,1.0
+11001,11500,7.7992,12417133.568,1991587200.0,1.0
+11501,12000,7.8328,19736858.624,8977612800.0,1.0
+12001,12500,7.83,305598.4,23749328.0,1.0
+12501,13000,7.8058,68349.136,15220312.0,1.0
+13001,13500,7.7986,243661.376,89982584.0,1.0
+13501,14000,7.8029,353022.144,33967996.0,1.0
+14001,14500,7.7932,5561296.896,703914240.0,1.0
+14501,15000,7.8114,1621969.024,163940480.0,1.0
+15001,15500,7.8004,316363.776,105838704.0,1.0
+15501,16000,7.8113,39209.184,7385705.5,1.0
+16001,16500,7.7727,74813.624,15877264.0,1.0
+16501,17000,7.7546,151585.264,35481012.0,1.0
+17001,17500,7.7075,10426.528,324472.8125,1.0
+17501,18000,7.6165,5841.039,308463.125,1.0
+18001,18500,7.549,17413.684,3653565.5,1.0
+18501,19000,7.4955,7608.817,988159.5625,1.0
+19001,19500,7.4081,6861.078,1417659.5,1.0
+19501,20000,7.3676,5175.8085,933903.625,1.0
+20001,20500,7.426,1747.8492,125891.2656,1.0
+20501,21000,7.268,7808.9935,2127733.0,1.0
+21001,21500,7.101,1423.578,210228.375,1.0
+21501,22000,6.8099,211.1428,11816.293,0.996
+22001,22500,6.4401,27.3203,920.8558,0.992
+22501,23000,6.0178,6.441,289.1143,0.972
+23001,23500,5.6746,4.7993,30.3995,0.956
+23501,24000,5.4995,3.6647,13.0331,0.928
+24001,24500,5.326,3.3736,14.5345,0.878
+24501,25000,5.1846,3.1234,79.6111,0.864
+25001,25500,5.0677,2.5231,11.0337,0.83
+25501,26000,4.9072,2.1976,8.0747,0.838
+26001,26500,4.8066,2.1376,10.0006,0.818
+26501,27000,4.6753,2.1215,21.268,0.856
+27001,27500,4.6294,2.8368,41.3304,0.86
+27501,28000,4.5755,2.1243,22.4167,0.802
+28001,28500,4.5268,1.8335,11.5507,0.796
+28501,29000,4.462,1.7914,27.395,0.764
+29001,29500,4.4037,1.6559,13.4478,0.76
+29501,30000,4.4009,1.5815,9.0915,0.718
+30001,30500,4.3442,1.4774,6.9297,0.7
+30501,31000,4.3286,1.8579,48.4739,0.758
+31001,31500,4.2859,1.4787,19.6523,0.722
+31501,32000,4.2687,3.6381,521.2945,0.746
+32001,32500,4.2599,1.9122,63.335,0.66
+32501,33000,4.227,1.7457,116.627,0.648
+33001,33500,4.2043,1.3489,3.9692,0.688
+33501,34000,4.1551,1.4007,21.5781,0.622
+34001,34500,4.1675,1.2549,7.3409,0.604
+34501,35000,4.1445,1.2692,3.5421,0.652
+35001,35500,4.1371,1.292,5.1026,0.632
+35501,36000,4.0983,1.4282,9.7385,0.664
+36001,36500,4.1223,3.7575,117.7014,0.884
+36501,37000,4.0695,4.904,348.9586,0.934
+37001,37500,4.0842,2.9045,229.8082,0.726
+37501,38000,4.0639,1.9381,152.6303,0.688
+38001,38500,4.0525,1.378,18.7875,0.608
+38501,39000,4.0368,1.4908,51.6213,0.622
+39001,39500,4.0236,1.5879,57.2907,0.668
+39501,40000,3.9865,1.1688,6.6591,0.594
+40001,40500,3.9875,1.2421,6.1756,0.644
+40501,41000,3.9942,1.2459,14.735,0.586
+41001,41500,3.9839,1.2265,7.9689,0.65
+41501,42000,3.9453,1.2154,8.9331,0.606
+42001,42500,3.9462,1.1613,12.8081,0.578
+42501,43000,3.9449,1.178,3.5934,0.636
+43001,43500,3.9106,1.1271,2.9759,0.57
+43501,44000,3.9122,1.1631,6.6498,0.576
+44001,44500,3.9059,1.1006,4.8825,0.522
+44501,45000,3.9081,1.1859,9.6619,0.63
+45001,45500,3.8967,1.0946,2.8455,0.562
+45501,46000,3.8778,1.1343,10.1249,0.566
+46001,46500,3.8677,1.1478,3.0913,0.594
+46501,47000,3.8585,1.141,8.0959,0.582
+47001,47500,3.856,1.1271,2.7162,0.584
+47501,48000,3.8641,1.1153,3.8281,0.596
+48001,48500,3.8704,1.1974,24.8528,0.58
+48501,49000,3.8467,1.15,6.6223,0.64
+49001,49500,3.8208,1.1355,10.5214,0.61
+49501,50000,3.8136,1.1372,3.2393,0.674
+50001,50500,3.8025,1.1066,6.0937,0.572
+50501,51000,3.7829,1.1579,2.7934,0.674
+51001,51500,3.7904,1.1646,5.5827,0.658
+51501,52000,3.7827,1.1351,2.7482,0.664
+52001,52500,3.7755,1.1332,5.5667,0.664
+52501,53000,3.8042,1.1419,5.2309,0.694
+53001,53500,3.7811,1.1615,5.0569,0.704
+53501,54000,3.799,1.1445,2.9256,0.674
+54001,54500,3.7553,1.1206,2.2743,0.644
+54501,55000,3.7891,1.1335,4.4303,0.682
+55001,55500,3.7658,1.128,3.0582,0.686
+55501,56000,3.7724,1.1893,9.2161,0.722
+56001,56500,3.7683,1.1439,2.5521,0.732
+56501,57000,3.7704,1.1785,3.8908,0.768
+57001,57500,3.7553,1.149,2.3439,0.7
+57501,58000,3.7509,1.1868,7.4161,0.75
+58001,58500,3.7416,1.2314,15.3189,0.794
+58501,59000,3.7374,1.1469,2.4297,0.774
+59001,59500,3.7469,1.1641,4.0544,0.714
+59501,60000,3.7033,1.18,9.3027,0.784
+60001,60500,3.7548,1.1707,3.158,0.776
+60501,61000,3.7601,1.2077,10.4621,0.814

profiles/scaled4.json

@@ -0,0 +1,121 @@
+{
+  "actual_outcome": {
+    "actual_terminal_step": 19600,
+    "attention_operand_format": "e2m1",
+    "evaluation_points": 9,
+    "final_c4_ppl": null,
+    "final_wikitext_ppl": null,
+    "first_nonfinite_step": 19576,
+    "full_budget_completed": false,
+    "initialization": "scaled",
+    "last_finite_evaluation": {
+      "c4_ppl": 1283.73413189331,
+      "step": 18000,
+      "wikitext_ppl": 2968.7661825600626
+    },
+    "last_finite_saved_step": 19500,
+    "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+    "native_json": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+    "native_json_sha256": "6ef01b6eb73a5a58a9a7a58d674b9052b33f8c6f43d13ae68586854312a4312e",
+    "planned_steps": 61000,
+    "planned_tokens": 1998848000,
+    "run": "scaled4",
+    "seed": 1234,
+    "status": "nonfinite"
+  },
+  "attention_operand_quantization": {
+    "block": 32,
+    "emulated": true,
+    "format": "e2m1",
+    "ops": "both"
+  },
+  "compile": false,
+  "init": "scaled",
+  "initialization_diagnostics": {
+    "R_init": 1.0,
+    "normalizer_radii": {
+      "layers.0.attn.kv_norm": 1.0,
+      "layers.0.attn_norm": 1.0,
+      "layers.0.ffn_norm": 1.0,
+      "layers.1.attn.kv_norm": 1.0,
+      "layers.1.attn_norm": 1.0,
+      "layers.1.ffn_norm": 1.0,
+      "layers.10.attn.kv_norm": 1.0,
+      "layers.10.attn_norm": 1.0,
+      "layers.10.ffn_norm": 1.0,
+      "layers.11.attn.kv_norm": 1.0,
+      "layers.11.attn_norm": 1.0,
+      "layers.11.ffn_norm": 1.0,
+      "layers.12.attn.kv_norm": 1.0,
+      "layers.12.attn_norm": 1.0,
+      "layers.12.ffn_norm": 1.0,
+      "layers.13.attn.kv_norm": 1.0,
+      "layers.13.attn_norm": 1.0,
+      "layers.13.ffn_norm": 1.0,
+      "layers.14.attn.kv_norm": 1.0,
+      "layers.14.attn_norm": 1.0,
+      "layers.14.ffn_norm": 1.0,
+      "layers.15.attn.kv_norm": 1.0,
+      "layers.15.attn_norm": 1.0,
+      "layers.15.ffn_norm": 1.0,
+      "layers.2.attn.kv_norm": 1.0,
+      "layers.2.attn_norm": 1.0,
+      "layers.2.ffn_norm": 1.0,
+      "layers.3.attn.kv_norm": 1.0,
+      "layers.3.attn_norm": 1.0,
+      "layers.3.ffn_norm": 1.0,
+      "layers.4.attn.kv_norm": 1.0,
+      "layers.4.attn_norm": 1.0,
+      "layers.4.ffn_norm": 1.0,
+      "layers.5.attn.kv_norm": 1.0,
+      "layers.5.attn_norm": 1.0,
+      "layers.5.ffn_norm": 1.0,
+      "layers.6.attn.kv_norm": 1.0,
+      "layers.6.attn_norm": 1.0,
+      "layers.6.ffn_norm": 1.0,
+      "layers.7.attn.kv_norm": 1.0,
+      "layers.7.attn_norm": 1.0,
+      "layers.7.ffn_norm": 1.0,
+      "layers.8.attn.kv_norm": 1.0,
+      "layers.8.attn_norm": 1.0,
+      "layers.8.ffn_norm": 1.0,
+      "layers.9.attn.kv_norm": 1.0,
+      "layers.9.attn_norm": 1.0,
+      "layers.9.ffn_norm": 1.0,
+      "norm": 1.0
+    },
+    "u_init": 1.0
+  },
+  "model": {
+    "curvature": 1.0,
+    "dim": 910,
+    "heads": 14,
+    "layers": 16,
+    "unique_parameter_elements": 966957719,
+    "vocab_size": 128256
+  },
+  "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+  "parameter_dtype": "float32",
+  "policy": false,
+  "precision": "bf16",
+  "recipe": {
+    "accumulation": 2,
+    "eval_every": 2000,
+    "final_lr": 0.0001,
+    "lr": 0.001,
+    "microbatch": 1,
+    "save_every": 500,
+    "steps": 61000,
+    "tokens_per_step": 32768,
+    "warmup_steps": 1220,
+    "world": 8
+  },
+  "run": "scaled4",
+  "schema": "helm.c42b.public-profile.v1",
+  "seed": 1234,
+  "source": {
+    "helm_revision": "7501deca8f413848bfef804be64ce874b72a3cd7",
+    "lowp_pr_revision": "5edb20302017712ac97bf39374f66db80f94bc51",
+    "native_trainer_revision": "5f968bd8d023298728e3ac43d6fefe64f786de8e"
+  }
+}

profiles/scaled8.json

@@ -0,0 +1,121 @@
+{
+  "actual_outcome": {
+    "actual_terminal_step": 37800,
+    "attention_operand_format": "e4m3",
+    "evaluation_points": 18,
+    "final_c4_ppl": null,
+    "final_wikitext_ppl": null,
+    "first_nonfinite_step": 37785,
+    "full_budget_completed": false,
+    "initialization": "scaled",
+    "last_finite_evaluation": {
+      "c4_ppl": 64.18778187722458,
+      "step": 36000,
+      "wikitext_ppl": 178.46032642706157
+    },
+    "last_finite_saved_step": 37500,
+    "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+    "native_json": "cells.d/scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+    "native_json_sha256": "566d1325afbb580f0d8485054e6b56e95eec2b96f2a7393a4c3a6a7cb5bff177",
+    "planned_steps": 61000,
+    "planned_tokens": 1998848000,
+    "run": "scaled8",
+    "seed": 1234,
+    "status": "nonfinite"
+  },
+  "attention_operand_quantization": {
+    "block": 32,
+    "emulated": true,
+    "format": "e4m3",
+    "ops": "both"
+  },
+  "compile": false,
+  "init": "scaled",
+  "initialization_diagnostics": {
+    "R_init": 1.0,
+    "normalizer_radii": {
+      "layers.0.attn.kv_norm": 1.0,
+      "layers.0.attn_norm": 1.0,
+      "layers.0.ffn_norm": 1.0,
+      "layers.1.attn.kv_norm": 1.0,
+      "layers.1.attn_norm": 1.0,
+      "layers.1.ffn_norm": 1.0,
+      "layers.10.attn.kv_norm": 1.0,
+      "layers.10.attn_norm": 1.0,
+      "layers.10.ffn_norm": 1.0,
+      "layers.11.attn.kv_norm": 1.0,
+      "layers.11.attn_norm": 1.0,
+      "layers.11.ffn_norm": 1.0,
+      "layers.12.attn.kv_norm": 1.0,
+      "layers.12.attn_norm": 1.0,
+      "layers.12.ffn_norm": 1.0,
+      "layers.13.attn.kv_norm": 1.0,
+      "layers.13.attn_norm": 1.0,
+      "layers.13.ffn_norm": 1.0,
+      "layers.14.attn.kv_norm": 1.0,
+      "layers.14.attn_norm": 1.0,
+      "layers.14.ffn_norm": 1.0,
+      "layers.15.attn.kv_norm": 1.0,
+      "layers.15.attn_norm": 1.0,
+      "layers.15.ffn_norm": 1.0,
+      "layers.2.attn.kv_norm": 1.0,
+      "layers.2.attn_norm": 1.0,
+      "layers.2.ffn_norm": 1.0,
+      "layers.3.attn.kv_norm": 1.0,
+      "layers.3.attn_norm": 1.0,
+      "layers.3.ffn_norm": 1.0,
+      "layers.4.attn.kv_norm": 1.0,
+      "layers.4.attn_norm": 1.0,
+      "layers.4.ffn_norm": 1.0,
+      "layers.5.attn.kv_norm": 1.0,
+      "layers.5.attn_norm": 1.0,
+      "layers.5.ffn_norm": 1.0,
+      "layers.6.attn.kv_norm": 1.0,
+      "layers.6.attn_norm": 1.0,
+      "layers.6.ffn_norm": 1.0,
+      "layers.7.attn.kv_norm": 1.0,
+      "layers.7.attn_norm": 1.0,
+      "layers.7.ffn_norm": 1.0,
+      "layers.8.attn.kv_norm": 1.0,
+      "layers.8.attn_norm": 1.0,
+      "layers.8.ffn_norm": 1.0,
+      "layers.9.attn.kv_norm": 1.0,
+      "layers.9.attn_norm": 1.0,
+      "layers.9.ffn_norm": 1.0,
+      "norm": 1.0
+    },
+    "u_init": 1.0
+  },
+  "model": {
+    "curvature": 1.0,
+    "dim": 910,
+    "heads": 14,
+    "layers": 16,
+    "unique_parameter_elements": 966957719,
+    "vocab_size": 128256
+  },
+  "native_cell_key": "scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+  "parameter_dtype": "float32",
+  "policy": false,
+  "precision": "bf16",
+  "recipe": {
+    "accumulation": 2,
+    "eval_every": 2000,
+    "final_lr": 0.0001,
+    "lr": 0.001,
+    "microbatch": 1,
+    "save_every": 500,
+    "steps": 61000,
+    "tokens_per_step": 32768,
+    "warmup_steps": 1220,
+    "world": 8
+  },
+  "run": "scaled8",
+  "schema": "helm.c42b.public-profile.v1",
+  "seed": 1234,
+  "source": {
+    "helm_revision": "7501deca8f413848bfef804be64ce874b72a3cd7",
+    "lowp_pr_revision": "5edb20302017712ac97bf39374f66db80f94bc51",
+    "native_trainer_revision": "5f968bd8d023298728e3ac43d6fefe64f786de8e"
+  }
+}

profiles/unit4.json

@@ -0,0 +1,117 @@
+{
+  "actual_outcome": {
+    "actual_terminal_step": 100,
+    "attention_operand_format": "e2m1",
+    "evaluation_points": 0,
+    "final_c4_ppl": null,
+    "final_wikitext_ppl": null,
+    "first_nonfinite_step": 94,
+    "full_budget_completed": false,
+    "initialization": "unit",
+    "last_finite_evaluation": null,
+    "last_finite_saved_step": null,
+    "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+    "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.json",
+    "native_json_sha256": "c8024707c0bcc82ac8f0d43e78fbc458c4f096f1cbbb97b04a711d161980872a",
+    "planned_steps": 61000,
+    "planned_tokens": 1998848000,
+    "run": "unit4",
+    "seed": 1234,
+    "status": "nonfinite"
+  },
+  "attention_operand_quantization": {
+    "block": 32,
+    "emulated": true,
+    "format": "e2m1",
+    "ops": "both"
+  },
+  "compile": false,
+  "init": "unit",
+  "initialization_diagnostics": {
+    "R_init": 30.1496,
+    "normalizer_radii": {
+      "layers.0.attn.kv_norm": 16.031,
+      "layers.0.attn_norm": 30.15,
+      "layers.0.ffn_norm": 30.15,
+      "layers.1.attn.kv_norm": 16.031,
+      "layers.1.attn_norm": 30.15,
+      "layers.1.ffn_norm": 30.15,
+      "layers.10.attn.kv_norm": 16.031,
+      "layers.10.attn_norm": 30.15,
+      "layers.10.ffn_norm": 30.15,
+      "layers.11.attn.kv_norm": 16.031,
+      "layers.11.attn_norm": 30.15,
+      "layers.11.ffn_norm": 30.15,
+      "layers.12.attn.kv_norm": 16.031,
+      "layers.12.attn_norm": 30.15,
+      "layers.12.ffn_norm": 30.15,
+      "layers.13.attn.kv_norm": 16.031,
+      "layers.13.attn_norm": 30.15,
+      "layers.13.ffn_norm": 30.15,
+      "layers.14.attn.kv_norm": 16.031,
+      "layers.14.attn_norm": 30.15,
+      "layers.14.ffn_norm": 30.15,
+      "layers.15.attn.kv_norm": 16.031,
+      "layers.15.attn_norm": 30.15,
+      "layers.15.ffn_norm": 30.15,
+      "layers.2.attn.kv_norm": 16.031,
+      "layers.2.attn_norm": 30.15,
+      "layers.2.ffn_norm": 30.15,
+      "layers.3.attn.kv_norm": 16.031,
+      "layers.3.attn_norm": 30.15,
+      "layers.3.ffn_norm": 30.15,
+      "layers.4.attn.kv_norm": 16.031,
+      "layers.4.attn_norm": 30.15,
+      "layers.4.ffn_norm": 30.15,
+      "layers.5.attn.kv_norm": 16.031,
+      "layers.5.attn_norm": 30.15,
+      "layers.5.ffn_norm": 30.15,
+      "layers.6.attn.kv_norm": 16.031,
+      "layers.6.attn_norm": 30.15,
+      "layers.6.ffn_norm": 30.15,
+      "layers.7.attn.kv_norm": 16.031,
+      "layers.7.attn_norm": 30.15,
+      "layers.7.ffn_norm": 30.15,
+      "layers.8.attn.kv_norm": 16.031,
+      "layers.8.attn_norm": 30.15,
+      "layers.8.ffn_norm": 30.15,
+      "layers.9.attn.kv_norm": 16.031,
+      "layers.9.attn_norm": 30.15,
+      "layers.9.ffn_norm": 30.15,
+      "norm": 30.15
+    },
+    "u_init": 30.1496
+  },
+  "model": {
+    "curvature": 1.0,
+    "dim": 910,
+    "heads": 14,
+    "layers": 16,
+    "unique_parameter_elements": 966957719,
+    "vocab_size": 128256
+  },
+  "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32",
+  "parameter_dtype": "float32",
+  "policy": false,
+  "precision": "bf16",
+  "recipe": {
+    "accumulation": 2,
+    "eval_every": 2000,
+    "final_lr": 0.0001,
+    "lr": 0.001,
+    "microbatch": 1,
+    "save_every": 500,
+    "steps": 61000,
+    "tokens_per_step": 32768,
+    "warmup_steps": 1220,
+    "world": 8
+  },
+  "run": "unit4",
+  "schema": "helm.c42b.public-profile.v1",
+  "seed": 1234,
+  "source": {
+    "helm_revision": "7501deca8f413848bfef804be64ce874b72a3cd7",
+    "lowp_pr_revision": "5edb20302017712ac97bf39374f66db80f94bc51",
+    "native_trainer_revision": "5f968bd8d023298728e3ac43d6fefe64f786de8e"
+  }
+}

profiles/unit8.json

@@ -0,0 +1,117 @@
+{
+  "actual_outcome": {
+    "actual_terminal_step": 500,
+    "attention_operand_format": "e4m3",
+    "evaluation_points": 0,
+    "final_c4_ppl": null,
+    "final_wikitext_ppl": null,
+    "first_nonfinite_step": 470,
+    "full_budget_completed": false,
+    "initialization": "unit",
+    "last_finite_evaluation": null,
+    "last_finite_saved_step": null,
+    "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+    "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.json",
+    "native_json_sha256": "c593f530ac19449f8adb4464c72383b51048ab95f629f2af5a3edff8ee2a6cf8",
+    "planned_steps": 61000,
+    "planned_tokens": 1998848000,
+    "run": "unit8",
+    "seed": 1234,
+    "status": "nonfinite"
+  },
+  "attention_operand_quantization": {
+    "block": 32,
+    "emulated": true,
+    "format": "e4m3",
+    "ops": "both"
+  },
+  "compile": false,
+  "init": "unit",
+  "initialization_diagnostics": {
+    "R_init": 30.1496,
+    "normalizer_radii": {
+      "layers.0.attn.kv_norm": 16.031,
+      "layers.0.attn_norm": 30.15,
+      "layers.0.ffn_norm": 30.15,
+      "layers.1.attn.kv_norm": 16.031,
+      "layers.1.attn_norm": 30.15,
+      "layers.1.ffn_norm": 30.15,
+      "layers.10.attn.kv_norm": 16.031,
+      "layers.10.attn_norm": 30.15,
+      "layers.10.ffn_norm": 30.15,
+      "layers.11.attn.kv_norm": 16.031,
+      "layers.11.attn_norm": 30.15,
+      "layers.11.ffn_norm": 30.15,
+      "layers.12.attn.kv_norm": 16.031,
+      "layers.12.attn_norm": 30.15,
+      "layers.12.ffn_norm": 30.15,
+      "layers.13.attn.kv_norm": 16.031,
+      "layers.13.attn_norm": 30.15,
+      "layers.13.ffn_norm": 30.15,
+      "layers.14.attn.kv_norm": 16.031,
+      "layers.14.attn_norm": 30.15,
+      "layers.14.ffn_norm": 30.15,
+      "layers.15.attn.kv_norm": 16.031,
+      "layers.15.attn_norm": 30.15,
+      "layers.15.ffn_norm": 30.15,
+      "layers.2.attn.kv_norm": 16.031,
+      "layers.2.attn_norm": 30.15,
+      "layers.2.ffn_norm": 30.15,
+      "layers.3.attn.kv_norm": 16.031,
+      "layers.3.attn_norm": 30.15,
+      "layers.3.ffn_norm": 30.15,
+      "layers.4.attn.kv_norm": 16.031,
+      "layers.4.attn_norm": 30.15,
+      "layers.4.ffn_norm": 30.15,
+      "layers.5.attn.kv_norm": 16.031,
+      "layers.5.attn_norm": 30.15,
+      "layers.5.ffn_norm": 30.15,
+      "layers.6.attn.kv_norm": 16.031,
+      "layers.6.attn_norm": 30.15,
+      "layers.6.ffn_norm": 30.15,
+      "layers.7.attn.kv_norm": 16.031,
+      "layers.7.attn_norm": 30.15,
+      "layers.7.ffn_norm": 30.15,
+      "layers.8.attn.kv_norm": 16.031,
+      "layers.8.attn_norm": 30.15,
+      "layers.8.ffn_norm": 30.15,
+      "layers.9.attn.kv_norm": 16.031,
+      "layers.9.attn_norm": 30.15,
+      "layers.9.ffn_norm": 30.15,
+      "norm": 30.15
+    },
+    "u_init": 30.1496
+  },
+  "model": {
+    "curvature": 1.0,
+    "dim": 910,
+    "heads": 14,
+    "layers": 16,
+    "unique_parameter_elements": 966957719,
+    "vocab_size": 128256
+  },
+  "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32",
+  "parameter_dtype": "float32",
+  "policy": false,
+  "precision": "bf16",
+  "recipe": {
+    "accumulation": 2,
+    "eval_every": 2000,
+    "final_lr": 0.0001,
+    "lr": 0.001,
+    "microbatch": 1,
+    "save_every": 500,
+    "steps": 61000,
+    "tokens_per_step": 32768,
+    "warmup_steps": 1220,
+    "world": 8
+  },
+  "run": "unit8",
+  "schema": "helm.c42b.public-profile.v1",
+  "seed": 1234,
+  "source": {
+    "helm_revision": "7501deca8f413848bfef804be64ce874b72a3cd7",
+    "lowp_pr_revision": "5edb20302017712ac97bf39374f66db80f94bc51",
+    "native_trainer_revision": "5f968bd8d023298728e3ac43d6fefe64f786de8e"
+  }
+}

profiles/unit_bf16_s7.json

@@ -0,0 +1,121 @@
+{
+  "actual_outcome": {
+    "actual_terminal_step": 21300,
+    "attention_operand_format": null,
+    "evaluation_points": 10,
+    "final_c4_ppl": null,
+    "final_wikitext_ppl": null,
+    "first_nonfinite_step": 21249,
+    "full_budget_completed": false,
+    "initialization": "unit",
+    "last_finite_evaluation": {
+      "c4_ppl": 1971.1374237795667,
+      "step": 20000,
+      "wikitext_ppl": 3768.617090639044
+    },
+    "last_finite_saved_step": 21000,
+    "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+    "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.json",
+    "native_json_sha256": "b8560c64449f735ab4d712e6a4df7aca7910bb856ed44e22bf2bf75768d549eb",
+    "planned_steps": 61000,
+    "planned_tokens": 1998848000,
+    "run": "unit_bf16_s7",
+    "seed": 7,
+    "status": "nonfinite"
+  },
+  "attention_operand_quantization": {
+    "block": null,
+    "emulated": false,
+    "format": null,
+    "ops": null
+  },
+  "compile": false,
+  "init": "unit",
+  "initialization_diagnostics": {
+    "R_init": 30.1496,
+    "normalizer_radii": {
+      "layers.0.attn.kv_norm": 16.032,
+      "layers.0.attn_norm": 30.15,
+      "layers.0.ffn_norm": 30.15,
+      "layers.1.attn.kv_norm": 16.031,
+      "layers.1.attn_norm": 30.15,
+      "layers.1.ffn_norm": 30.15,
+      "layers.10.attn.kv_norm": 16.031,
+      "layers.10.attn_norm": 30.15,
+      "layers.10.ffn_norm": 30.15,
+      "layers.11.attn.kv_norm": 16.031,
+      "layers.11.attn_norm": 30.15,
+      "layers.11.ffn_norm": 30.15,
+      "layers.12.attn.kv_norm": 16.031,
+      "layers.12.attn_norm": 30.15,
+      "layers.12.ffn_norm": 30.15,
+      "layers.13.attn.kv_norm": 16.031,
+      "layers.13.attn_norm": 30.15,
+      "layers.13.ffn_norm": 30.15,
+      "layers.14.attn.kv_norm": 16.031,
+      "layers.14.attn_norm": 30.15,
+      "layers.14.ffn_norm": 30.15,
+      "layers.15.attn.kv_norm": 16.031,
+      "layers.15.attn_norm": 30.15,
+      "layers.15.ffn_norm": 30.15,
+      "layers.2.attn.kv_norm": 16.031,
+      "layers.2.attn_norm": 30.15,
+      "layers.2.ffn_norm": 30.15,
+      "layers.3.attn.kv_norm": 16.031,
+      "layers.3.attn_norm": 30.15,
+      "layers.3.ffn_norm": 30.15,
+      "layers.4.attn.kv_norm": 16.031,
+      "layers.4.attn_norm": 30.15,
+      "layers.4.ffn_norm": 30.15,
+      "layers.5.attn.kv_norm": 16.031,
+      "layers.5.attn_norm": 30.15,
+      "layers.5.ffn_norm": 30.15,
+      "layers.6.attn.kv_norm": 16.031,
+      "layers.6.attn_norm": 30.15,
+      "layers.6.ffn_norm": 30.15,
+      "layers.7.attn.kv_norm": 16.031,
+      "layers.7.attn_norm": 30.15,
+      "layers.7.ffn_norm": 30.15,
+      "layers.8.attn.kv_norm": 16.031,
+      "layers.8.attn_norm": 30.15,
+      "layers.8.ffn_norm": 30.15,
+      "layers.9.attn.kv_norm": 16.031,
+      "layers.9.attn_norm": 30.15,
+      "layers.9.ffn_norm": 30.15,
+      "norm": 30.15
+    },
+    "u_init": 30.1496
+  },
+  "model": {
+    "curvature": 1.0,
+    "dim": 910,
+    "heads": 14,
+    "layers": 16,
+    "unique_parameter_elements": 966957719,
+    "vocab_size": 128256
+  },
+  "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8",
+  "parameter_dtype": "float32",
+  "policy": false,
+  "precision": "bf16",
+  "recipe": {
+    "accumulation": 2,
+    "eval_every": 2000,
+    "final_lr": 0.0001,
+    "lr": 0.001,
+    "microbatch": 1,
+    "save_every": 500,
+    "steps": 61000,
+    "tokens_per_step": 32768,
+    "warmup_steps": 1220,
+    "world": 8
+  },
+  "run": "unit_bf16_s7",
+  "schema": "helm.c42b.public-profile.v1",
+  "seed": 7,
+  "source": {
+    "helm_revision": "7501deca8f413848bfef804be64ce874b72a3cd7",
+    "lowp_pr_revision": "5edb20302017712ac97bf39374f66db80f94bc51",
+    "native_trainer_revision": "5f968bd8d023298728e3ac43d6fefe64f786de8e"
+  }
+}

profiles/unit_bf16_s99.json

@@ -0,0 +1,121 @@
+{
+  "actual_outcome": {
+    "actual_terminal_step": 61000,
+    "attention_operand_format": null,
+    "evaluation_points": 31,
+    "final_c4_ppl": 42.14531938222052,
+    "final_wikitext_ppl": 150.1752538901007,
+    "first_nonfinite_step": null,
+    "full_budget_completed": true,
+    "initialization": "unit",
+    "last_finite_evaluation": {
+      "c4_ppl": 42.14531938222052,
+      "step": 61000,
+      "wikitext_ppl": 150.1752538901007
+    },
+    "last_finite_saved_step": null,
+    "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+    "native_json": "cells.d/unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8.json",
+    "native_json_sha256": "d396be378862680c3c53f90e0d7b29f77fbf178d1364d5af364b9a8f91202428",
+    "planned_steps": 61000,
+    "planned_tokens": 1998848000,
+    "run": "unit_bf16_s99",
+    "seed": 99,
+    "status": "finished"
+  },
+  "attention_operand_quantization": {
+    "block": null,
+    "emulated": false,
+    "format": null,
+    "ops": null
+  },
+  "compile": false,
+  "init": "unit",
+  "initialization_diagnostics": {
+    "R_init": 30.1496,
+    "normalizer_radii": {
+      "layers.0.attn.kv_norm": 16.031,
+      "layers.0.attn_norm": 30.15,
+      "layers.0.ffn_norm": 30.15,
+      "layers.1.attn.kv_norm": 16.031,
+      "layers.1.attn_norm": 30.15,
+      "layers.1.ffn_norm": 30.15,
+      "layers.10.attn.kv_norm": 16.031,
+      "layers.10.attn_norm": 30.15,
+      "layers.10.ffn_norm": 30.15,
+      "layers.11.attn.kv_norm": 16.031,
+      "layers.11.attn_norm": 30.15,
+      "layers.11.ffn_norm": 30.15,
+      "layers.12.attn.kv_norm": 16.031,
+      "layers.12.attn_norm": 30.15,
+      "layers.12.ffn_norm": 30.15,
+      "layers.13.attn.kv_norm": 16.031,
+      "layers.13.attn_norm": 30.15,
+      "layers.13.ffn_norm": 30.15,
+      "layers.14.attn.kv_norm": 16.031,
+      "layers.14.attn_norm": 30.15,
+      "layers.14.ffn_norm": 30.15,
+      "layers.15.attn.kv_norm": 16.031,
+      "layers.15.attn_norm": 30.15,
+      "layers.15.ffn_norm": 30.15,
+      "layers.2.attn.kv_norm": 16.031,
+      "layers.2.attn_norm": 30.15,
+      "layers.2.ffn_norm": 30.15,
+      "layers.3.attn.kv_norm": 16.031,
+      "layers.3.attn_norm": 30.15,
+      "layers.3.ffn_norm": 30.15,
+      "layers.4.attn.kv_norm": 16.031,
+      "layers.4.attn_norm": 30.15,
+      "layers.4.ffn_norm": 30.15,
+      "layers.5.attn.kv_norm": 16.031,
+      "layers.5.attn_norm": 30.15,
+      "layers.5.ffn_norm": 30.15,
+      "layers.6.attn.kv_norm": 16.031,
+      "layers.6.attn_norm": 30.15,
+      "layers.6.ffn_norm": 30.15,
+      "layers.7.attn.kv_norm": 16.031,
+      "layers.7.attn_norm": 30.15,
+      "layers.7.ffn_norm": 30.15,
+      "layers.8.attn.kv_norm": 16.031,
+      "layers.8.attn_norm": 30.15,
+      "layers.8.ffn_norm": 30.15,
+      "layers.9.attn.kv_norm": 16.031,
+      "layers.9.attn_norm": 30.15,
+      "layers.9.ffn_norm": 30.15,
+      "norm": 30.15
+    },
+    "u_init": 30.1496
+  },
+  "model": {
+    "curvature": 1.0,
+    "dim": 910,
+    "heads": 14,
+    "layers": 16,
+    "unique_parameter_elements": 966957719,
+    "vocab_size": 128256
+  },
+  "native_cell_key": "unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8",
+  "parameter_dtype": "float32",
+  "policy": false,
+  "precision": "bf16",
+  "recipe": {
+    "accumulation": 2,
+    "eval_every": 2000,
+    "final_lr": 0.0001,
+    "lr": 0.001,
+    "microbatch": 1,
+    "save_every": 500,
+    "steps": 61000,
+    "tokens_per_step": 32768,
+    "warmup_steps": 1220,
+    "world": 8
+  },
+  "run": "unit_bf16_s99",
+  "schema": "helm.c42b.public-profile.v1",
+  "seed": 99,
+  "source": {
+    "helm_revision": "7501deca8f413848bfef804be64ce874b72a3cd7",
+    "lowp_pr_revision": "5edb20302017712ac97bf39374f66db80f94bc51",
+    "native_trainer_revision": "5f968bd8d023298728e3ac43d6fefe64f786de8e"
+  }
+}

scientific_logs/scaled4.log

@@ -0,0 +1,108 @@
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] cinner and lorentzian_centroid of attention: operands rounded to e2m1, power-of-two scale per block of 32
+[cfg] scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32
+[data] corpus c42b (c4_2b_tokens.pt)
+[data] 1999.48M training tokens; this cell processes 1998.8M, 1.00 of the corpus
+[ood] 256 sequences from wikitext_train_tokens.pt
+[lowp] first rounded cinner: q (4, 14, 2048, 130) torch.float32, k (4, 14, 2048, 130)
+[lowp] first rounded centroid: values (4, 14, 2048, 65) torch.float32, weights (4, 14, 2048, 2048) torch.float32
+[sched] warmup 1220 steps to 0.001, cosine to 0.0001 at step 61000
+[health] steps 1-500: loss 9.7027  grad-norm mean 0.638 max 4.574  clipped 10%
+[resume] saved at step 500
+[health] steps 501-1000: loss 6.4858  grad-norm mean 0.998 max 2.863  clipped 40%
+[resume] saved at step 1000
+[health] steps 1001-1500: loss 5.7720  grad-norm mean 0.944 max 2.586  clipped 35%
+[resume] saved at step 1500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step  2000  ppl     202.9  ood     727.7  (31m train)
+[health] steps 1501-2000: loss 5.4105  grad-norm mean 0.826 max 12.229  clipped 14%
+[resume] saved at step 2000
+[health] steps 2001-2500: loss 5.6805  grad-norm mean 511.654 max 45749.133  clipped 96%
+[resume] saved at step 2500
+[health] steps 2501-3000: loss 5.9166  grad-norm mean 176220.096 max 74396832.000  clipped 100%
+[resume] saved at step 3000
+[health] steps 3001-3500: loss 6.0195  grad-norm mean 1298834.048 max 402409728.000  clipped 100%
+[resume] saved at step 3500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step  4000  ppl     423.7  ood    1275.3  (65m train)
+[health] steps 3501-4000: loss 6.5126  grad-norm mean 52333150.208 max 3462804224.000  clipped 100%
+[resume] saved at step 4000
+[health] steps 4001-4500: loss 6.8108  grad-norm mean 97818525.696 max 38629756928.000  clipped 100%
+[resume] saved at step 4500
+[health] steps 4501-5000: loss 6.4227  grad-norm mean 542141.824 max 67260152.000  clipped 100%
+[resume] saved at step 5000
+[health] steps 5001-5500: loss 6.3261  grad-norm mean 475369.824 max 37219236.000  clipped 100%
+[resume] saved at step 5500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step  6000  ppl    1489.0  ood    2830.9  (98m train)
+[health] steps 5501-6000: loss 6.5516  grad-norm mean 76535504.896 max 10348393472.000  clipped 100%
+[resume] saved at step 6000
+[health] steps 6001-6500: loss 6.7598  grad-norm mean 15476954.112 max 2227032576.000  clipped 100%
+[resume] saved at step 6500
+[health] steps 6501-7000: loss 6.2951  grad-norm mean 19520233.472 max 4598260224.000  clipped 100%
+[resume] saved at step 7000
+[health] steps 7001-7500: loss 6.2324  grad-norm mean 401158111.232 max 78418305024.000  clipped 100%
+[resume] saved at step 7500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step  8000  ppl    2921.7  ood    6316.5  (130m train)
+[health] steps 7501-8000: loss 7.0267  grad-norm mean 142769815.552 max 21067493376.000  clipped 100%
+[resume] saved at step 8000
+[health] steps 8001-8500: loss 7.8348  grad-norm mean 44360120926.208 max 21826317058048.000  clipped 100%
+[resume] saved at step 8500
+[health] steps 8501-9000: loss 7.5718  grad-norm mean 4380696.064 max 403556960.000  clipped 100%
+[resume] saved at step 9000
+[health] steps 9001-9500: loss 7.5329  grad-norm mean 6421007.360 max 2302962944.000  clipped 100%
+[resume] saved at step 9500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step 10000  ppl    2477.4  ood    4583.3  (158m train)
+[health] steps 9501-10000: loss 7.7485  grad-norm mean 1609620914.176 max 104182497280.000  clipped 100%
+[resume] saved at step 10000
+[health] steps 10001-10500: loss 7.8401  grad-norm mean 9454984626.176 max 3724470124544.000  clipped 100%
+[resume] saved at step 10500
+[health] steps 10501-11000: loss 7.5425  grad-norm mean 1184740016.128 max 467213385728.000  clipped 100%
+[resume] saved at step 11000
+[health] steps 11001-11500: loss 7.2397  grad-norm mean 192504086.528 max 29852839936.000  clipped 100%
+[resume] saved at step 11500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step 12000  ppl    1390.7  ood    3173.9  (187m train)
+[health] steps 11501-12000: loss 7.2025  grad-norm mean 981352120.320 max 118586875904.000  clipped 100%
+[resume] saved at step 12000
+[health] steps 12001-12500: loss 7.0402  grad-norm mean 44174032.896 max 8867602432.000  clipped 100%
+[resume] saved at step 12500
+[health] steps 12501-13000: loss 6.8783  grad-norm mean 1848472.064 max 434353472.000  clipped 100%
+[resume] saved at step 13000
+[health] steps 13001-13500: loss 6.9460  grad-norm mean 621718.016 max 103816272.000  clipped 100%
+[resume] saved at step 13500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step 14000  ppl     819.6  ood    2152.9  (218m train)
+[health] steps 13501-14000: loss 6.8234  grad-norm mean 5290834.944 max 960473664.000  clipped 100%
+[resume] saved at step 14000
+[health] steps 14001-14500: loss 6.5601  grad-norm mean 1377414.912 max 465742496.000  clipped 100%
+[resume] saved at step 14500
+[health] steps 14501-15000: loss 6.4952  grad-norm mean 11370491805.696 max 5675421270016.000  clipped 100%
+[resume] saved at step 15000
+[health] steps 15001-15500: loss 6.9291  grad-norm mean 18377663840.256 max 3771178418176.000  clipped 100%
+[resume] saved at step 15500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step 16000  ppl    1377.5  ood    2871.0  (249m train)
+[health] steps 15501-16000: loss 7.0357  grad-norm mean 7124707035840.512 max 3476155403337728.000  clipped 100%
+[resume] saved at step 16000
+[health] steps 16001-16500: loss 7.2548  grad-norm mean 668064915914.752 max 253003755945984.000  clipped 100%
+[resume] saved at step 16500
+[health] steps 16501-17000: loss 7.2793  grad-norm mean 17794508783.616 max 3162229178368.000  clipped 100%
+[resume] saved at step 17000
+[health] steps 17001-17500: loss 7.3164  grad-norm mean 2411633311.744 max 1038175633408.000  clipped 100%
+[resume] saved at step 17500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32] step 18000  ppl    1283.7  ood    2968.8  (279m train)
+[health] steps 17501-18000: loss 7.1763  grad-norm mean 3796626571.264 max 923216904192.000  clipped 100%
+[resume] saved at step 18000
+[health] steps 18001-18500: loss 7.1747  grad-norm mean 6283812405.248 max 2384119988224.000  clipped 100%
+[resume] saved at step 18500
+[health] steps 18501-19000: loss 6.9052  grad-norm mean 9985686.528 max 2451557120.000  clipped 100%
+[resume] saved at step 19000
+[health] steps 19001-19500: loss 6.5140  grad-norm mean 341344.224 max 33471126.000  clipped 100%
+[resume] saved at step 19500
+[nonfinite] loss or gradient norm not finite, first at step 19576 (seen at step 19600); last resume file written at step 19500
+[health] steps 19501-19600: loss 6.3348  grad-norm mean 26860.602 max 1098950.125  clipped 100%
+[done] scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32: NONFINITE first at step 19576; state from step 19500 kept in prenan_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32.pt, 305 min
+[lowp] calls with rounded operands: {'cinner': 636432, 'centroid': 636432}
+[lowp] calls with rounded operands: {'cinner': 636432, 'centroid': 636432}

scientific_logs/scaled8.log

@@ -0,0 +1,189 @@
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] cinner and lorentzian_centroid of attention: operands rounded to e4m3, power-of-two scale per block of 32
+[cfg] scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32
+[data] corpus c42b (c4_2b_tokens.pt)
+[data] 1999.48M training tokens; this cell processes 1998.8M, 1.00 of the corpus
+[ood] 256 sequences from wikitext_train_tokens.pt
+[lowp] first rounded cinner: q (4, 14, 2048, 130) torch.float32, k (4, 14, 2048, 130)
+[lowp] first rounded centroid: values (4, 14, 2048, 65) torch.float32, weights (4, 14, 2048, 2048) torch.float32
+[sched] warmup 1220 steps to 0.001, cosine to 0.0001 at step 61000
+[health] steps 1-500: loss 9.7014  grad-norm mean 0.611 max 3.143  clipped 8%
+[resume] saved at step 500
+[health] steps 501-1000: loss 6.4802  grad-norm mean 0.990 max 2.741  clipped 38%
+[resume] saved at step 1000
+[health] steps 1001-1500: loss 5.6738  grad-norm mean 0.977 max 2.232  clipped 38%
+[resume] saved at step 1500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step  2000  ppl     174.7  ood     610.9  (29m train)
+[health] steps 1501-2000: loss 5.2693  grad-norm mean 0.748 max 1.722  clipped 10%
+[resume] saved at step 2000
+[health] steps 2001-2500: loss 5.0838  grad-norm mean 0.660 max 1.253  clipped 2%
+[resume] saved at step 2500
+[health] steps 2501-3000: loss 4.9109  grad-norm mean 0.613 max 0.961  clipped 0%
+[resume] saved at step 3000
+[health] steps 3001-3500: loss 4.7996  grad-norm mean 0.574 max 0.998  clipped 0%
+[resume] saved at step 3500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step  4000  ppl      96.7  ood     278.6  (61m train)
+[health] steps 3501-4000: loss 4.6850  grad-norm mean 0.546 max 1.361  clipped 0%
+[resume] saved at step 4000
+[health] steps 4001-4500: loss 4.5061  grad-norm mean 0.520 max 0.765  clipped 0%
+[resume] saved at step 4500
+[health] steps 4501-5000: loss 4.3982  grad-norm mean 0.500 max 1.114  clipped 0%
+[resume] saved at step 5000
+[health] steps 5001-5500: loss 4.3312  grad-norm mean 0.478 max 0.961  clipped 0%
+[resume] saved at step 5500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step  6000  ppl      72.0  ood     210.9  (94m train)
+[health] steps 5501-6000: loss 4.2797  grad-norm mean 0.465 max 0.880  clipped 0%
+[resume] saved at step 6000
+[health] steps 6001-6500: loss 4.2510  grad-norm mean 0.456 max 1.546  clipped 0%
+[resume] saved at step 6500
+[health] steps 6501-7000: loss 4.1993  grad-norm mean 0.454 max 1.891  clipped 0%
+[resume] saved at step 7000
+[health] steps 7001-7500: loss 4.1884  grad-norm mean 0.451 max 1.977  clipped 0%
+[resume] saved at step 7500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step  8000  ppl      63.2  ood     181.1  (127m train)
+[health] steps 7501-8000: loss 4.1425  grad-norm mean 0.452 max 1.302  clipped 0%
+[resume] saved at step 8000
+[health] steps 8001-8500: loss 4.1160  grad-norm mean 0.455 max 1.074  clipped 0%
+[resume] saved at step 8500
+[health] steps 8501-9000: loss 4.1120  grad-norm mean 0.473 max 9.420  clipped 0%
+[resume] saved at step 9000
+[health] steps 9001-9500: loss 4.0714  grad-norm mean 0.458 max 1.143  clipped 0%
+[resume] saved at step 9500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 10000  ppl      58.6  ood     178.9  (160m train)
+[health] steps 9501-10000: loss 4.0591  grad-norm mean 0.467 max 2.614  clipped 1%
+[resume] saved at step 10000
+[health] steps 10001-10500: loss 4.0487  grad-norm mean 0.557 max 32.494  clipped 2%
+[resume] saved at step 10500
+[health] steps 10501-11000: loss 4.0460  grad-norm mean 1.536 max 308.973  clipped 10%
+[resume] saved at step 11000
+[health] steps 11001-11500: loss 4.1673  grad-norm mean 10.734 max 1052.016  clipped 53%
+[resume] saved at step 11500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 12000  ppl     184.7  ood     604.5  (192m train)
+[health] steps 11501-12000: loss 5.0044  grad-norm mean 461.940 max 41910.043  clipped 100%
+[resume] saved at step 12000
+[health] steps 12001-12500: loss 5.3022  grad-norm mean 4680.476 max 841680.438  clipped 100%
+[resume] saved at step 12500
+[health] steps 12501-13000: loss 5.9042  grad-norm mean 1014261.056 max 270517312.000  clipped 100%
+[resume] saved at step 13000
+[health] steps 13001-13500: loss 5.7568  grad-norm mean 9639.093 max 1885640.625  clipped 99%
+[resume] saved at step 13500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 14000  ppl     137.9  ood     454.6  (223m train)
+[health] steps 13501-14000: loss 5.1947  grad-norm mean 1278.051 max 536246.062  clipped 97%
+[resume] saved at step 14000
+[health] steps 14001-14500: loss 4.8210  grad-norm mean 191.150 max 15655.746  clipped 100%
+[resume] saved at step 14500
+[health] steps 14501-15000: loss 4.6650  grad-norm mean 611.571 max 38539.738  clipped 100%
+[resume] saved at step 15000
+[health] steps 15001-15500: loss 4.6244  grad-norm mean 851.901 max 19325.949  clipped 100%
+[resume] saved at step 15500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 16000  ppl     169.9  ood     513.6  (255m train)
+[health] steps 15501-16000: loss 5.1716  grad-norm mean 220322.144 max 69922816.000  clipped 100%
+[resume] saved at step 16000
+[health] steps 16001-16500: loss 5.0968  grad-norm mean 45862.900 max 2904747.000  clipped 100%
+[resume] saved at step 16500
+[health] steps 16501-17000: loss 4.6144  grad-norm mean 2933.390 max 565594.688  clipped 100%
+[resume] saved at step 17000
+[health] steps 17001-17500: loss 4.5892  grad-norm mean 1432.044 max 154115.094  clipped 100%
+[resume] saved at step 17500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 18000  ppl     100.1  ood     253.8  (285m train)
+[health] steps 17501-18000: loss 4.5908  grad-norm mean 1931.268 max 117779.039  clipped 100%
+[resume] saved at step 18000
+[health] steps 18001-18500: loss 4.6279  grad-norm mean 7236.754 max 314153.250  clipped 100%
+[resume] saved at step 18500
+[health] steps 18501-19000: loss 4.5665  grad-norm mean 7100.872 max 991551.688  clipped 100%
+[resume] saved at step 19000
+[health] steps 19001-19500: loss 4.6523  grad-norm mean 4891.302 max 240722.531  clipped 100%
+[resume] saved at step 19500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 20000  ppl     133.8  ood     429.1  (316m train)
+[health] steps 19501-20000: loss 4.6961  grad-norm mean 26726.336 max 3621682.250  clipped 100%
+[resume] saved at step 20000
+[health] steps 20001-20500: loss 4.5920  grad-norm mean 2473.426 max 126145.273  clipped 100%
+[resume] saved at step 20500
+[health] steps 20501-21000: loss 4.5865  grad-norm mean 7740.638 max 278614.906  clipped 100%
+[resume] saved at step 21000
+[health] steps 21001-21500: loss 4.7668  grad-norm mean 96743.416 max 11756970.000  clipped 100%
+[resume] saved at step 21500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 22000  ppl      88.1  ood     227.0  (346m train)
+[health] steps 21501-22000: loss 4.5791  grad-norm mean 5707.725 max 316491.719  clipped 100%
+[resume] saved at step 22000
+[health] steps 22001-22500: loss 4.5040  grad-norm mean 17389.226 max 2030029.000  clipped 100%
+[resume] saved at step 22500
+[health] steps 22501-23000: loss 4.4457  grad-norm mean 6945.707 max 1344003.750  clipped 100%
+[resume] saved at step 23000
+[health] steps 23001-23500: loss 4.6888  grad-norm mean 98770.912 max 42497904.000  clipped 100%
+[resume] saved at step 23500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 24000  ppl     109.3  ood     302.6  (378m train)
+[health] steps 23501-24000: loss 4.7402  grad-norm mean 70396.968 max 11657271.000  clipped 100%
+[resume] saved at step 24000
+[health] steps 24001-24500: loss 4.7429  grad-norm mean 67652.272 max 9392578.000  clipped 100%
+[resume] saved at step 24500
+[health] steps 24501-25000: loss 4.6507  grad-norm mean 37201.272 max 2190922.500  clipped 100%
+[resume] saved at step 25000
+[health] steps 25001-25500: loss 4.6586  grad-norm mean 137704.464 max 9793316.000  clipped 100%
+[resume] saved at step 25500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 26000  ppl     103.2  ood     249.4  (408m train)
+[health] steps 25501-26000: loss 4.5900  grad-norm mean 44320.172 max 7772814.000  clipped 100%
+[resume] saved at step 26000
+[health] steps 26001-26500: loss 4.4964  grad-norm mean 8890.997 max 737653.000  clipped 100%
+[resume] saved at step 26500
+[health] steps 26501-27000: loss 4.5432  grad-norm mean 15652.788 max 3759203.250  clipped 100%
+[resume] saved at step 27000
+[health] steps 27001-27500: loss 4.5528  grad-norm mean 16914.748 max 1196604.375  clipped 100%
+[resume] saved at step 27500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 28000  ppl      78.4  ood     189.3  (439m train)
+[health] steps 27501-28000: loss 4.4514  grad-norm mean 11789.215 max 2854084.000  clipped 100%
+[resume] saved at step 28000
+[health] steps 28001-28500: loss 4.3895  grad-norm mean 1443.749 max 95537.039  clipped 100%
+[resume] saved at step 28500
+[health] steps 28501-29000: loss 4.4212  grad-norm mean 1729.430 max 85023.133  clipped 100%
+[resume] saved at step 29000
+[health] steps 29001-29500: loss 4.3516  grad-norm mean 394.430 max 33271.633  clipped 100%
+[resume] saved at step 29500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 30000  ppl      74.9  ood     195.6  (471m train)
+[health] steps 29501-30000: loss 4.3481  grad-norm mean 685.311 max 41030.555  clipped 100%
+[resume] saved at step 30000
+[health] steps 30001-30500: loss 4.3093  grad-norm mean 355.514 max 61418.977  clipped 99%
+[resume] saved at step 30500
+[health] steps 30501-31000: loss 4.3154  grad-norm mean 550.734 max 178044.500  clipped 99%
+[resume] saved at step 31000
+[health] steps 31001-31500: loss 4.2759  grad-norm mean 244.727 max 14119.308  clipped 98%
+[resume] saved at step 31500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 32000  ppl      70.9  ood     182.0  (503m train)
+[health] steps 31501-32000: loss 4.2640  grad-norm mean 960.290 max 178336.344  clipped 99%
+[resume] saved at step 32000
+[health] steps 32001-32500: loss 4.2232  grad-norm mean 2247.042 max 881971.250  clipped 100%
+[resume] saved at step 32500
+[health] steps 32501-33000: loss 4.2433  grad-norm mean 5435.551 max 709869.625  clipped 100%
+[resume] saved at step 33000
+[health] steps 33001-33500: loss 4.2045  grad-norm mean 17925.334 max 2200165.250  clipped 100%
+[resume] saved at step 33500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 34000  ppl      72.7  ood     194.8  (536m train)
+[health] steps 33501-34000: loss 4.2324  grad-norm mean 650.642 max 26031.838  clipped 100%
+[resume] saved at step 34000
+[health] steps 34001-34500: loss 4.2475  grad-norm mean 322.488 max 28030.752  clipped 98%
+[resume] saved at step 34500
+[health] steps 34501-35000: loss 4.2041  grad-norm mean 2849.214 max 1364074.625  clipped 94%
+[resume] saved at step 35000
+[health] steps 35001-35500: loss 4.2382  grad-norm mean 251.112 max 25658.379  clipped 98%
+[resume] saved at step 35500
+[scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32] step 36000  ppl      64.2  ood     178.5  (569m train)
+[health] steps 35501-36000: loss 4.2298  grad-norm mean 121.190 max 5135.478  clipped 100%
+[resume] saved at step 36000
+[health] steps 36001-36500: loss 4.2358  grad-norm mean 1492.367 max 140415.734  clipped 100%
+[resume] saved at step 36500
+[health] steps 36501-37000: loss 4.3222  grad-norm mean 1445.754 max 349021.250  clipped 100%
+[resume] saved at step 37000
+[health] steps 37001-37500: loss 4.1971  grad-norm mean 325.861 max 35151.117  clipped 98%
+[resume] saved at step 37500
+[nonfinite] loss or gradient norm not finite, first at step 37785 (seen at step 37800); last resume file written at step 37500
+[health] steps 37501-37800: loss 4.1911  grad-norm mean 98.319 max 5155.610  clipped 95%
+[done] scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32: NONFINITE first at step 37785; state from step 37500 kept in prenan_scaled_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32.pt, 599 min
+[lowp] calls with rounded operands: {'cinner': 1228048, 'centroid': 1228048}
+[lowp] calls with rounded operands: {'cinner': 1228048, 'centroid': 1228048}

scientific_logs/unit4.log

@@ -0,0 +1,21 @@
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] train_cell.py with 4-bit attention operands: format e2m1, block 32, operations both, key suffix _f4e2m1b32
+[lowp] cinner and lorentzian_centroid of attention: operands rounded to e2m1, power-of-two scale per block of 32
+[cfg] unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32
+[data] corpus c42b (c4_2b_tokens.pt)
+[data] 1999.48M training tokens; this cell processes 1998.8M, 1.00 of the corpus
+[ood] 256 sequences from wikitext_train_tokens.pt
+[lowp] first rounded cinner: q (4, 14, 2048, 130) torch.float32, k (4, 14, 2048, 130)
+[lowp] first rounded centroid: values (4, 14, 2048, 65) torch.float32, weights (4, 14, 2048, 2048) torch.float32
+[sched] warmup 1220 steps to 0.001, cosine to 0.0001 at step 61000
+[nonfinite] loss or gradient norm not finite, first at step 94 (seen at step 100); last resume file written at step 0
+[health] steps 1-100: loss 12.0917  grad-norm mean 1666162159726352128.000 max 17868244826848231424.000  clipped 100%
+[done] unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f4e2m1b32: NONFINITE first at step 94; no resume file had been written yet, 2 min
+[lowp] calls with rounded operands: {'cinner': 3216, 'centroid': 3216}
+[lowp] calls with rounded operands: {'cinner': 3216, 'centroid': 3216}

scientific_logs/unit8.log

@@ -0,0 +1,21 @@
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] train_cell.py with 8-bit attention operands: format e4m3, block 32, operations both, key suffix _f8e4m3b32
+[lowp] cinner and lorentzian_centroid of attention: operands rounded to e4m3, power-of-two scale per block of 32
+[cfg] unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32
+[data] corpus c42b (c4_2b_tokens.pt)
+[data] 1999.48M training tokens; this cell processes 1998.8M, 1.00 of the corpus
+[ood] 256 sequences from wikitext_train_tokens.pt
+[lowp] first rounded cinner: q (4, 14, 2048, 130) torch.float32, k (4, 14, 2048, 130)
+[lowp] first rounded centroid: values (4, 14, 2048, 65) torch.float32, weights (4, 14, 2048, 2048) torch.float32
+[sched] warmup 1220 steps to 0.001, cosine to 0.0001 at step 61000
+[nonfinite] loss or gradient norm not finite, first at step 470 (seen at step 500); last resume file written at step 0
+[health] steps 1-500: loss 12.0859  grad-norm mean 68915436589917096.000 max 4652229410190000128.000  clipped 100%
+[done] unit_bf16_d910_c1_lr0.001_acc2_s1234_st61000_ev2000_corpc42b_perm_cos_w8_f8e4m3b32: NONFINITE first at step 470; no resume file had been written yet, 7 min
+[lowp] calls with rounded operands: {'cinner': 16016, 'centroid': 16016}
+[lowp] calls with rounded operands: {'cinner': 16016, 'centroid': 16016}

scientific_logs/unit_bf16_s7.log

@@ -0,0 +1,102 @@
+[cfg] unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8
+[data] corpus c42b (c4_2b_tokens.pt)
+[data] 1999.48M training tokens; this cell processes 1998.8M, 1.00 of the corpus
+[ood] 256 sequences from wikitext_train_tokens.pt
+[sched] warmup 1220 steps to 0.001, cosine to 0.0001 at step 61000
+[health] steps 1-500: loss 8.2741  grad-norm mean 279988968357.888 max 48261033361408.000  clipped 79%
+[resume] saved at step 500
+[health] steps 501-1000: loss 7.8433  grad-norm mean 5975770930171871.000 max 1304709634577137664.000  clipped 100%
+[resume] saved at step 1000
+[health] steps 1001-1500: loss 7.8800  grad-norm mean 2290687066058522.500 max 731806901901197312.000  clipped 100%
+[resume] saved at step 1500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step  2000  ppl    3081.7  ood    5205.4  (20m train)
+[health] steps 1501-2000: loss 7.9409  grad-norm mean 3525511466696311.000 max 486615602748719104.000  clipped 100%
+[resume] saved at step 2000
+[health] steps 2001-2500: loss 7.9134  grad-norm mean 73541376389677.062 max 33477262175109120.000  clipped 100%
+[resume] saved at step 2500
+[health] steps 2501-3000: loss 7.8792  grad-norm mean 4096278887661.568 max 672137216000000.000  clipped 100%
+[resume] saved at step 3000
+[health] steps 3001-3500: loss 7.9836  grad-norm mean 89853490381193.219 max 14693937818107904.000  clipped 100%
+[resume] saved at step 3500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step  4000  ppl    3725.4  ood    6150.9  (43m train)
+[health] steps 3501-4000: loss 8.1615  grad-norm mean 18353447522795.520 max 6111865163743232.000  clipped 100%
+[resume] saved at step 4000
+[health] steps 4001-4500: loss 8.3271  grad-norm mean 13516694798467.072 max 3735046905135104.000  clipped 100%
+[resume] saved at step 4500
+[health] steps 4501-5000: loss 8.6039  grad-norm mean 2092514911715.328 max 214209128300544.000  clipped 100%
+[resume] saved at step 5000
+[health] steps 5001-5500: loss 8.7694  grad-norm mean 1035891418071.040 max 238156959973376.000  clipped 100%
+[resume] saved at step 5500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step  6000  ppl    6292.4  ood    9973.1  (66m train)
+[health] steps 5501-6000: loss 8.6709  grad-norm mean 209231848407.040 max 44125311205376.000  clipped 100%
+[resume] saved at step 6000
+[health] steps 6001-6500: loss 8.7985  grad-norm mean 134661242093.568 max 12161647640576.000  clipped 100%
+[resume] saved at step 6500
+[health] steps 6501-7000: loss 8.9280  grad-norm mean 156913174052.864 max 35240881946624.000  clipped 100%
+[resume] saved at step 7000
+[health] steps 7001-7500: loss 8.9580  grad-norm mean 1351027328548.864 max 595335684554752.000  clipped 100%
+[resume] saved at step 7500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step  8000  ppl    7223.0  ood   11966.0  (90m train)
+[health] steps 7501-8000: loss 8.8752  grad-norm mean 46685258514.432 max 9556080984064.000  clipped 100%
+[resume] saved at step 8000
+[health] steps 8001-8500: loss 8.9012  grad-norm mean 1064509120708.608 max 226382458126336.000  clipped 100%
+[resume] saved at step 8500
+[health] steps 8501-9000: loss 8.8973  grad-norm mean 302028072943.616 max 146165672706048.000  clipped 100%
+[resume] saved at step 9000
+[health] steps 9001-9500: loss 8.8310  grad-norm mean 12558347010.048 max 1865004744704.000  clipped 100%
+[resume] saved at step 9500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step 10000  ppl    3740.6  ood    7218.2  (115m train)
+[health] steps 9501-10000: loss 8.5071  grad-norm mean 1132090359.808 max 304874225664.000  clipped 100%
+[resume] saved at step 10000
+[health] steps 10001-10500: loss 7.9842  grad-norm mean 17828536.320 max 2609804032.000  clipped 100%
+[resume] saved at step 10500
+[health] steps 10501-11000: loss 7.8464  grad-norm mean 2342629.376 max 242555216.000  clipped 100%
+[resume] saved at step 11000
+[health] steps 11001-11500: loss 7.8282  grad-norm mean 728957.952 max 23701910.000  clipped 100%
+[resume] saved at step 11500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step 12000  ppl    2544.8  ood    4938.9  (139m train)
+[health] steps 11501-12000: loss 7.8193  grad-norm mean 586352.832 max 128272416.000  clipped 100%
+[resume] saved at step 12000
+[health] steps 12001-12500: loss 7.8078  grad-norm mean 923206.848 max 135035104.000  clipped 100%
+[resume] saved at step 12500
+[health] steps 12501-13000: loss 7.8106  grad-norm mean 5223426.048 max 2275896320.000  clipped 100%
+[resume] saved at step 13000
+[health] steps 13001-13500: loss 7.8208  grad-norm mean 489591.456 max 76937816.000  clipped 100%
+[resume] saved at step 13500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step 14000  ppl    2546.1  ood    4870.6  (165m train)
+[health] steps 13501-14000: loss 7.8182  grad-norm mean 4563990.528 max 1242059392.000  clipped 100%
+[resume] saved at step 14000
+[health] steps 14001-14500: loss 7.8242  grad-norm mean 1906761.728 max 546927040.000  clipped 100%
+[resume] saved at step 14500
+[health] steps 14501-15000: loss 7.8063  grad-norm mean 528363.680 max 64105964.000  clipped 100%
+[resume] saved at step 15000
+[health] steps 15001-15500: loss 7.8116  grad-norm mean 2287449.856 max 744769856.000  clipped 100%
+[resume] saved at step 15500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step 16000  ppl    2555.3  ood    4668.3  (192m train)
+[health] steps 15501-16000: loss 7.8181  grad-norm mean 89636.696 max 5071824.500  clipped 100%
+[resume] saved at step 16000
+[health] steps 16001-16500: loss 7.8105  grad-norm mean 302070.016 max 61053604.000  clipped 100%
+[resume] saved at step 16500
+[health] steps 16501-17000: loss 7.7890  grad-norm mean 213601.184 max 30829950.000  clipped 100%
+[resume] saved at step 17000
+[health] steps 17001-17500: loss 7.7641  grad-norm mean 530482.144 max 144564000.000  clipped 100%
+[resume] saved at step 17500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step 18000  ppl    2444.2  ood    4608.3  (218m train)
+[health] steps 17501-18000: loss 7.7736  grad-norm mean 35051.048 max 3545721.250  clipped 100%
+[resume] saved at step 18000
+[health] steps 18001-18500: loss 7.7783  grad-norm mean 79231.312 max 19613498.000  clipped 100%
+[resume] saved at step 18500
+[health] steps 18501-19000: loss 7.6963  grad-norm mean 94032.744 max 8377379.000  clipped 100%
+[resume] saved at step 19000
+[health] steps 19001-19500: loss 7.6139  grad-norm mean 16569.631 max 932935.875  clipped 100%
+[resume] saved at step 19500
+[unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8] step 20000  ppl    1971.1  ood    3768.6  (245m train)
+[health] steps 19501-20000: loss 7.5621  grad-norm mean 7749.368 max 1046033.375  clipped 100%
+[resume] saved at step 20000
+[health] steps 20001-20500: loss 7.4969  grad-norm mean 6382.753 max 902437.750  clipped 100%
+[resume] saved at step 20500
+[health] steps 20501-21000: loss 7.3791  grad-norm mean 3381.047 max 425152.406  clipped 100%
+[resume] saved at step 21000
+[nonfinite] loss or gradient norm not finite, first at step 21249 (seen at step 21300); last resume file written at step 21000
+[health] steps 21001-21300: loss 7.3324  grad-norm mean 2397.107 max 89643.586  clipped 100%
+[done] unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8: NONFINITE first at step 21249; state from step 21000 kept in prenan_unit_bf16_d910_c1_lr0.001_acc2_s7_st61000_ev2000_corpc42b_perm_cos_w8.pt, 263 min

scientific_logs/unit_bf16_s99.log

@@ -0,0 +1,280 @@
+[cfg] unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8
+[data] corpus c42b (c4_2b_tokens.pt)
+[data] 1999.48M training tokens; this cell processes 1998.8M, 1.00 of the corpus
+[ood] 256 sequences from wikitext_train_tokens.pt
+[sched] warmup 1220 steps to 0.001, cosine to 0.0001 at step 61000
+[health] steps 1-500: loss 8.2677  grad-norm mean 4029060218.880 max 1159352942592.000  clipped 79%
+[resume] saved at step 500
+[health] steps 501-1000: loss 7.8374  grad-norm mean 23309431004572156.000 max 7506468687163949056.000  clipped 100%
+[resume] saved at step 1000
+[health] steps 1001-1500: loss 7.8911  grad-norm mean 895374512256712.750 max 248232548314906624.000  clipped 100%
+[resume] saved at step 1500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step  2000  ppl    3147.4  ood    5423.8  (21m train)
+[health] steps 1501-2000: loss 7.9687  grad-norm mean 577116679708868.625 max 218582155687624704.000  clipped 100%
+[resume] saved at step 2000
+[health] steps 2001-2500: loss 8.0919  grad-norm mean 87295339140218.875 max 38324426973904896.000  clipped 100%
+[resume] saved at step 2500
+[health] steps 2501-3000: loss 7.9680  grad-norm mean 39968789562916.867 max 19767367936507904.000  clipped 100%
+[resume] saved at step 3000
+[health] steps 3001-3500: loss 7.9204  grad-norm mean 200710566182.912 max 38664566472704.000  clipped 100%
+[resume] saved at step 3500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step  4000  ppl    2707.6  ood    4847.1  (43m train)
+[health] steps 3501-4000: loss 7.8349  grad-norm mean 358843416.576 max 27402854400.000  clipped 100%
+[resume] saved at step 4000
+[health] steps 4001-4500: loss 7.8499  grad-norm mean 138364409872.384 max 50494122754048.000  clipped 100%
+[resume] saved at step 4500
+[health] steps 4501-5000: loss 7.8386  grad-norm mean 2303797231.616 max 319101075456.000  clipped 100%
+[resume] saved at step 5000
+[health] steps 5001-5500: loss 7.8155  grad-norm mean 1248364724.224 max 219623407616.000  clipped 100%
+[resume] saved at step 5500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step  6000  ppl    2575.2  ood    4918.8  (66m train)
+[health] steps 5501-6000: loss 7.8476  grad-norm mean 45163974.656 max 11680719872.000  clipped 100%
+[resume] saved at step 6000
+[health] steps 6001-6500: loss 7.8398  grad-norm mean 205707673.600 max 14566643712.000  clipped 100%
+[resume] saved at step 6500
+[health] steps 6501-7000: loss 7.9865  grad-norm mean 6965250.560 max 1029098112.000  clipped 100%
+[resume] saved at step 7000
+[health] steps 7001-7500: loss 7.8599  grad-norm mean 66527158.272 max 14402017280.000  clipped 100%
+[resume] saved at step 7500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step  8000  ppl    2622.7  ood    4914.7  (87m train)
+[health] steps 7501-8000: loss 7.8567  grad-norm mean 646149505.024 max 156670820352.000  clipped 100%
+[resume] saved at step 8000
+[health] steps 8001-8500: loss 7.8375  grad-norm mean 987057.536 max 129652760.000  clipped 100%
+[resume] saved at step 8500
+[health] steps 8501-9000: loss 7.7812  grad-norm mean 127671582.720 max 56667426816.000  clipped 100%
+[resume] saved at step 9000
+[health] steps 9001-9500: loss 7.7905  grad-norm mean 50559336.448 max 4162972160.000  clipped 100%
+[resume] saved at step 9500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 10000  ppl    2503.0  ood    4768.3  (106m train)
+[health] steps 9501-10000: loss 7.7827  grad-norm mean 41094246.400 max 4887258624.000  clipped 100%
+[resume] saved at step 10000
+[health] steps 10001-10500: loss 7.8093  grad-norm mean 53485305.856 max 23333797888.000  clipped 100%
+[resume] saved at step 10500
+[health] steps 10501-11000: loss 7.8024  grad-norm mean 29450575.872 max 6264079360.000  clipped 100%
+[resume] saved at step 11000
+[health] steps 11001-11500: loss 7.7992  grad-norm mean 12417133.568 max 1991587200.000  clipped 100%
+[resume] saved at step 11500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 12000  ppl    2555.6  ood    4784.5  (129m train)
+[health] steps 11501-12000: loss 7.8328  grad-norm mean 19736858.624 max 8977612800.000  clipped 100%
+[resume] saved at step 12000
+[health] steps 12001-12500: loss 7.8300  grad-norm mean 305598.400 max 23749328.000  clipped 100%
+[resume] saved at step 12500
+[health] steps 12501-13000: loss 7.8058  grad-norm mean 68349.136 max 15220312.000  clipped 100%
+[resume] saved at step 13000
+[health] steps 13001-13500: loss 7.7986  grad-norm mean 243661.376 max 89982584.000  clipped 100%
+[resume] saved at step 13500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 14000  ppl    2508.3  ood    4704.5  (151m train)
+[health] steps 13501-14000: loss 7.8029  grad-norm mean 353022.144 max 33967996.000  clipped 100%
+[resume] saved at step 14000
+[health] steps 14001-14500: loss 7.7932  grad-norm mean 5561296.896 max 703914240.000  clipped 100%
+[resume] saved at step 14500
+[health] steps 14501-15000: loss 7.8114  grad-norm mean 1621969.024 max 163940480.000  clipped 100%
+[resume] saved at step 15000
+[health] steps 15001-15500: loss 7.8004  grad-norm mean 316363.776 max 105838704.000  clipped 100%
+[resume] saved at step 15500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 16000  ppl    2484.9  ood    4599.5  (174m train)
+[health] steps 15501-16000: loss 7.8113  grad-norm mean 39209.184 max 7385705.500  clipped 100%
+[resume] saved at step 16000
+[health] steps 16001-16500: loss 7.7727  grad-norm mean 74813.624 max 15877264.000  clipped 100%
+[resume] saved at step 16500
+[health] steps 16501-17000: loss 7.7546  grad-norm mean 151585.264 max 35481012.000  clipped 100%
+[resume] saved at step 17000
+[health] steps 17001-17500: loss 7.7075  grad-norm mean 10426.528 max 324472.812  clipped 100%
+[resume] saved at step 17500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 18000  ppl    2008.4  ood    4011.7  (201m train)
+[health] steps 17501-18000: loss 7.6165  grad-norm mean 5841.039 max 308463.125  clipped 100%
+[resume] saved at step 18000
+[health] steps 18001-18500: loss 7.5490  grad-norm mean 17413.684 max 3653565.500  clipped 100%
+[resume] saved at step 18500
+[health] steps 18501-19000: loss 7.4955  grad-norm mean 7608.817 max 988159.562  clipped 100%
+[resume] saved at step 19000
+[health] steps 19001-19500: loss 7.4081  grad-norm mean 6861.078 max 1417659.500  clipped 100%
+[resume] saved at step 19500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 20000  ppl    1696.9  ood    3996.4  (229m train)
+[health] steps 19501-20000: loss 7.3676  grad-norm mean 5175.809 max 933903.625  clipped 100%
+[resume] saved at step 20000
+[health] steps 20001-20500: loss 7.4260  grad-norm mean 1747.849 max 125891.266  clipped 100%
+[resume] saved at step 20500
+[health] steps 20501-21000: loss 7.2680  grad-norm mean 7808.993 max 2127733.000  clipped 100%
+[resume] saved at step 21000
+[health] steps 21001-21500: loss 7.1010  grad-norm mean 1423.578 max 210228.375  clipped 100%
+[resume] saved at step 21500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 22000  ppl     747.9  ood    1893.0  (258m train)
+[health] steps 21501-22000: loss 6.8099  grad-norm mean 211.143 max 11816.293  clipped 100%
+[resume] saved at step 22000
+[health] steps 22001-22500: loss 6.4401  grad-norm mean 27.320 max 920.856  clipped 99%
+[resume] saved at step 22500
+[health] steps 22501-23000: loss 6.0178  grad-norm mean 6.441 max 289.114  clipped 97%
+[resume] saved at step 23000
+[health] steps 23001-23500: loss 5.6746  grad-norm mean 4.799 max 30.399  clipped 96%
+[resume] saved at step 23500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 24000  ppl     225.3  ood     811.6  (287m train)
+[health] steps 23501-24000: loss 5.4995  grad-norm mean 3.665 max 13.033  clipped 93%
+[resume] saved at step 24000
+[health] steps 24001-24500: loss 5.3260  grad-norm mean 3.374 max 14.534  clipped 88%
+[resume] saved at step 24500
+[health] steps 24501-25000: loss 5.1846  grad-norm mean 3.123 max 79.611  clipped 86%
+[resume] saved at step 25000
+[health] steps 25001-25500: loss 5.0677  grad-norm mean 2.523 max 11.034  clipped 83%
+[resume] saved at step 25500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 26000  ppl     130.1  ood     421.0  (317m train)
+[health] steps 25501-26000: loss 4.9072  grad-norm mean 2.198 max 8.075  clipped 84%
+[resume] saved at step 26000
+[health] steps 26001-26500: loss 4.8066  grad-norm mean 2.138 max 10.001  clipped 82%
+[resume] saved at step 26500
+[health] steps 26501-27000: loss 4.6753  grad-norm mean 2.122 max 21.268  clipped 86%
+[resume] saved at step 27000
+[health] steps 27001-27500: loss 4.6294  grad-norm mean 2.837 max 41.330  clipped 86%
+[resume] saved at step 27500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 28000  ppl      95.7  ood     273.5  (347m train)
+[health] steps 27501-28000: loss 4.5755  grad-norm mean 2.124 max 22.417  clipped 80%
+[resume] saved at step 28000
+[health] steps 28001-28500: loss 4.5268  grad-norm mean 1.833 max 11.551  clipped 80%
+[resume] saved at step 28500
+[health] steps 28501-29000: loss 4.4620  grad-norm mean 1.791 max 27.395  clipped 76%
+[resume] saved at step 29000
+[health] steps 29001-29500: loss 4.4037  grad-norm mean 1.656 max 13.448  clipped 76%
+[resume] saved at step 29500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 30000  ppl      79.4  ood     226.9  (377m train)
+[health] steps 29501-30000: loss 4.4009  grad-norm mean 1.581 max 9.091  clipped 72%
+[resume] saved at step 30000
+[health] steps 30001-30500: loss 4.3442  grad-norm mean 1.477 max 6.930  clipped 70%
+[resume] saved at step 30500
+[health] steps 30501-31000: loss 4.3286  grad-norm mean 1.858 max 48.474  clipped 76%
+[resume] saved at step 31000
+[health] steps 31001-31500: loss 4.2859  grad-norm mean 1.479 max 19.652  clipped 72%
+[resume] saved at step 31500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 32000  ppl      72.3  ood     208.1  (407m train)
+[health] steps 31501-32000: loss 4.2687  grad-norm mean 3.638 max 521.294  clipped 75%
+[resume] saved at step 32000
+[health] steps 32001-32500: loss 4.2599  grad-norm mean 1.912 max 63.335  clipped 66%
+[resume] saved at step 32500
+[health] steps 32501-33000: loss 4.2270  grad-norm mean 1.746 max 116.627  clipped 65%
+[resume] saved at step 33000
+[health] steps 33001-33500: loss 4.2043  grad-norm mean 1.349 max 3.969  clipped 69%
+[resume] saved at step 33500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 34000  ppl      65.5  ood     184.7  (437m train)
+[health] steps 33501-34000: loss 4.1551  grad-norm mean 1.401 max 21.578  clipped 62%
+[resume] saved at step 34000
+[health] steps 34001-34500: loss 4.1675  grad-norm mean 1.255 max 7.341  clipped 60%
+[resume] saved at step 34500
+[health] steps 34501-35000: loss 4.1445  grad-norm mean 1.269 max 3.542  clipped 65%
+[resume] saved at step 35000
+[health] steps 35001-35500: loss 4.1371  grad-norm mean 1.292 max 5.103  clipped 63%
+[resume] saved at step 35500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 36000  ppl      61.0  ood     168.0  (467m train)
+[health] steps 35501-36000: loss 4.0983  grad-norm mean 1.428 max 9.739  clipped 66%
+[resume] saved at step 36000
+[health] steps 36001-36500: loss 4.1223  grad-norm mean 3.757 max 117.701  clipped 88%
+[resume] saved at step 36500
+[health] steps 36501-37000: loss 4.0695  grad-norm mean 4.904 max 348.959  clipped 93%
+[resume] saved at step 37000
+[health] steps 37001-37500: loss 4.0842  grad-norm mean 2.905 max 229.808  clipped 73%
+[resume] saved at step 37500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 38000  ppl      57.8  ood     159.0  (496m train)
+[health] steps 37501-38000: loss 4.0639  grad-norm mean 1.938 max 152.630  clipped 69%
+[resume] saved at step 38000
+[health] steps 38001-38500: loss 4.0525  grad-norm mean 1.378 max 18.788  clipped 61%
+[resume] saved at step 38500
+[health] steps 38501-39000: loss 4.0368  grad-norm mean 1.491 max 51.621  clipped 62%
+[resume] saved at step 39000
+[health] steps 39001-39500: loss 4.0236  grad-norm mean 1.588 max 57.291  clipped 67%
+[resume] saved at step 39500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 40000  ppl      54.9  ood     164.4  (526m train)
+[health] steps 39501-40000: loss 3.9865  grad-norm mean 1.169 max 6.659  clipped 59%
+[resume] saved at step 40000
+[health] steps 40001-40500: loss 3.9875  grad-norm mean 1.242 max 6.176  clipped 64%
+[resume] saved at step 40500
+[health] steps 40501-41000: loss 3.9942  grad-norm mean 1.246 max 14.735  clipped 59%
+[resume] saved at step 41000
+[health] steps 41001-41500: loss 3.9839  grad-norm mean 1.226 max 7.969  clipped 65%
+[resume] saved at step 41500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 42000  ppl      52.4  ood     154.6  (555m train)
+[health] steps 41501-42000: loss 3.9453  grad-norm mean 1.215 max 8.933  clipped 61%
+[resume] saved at step 42000
+[health] steps 42001-42500: loss 3.9462  grad-norm mean 1.161 max 12.808  clipped 58%
+[resume] saved at step 42500
+[health] steps 42501-43000: loss 3.9449  grad-norm mean 1.178 max 3.593  clipped 64%
+[resume] saved at step 43000
+[health] steps 43001-43500: loss 3.9106  grad-norm mean 1.127 max 2.976  clipped 57%
+[resume] saved at step 43500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 44000  ppl      50.4  ood     153.8  (584m train)
+[health] steps 43501-44000: loss 3.9122  grad-norm mean 1.163 max 6.650  clipped 58%
+[resume] saved at step 44000
+[health] steps 44001-44500: loss 3.9059  grad-norm mean 1.101 max 4.883  clipped 52%
+[resume] saved at step 44500
+[health] steps 44501-45000: loss 3.9081  grad-norm mean 1.186 max 9.662  clipped 63%
+[resume] saved at step 45000
+[health] steps 45001-45500: loss 3.8967  grad-norm mean 1.095 max 2.845  clipped 56%
+[resume] saved at step 45500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 46000  ppl      48.9  ood     150.0  (613m train)
+[health] steps 45501-46000: loss 3.8778  grad-norm mean 1.134 max 10.125  clipped 57%
+[resume] saved at step 46000
+[health] steps 46001-46500: loss 3.8677  grad-norm mean 1.148 max 3.091  clipped 59%
+[resume] saved at step 46500
+[health] steps 46501-47000: loss 3.8585  grad-norm mean 1.141 max 8.096  clipped 58%
+[resume] saved at step 47000
+[health] steps 47001-47500: loss 3.8560  grad-norm mean 1.127 max 2.716  clipped 58%
+[resume] saved at step 47500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 48000  ppl      47.3  ood     154.9  (642m train)
+[health] steps 47501-48000: loss 3.8641  grad-norm mean 1.115 max 3.828  clipped 60%
+[resume] saved at step 48000
+[health] steps 48001-48500: loss 3.8704  grad-norm mean 1.197 max 24.853  clipped 58%
+[resume] saved at step 48500
+[health] steps 48501-49000: loss 3.8467  grad-norm mean 1.150 max 6.622  clipped 64%
+[resume] saved at step 49000
+[health] steps 49001-49500: loss 3.8208  grad-norm mean 1.135 max 10.521  clipped 61%
+[resume] saved at step 49500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 50000  ppl      46.0  ood     159.2  (671m train)
+[health] steps 49501-50000: loss 3.8136  grad-norm mean 1.137 max 3.239  clipped 67%
+[resume] saved at step 50000
+[health] steps 50001-50500: loss 3.8025  grad-norm mean 1.107 max 6.094  clipped 57%
+[resume] saved at step 50500
+[health] steps 50501-51000: loss 3.7829  grad-norm mean 1.158 max 2.793  clipped 67%
+[resume] saved at step 51000
+[health] steps 51001-51500: loss 3.7904  grad-norm mean 1.165 max 5.583  clipped 66%
+[resume] saved at step 51500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 52000  ppl      44.9  ood     155.2  (700m train)
+[health] steps 51501-52000: loss 3.7827  grad-norm mean 1.135 max 2.748  clipped 66%
+[resume] saved at step 52000
+[health] steps 52001-52500: loss 3.7755  grad-norm mean 1.133 max 5.567  clipped 66%
+[resume] saved at step 52500
+[health] steps 52501-53000: loss 3.8042  grad-norm mean 1.142 max 5.231  clipped 69%
+[resume] saved at step 53000
+[health] steps 53001-53500: loss 3.7811  grad-norm mean 1.161 max 5.057  clipped 70%
+[resume] saved at step 53500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 54000  ppl      44.2  ood     153.8  (729m train)
+[health] steps 53501-54000: loss 3.7990  grad-norm mean 1.145 max 2.926  clipped 67%
+[resume] saved at step 54000
+[health] steps 54001-54500: loss 3.7553  grad-norm mean 1.121 max 2.274  clipped 64%
+[resume] saved at step 54500
+[health] steps 54501-55000: loss 3.7891  grad-norm mean 1.133 max 4.430  clipped 68%
+[resume] saved at step 55000
+[health] steps 55001-55500: loss 3.7658  grad-norm mean 1.128 max 3.058  clipped 69%
+[resume] saved at step 55500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 56000  ppl      43.5  ood     156.6  (757m train)
+[health] steps 55501-56000: loss 3.7724  grad-norm mean 1.189 max 9.216  clipped 72%
+[resume] saved at step 56000
+[health] steps 56001-56500: loss 3.7683  grad-norm mean 1.144 max 2.552  clipped 73%
+[resume] saved at step 56500
+[health] steps 56501-57000: loss 3.7704  grad-norm mean 1.179 max 3.891  clipped 77%
+[resume] saved at step 57000
+[health] steps 57001-57500: loss 3.7553  grad-norm mean 1.149 max 2.344  clipped 70%
+[resume] saved at step 57500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 58000  ppl      42.9  ood     158.2  (786m train)
+[health] steps 57501-58000: loss 3.7509  grad-norm mean 1.187 max 7.416  clipped 75%
+[resume] saved at step 58000
+[health] steps 58001-58500: loss 3.7416  grad-norm mean 1.231 max 15.319  clipped 79%
+[resume] saved at step 58500
+[health] steps 58501-59000: loss 3.7374  grad-norm mean 1.147 max 2.430  clipped 77%
+[resume] saved at step 59000
+[health] steps 59001-59500: loss 3.7469  grad-norm mean 1.164 max 4.054  clipped 71%
+[resume] saved at step 59500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 60000  ppl      42.3  ood     141.4  (814m train)
+[health] steps 59501-60000: loss 3.7033  grad-norm mean 1.180 max 9.303  clipped 78%
+[resume] saved at step 60000
+[health] steps 60001-60500: loss 3.7548  grad-norm mean 1.171 max 3.158  clipped 78%
+[resume] saved at step 60500
+[unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8] step 61000  ppl      42.1  ood     150.2  (828m train)
+[health] steps 60501-61000: loss 3.7601  grad-norm mean 1.208 max 10.462  clipped 81%
+[done] unit_bf16_d910_c1_lr0.001_acc2_s99_st61000_ev2000_corpc42b_perm_cos_w8: ppl 42.1, ood 150.2, 828 min, trained