main <- step 76300

README.md

@@ -0,0 +1,19 @@
+---
+license: apache-2.0
+library_name: pytorch
+tags: [dynamic-chunking, tokenization, hnet, byte-level, beetle-hnet, legacy-trainer]
+---
+
+# specfull_B_s0
+
+Byte-level H-Net run from [beetle-hnet](https://github.com/suchirsalhan/beetle-hnet), trained
+with the legacy trainer (`legacy/tier1/train.py`). Split out of the bundled repo
+[`AdaptiveChunking/hnet-chunking-spec-82m`](https://huggingface.co/AdaptiveChunking/hnet-chunking-spec-82m) into the Beetle per-run layout.
+
+- **`step-N` branches**: 30 retained checkpoints (0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 4000, 8000 ...),
+  each with `model.safetensors` and `config.json` (`model_config`, training config and the
+  `tied_weights` map: restore with `sd[dst] = sd[src]` for every entry).
+- **`main`**: the latest weights plus `final.json` and `log.jsonl`.
+- The legacy trainer kept no optimiser state, so there is no `training_state/` here. FLORES+
+  boundary probes for this run are in the dataset
+  [`AdaptiveChunking/hnet-chunking-probes`](https://huggingface.co/datasets/AdaptiveChunking/hnet-chunking-probes) under `specfull_B_s0/`.

config.json

@@ -0,0 +1,211 @@
+{
+  "run": "specfull_B_s0",
+  "model_config": {
+    "vocab_size": 256,
+    "d_enc": 384,
+    "d_main": 768,
+    "n_enc": 3,
+    "n_dec": 3,
+    "n_main": 10,
+    "n_heads_enc": 6,
+    "n_heads_main": 12,
+    "seq_len": 1024,
+    "dropout": 0.0,
+    "target_ratio": 6.0,
+    "max_chunk_mult": 3.0,
+    "dechunk": "ema",
+    "router_noise": 0.0,
+    "rope_base": 10000.0
+  },
+  "tied_weights": {
+    "head.weight": "emb.weight"
+  },
+  "train_config": {
+    "condition": "B",
+    "seed": 0,
+    "out_dir": null,
+    "run_name": "specfull_B_s0",
+    "corpus": "corpus",
+    "langs": null,
+    "seq_len": 1024,
+    "batch_size": 64,
+    "grad_accum": 1,
+    "steps": 76300,
+    "lr": 0.0003,
+    "min_lr_frac": 0.1,
+    "warmup": 2000,
+    "weight_decay": 0.1,
+    "grad_clip": 1.0,
+    "betas": [
+      0.9,
+      0.95
+    ],
+    "router_lr_mult": 1.0,
+    "d_enc": 384,
+    "d_main": 768,
+    "n_enc": 3,
+    "n_dec": 3,
+    "n_main": 10,
+    "n_heads_enc": 6,
+    "n_heads_main": 12,
+    "dechunk": "ema",
+    "target_ratio": 6.0,
+    "alpha": 0.03,
+    "alpha_parity_cap": 0.06,
+    "global_anchor": 0.0,
+    "premium_clip": 4.0,
+    "premium_kind": "flores_per_sent",
+    "mixture": null,
+    "doc_filters": null,
+    "tau1": 0.4,
+    "tau2": 0.7,
+    "freeze_scope": "router",
+    "ckpt_every": 4000,
+    "eval_every": 4000,
+    "log_every": 500,
+    "probe_sents": 128,
+    "eval_batches": 8,
+    "keep_ckpt": true,
+    "amp": "bf16",
+    "compile": false,
+    "cpc_decay": 0.9,
+    "device": "cuda",
+    "log_spaced_probe": true,
+    "probe_bpb_batches": 4,
+    "dump_reps": true,
+    "dump_reps_thresholds": [
+      0.3,
+      0.4,
+      0.5,
+      0.6,
+      0.7
+    ],
+    "router_noise": 0.0,
+    "genreg_target_entropy": null,
+    "genreg_eta": 0.05
+  },
+  "checkpoints": [
+    {
+      "tag": "000000",
+      "step": 0
+    },
+    {
+      "tag": "000001",
+      "step": 1
+    },
+    {
+      "tag": "000002",
+      "step": 2
+    },
+    {
+      "tag": "000004",
+      "step": 4
+    },
+    {
+      "tag": "000008",
+      "step": 8
+    },
+    {
+      "tag": "000016",
+      "step": 16
+    },
+    {
+      "tag": "000032",
+      "step": 32
+    },
+    {
+      "tag": "000064",
+      "step": 64
+    },
+    {
+      "tag": "000128",
+      "step": 128
+    },
+    {
+      "tag": "000256",
+      "step": 256
+    },
+    {
+      "tag": "004000",
+      "step": 4000
+    },
+    {
+      "tag": "008000",
+      "step": 8000
+    },
+    {
+      "tag": "012000",
+      "step": 12000
+    },
+    {
+      "tag": "016000",
+      "step": 16000
+    },
+    {
+      "tag": "020000",
+      "step": 20000
+    },
+    {
+      "tag": "024000",
+      "step": 24000
+    },
+    {
+      "tag": "028000",
+      "step": 28000
+    },
+    {
+      "tag": "032000",
+      "step": 32000
+    },
+    {
+      "tag": "036000",
+      "step": 36000
+    },
+    {
+      "tag": "040000",
+      "step": 40000
+    },
+    {
+      "tag": "044000",
+      "step": 44000
+    },
+    {
+      "tag": "048000",
+      "step": 48000
+    },
+    {
+      "tag": "052000",
+      "step": 52000
+    },
+    {
+      "tag": "056000",
+      "step": 56000
+    },
+    {
+      "tag": "060000",
+      "step": 60000
+    },
+    {
+      "tag": "064000",
+      "step": 64000
+    },
+    {
+      "tag": "068000",
+      "step": 68000
+    },
+    {
+      "tag": "072000",
+      "step": 72000
+    },
+    {
+      "tag": "076000",
+      "step": 76000
+    },
+    {
+      "tag": "076300",
+      "step": 76300
+    }
+  ],
+  "step": 76300,
+  "source": "AdaptiveChunking/hnet-chunking-spec-82m/specfull_B_s0/076300.safetensors"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d38848ef0b8e3ad84b3b5e78b7538d9633bcb10534055c2e5a9551233160bd80
+size 329898344