main <- step 6000

README.md

@@ -0,0 +1,31 @@
+---
+license: apache-2.0
+library_name: pytorch
+tags: [dynamic-chunking, tokenization, hnet, byte-level, beetle-hnet]
+---
+
+# ali_ali_s0
+
+A byte-level H-Net run from [beetle-hnet](https://github.com/suchirsalhan/beetle-hnet)
+(`architecture=ali`, `objective=ratio`
+`schedule=None`, `dechunk=ema`, seed 0,
+6000 steps x 32 x 1024 bytes, langs ['en', 'tr', 'fi']).
+
+## Layout (Beetle-style)
+
+- **`step-N` branches**, one per kept checkpoint on a phase-aware log2 schedule (0, 1, 2, 4, 6,
+  8, 16, ..., 2^k from every objective-phase start, plus the final step). Each holds
+  `model.safetensors`, `config.json`, `training_config.yaml` and the full resumable trainer
+  state `training_state/checkpoint.pt` (for inventory arms this includes the inventory).
+- **`main`**: the latest step's weights and config, plus `log.jsonl`, `final.json` and the
+  FLORES+ boundary probes `probe/` once the run has finished.
+
+```python
+from huggingface_hub import hf_hub_download
+import torch
+p = hf_hub_download("AdaptiveChunking/ali_ali_s0", "training_state/checkpoint.pt", revision="step-1024")
+state = torch.load(p, map_location="cpu", weights_only=False)
+```
+
+`model.safetensors` stores each tied tensor once; `config.json["tied_weights"]` maps the
+dropped names to the stored ones.

config.json

@@ -0,0 +1,187 @@
+{
+  "arch": "ali",
+  "step": 6000,
+  "model_config": {
+    "vocab_size": 256,
+    "d_enc": 256,
+    "d_main": 512,
+    "n_enc": 2,
+    "n_dec": 2,
+    "n_main": 6,
+    "n_heads_enc": 4,
+    "n_heads_main": 8,
+    "seq_len": 1024,
+    "dropout": 0.0,
+    "target_ratio": 6.0,
+    "max_chunk_mult": 3.0,
+    "dechunk": "ema",
+    "router_noise": 0.0,
+    "rope_base": 10000.0
+  },
+  "ali_config": {
+    "enabled": true,
+    "budget_B": 65536,
+    "staging_capacity": 262144,
+    "policy": "utility",
+    "candidate_types": [
+      "surface",
+      "split",
+      "merge",
+      "skel"
+    ],
+    "gate": "vector",
+    "retrieval": true,
+    "memory_scope": "persistent",
+    "read_dropout": 0.1,
+    "write_rule": "mean",
+    "eta_min": 0.0,
+    "consolidate_every": 50,
+    "probe_seqs": 2,
+    "probe_reps": 8,
+    "horizon": 32,
+    "predictor": "mlp",
+    "lambda_util": 0.1,
+    "max_span": 64,
+    "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json",
+    "gate_hidden": 0,
+    "type_dim": 16,
+    "gate_bias_init": -2.0
+  },
+  "tied_weights": {
+    "head.weight": "emb.weight"
+  },
+  "config": {
+    "model": {
+      "architecture": "ali",
+      "d_enc": 256,
+      "d_main": 512,
+      "n_enc": 2,
+      "n_dec": 2,
+      "n_main": 6,
+      "n_heads_enc": 4,
+      "n_heads_main": 8,
+      "seq_len": 1024,
+      "dropout": 0.0,
+      "target_ratio": 6.0,
+      "max_chunk_mult": 3.0,
+      "dechunk": "ema",
+      "router_noise": 0.0,
+      "rope_base": 10000.0
+    },
+    "memory": {
+      "ali": {
+        "budget_B": 65536,
+        "staging_capacity": 262144,
+        "policy": "utility",
+        "candidate_types": [
+          "surface",
+          "split",
+          "merge",
+          "skel"
+        ],
+        "write_rule": "mean",
+        "eta_min": 0.0,
+        "memory_scope": "persistent",
+        "consolidate_every": 50,
+        "max_span": 64,
+        "gate": "vector",
+        "retrieval": true,
+        "read_dropout": 0.1,
+        "gate_hidden": 0,
+        "type_dim": 16,
+        "gate_bias_init": -2.0,
+        "probe_seqs": 2,
+        "probe_reps": 8,
+        "horizon": 32,
+        "predictor": "mlp",
+        "lambda_util": 0.1,
+        "enabled": true,
+        "inventory_kwargs": {},
+        "probe_mix": null,
+        "util_batch_slots": 1024
+      },
+      "pkm": {
+        "n_keys": 256,
+        "topk": 8,
+        "heads": 1,
+        "d_key": null,
+        "value_lr_mult": 1.0
+      }
+    },
+    "objective": {
+      "name": "ratio",
+      "schedule": null,
+      "alpha": 0.03,
+      "alpha_parity_cap": 0.06,
+      "global_anchor": 0.0,
+      "premium_clip": 4.0,
+      "premium_kind": "flores_per_sent",
+      "tau1": 0.4,
+      "tau2": 0.7,
+      "freeze_scope": "router",
+      "cpc_decay": 0.9,
+      "genreg_target_entropy": null,
+      "genreg_eta": 0.05
+    },
+    "data": {
+      "corpus": "corpus",
+      "langs": [
+        "en",
+        "tr",
+        "fi"
+      ],
+      "mixture": null,
+      "doc_filters": null,
+      "byte_classes": "scratch/data/byte_classes.json"
+    },
+    "training": {
+      "seed": 0,
+      "max_steps": 6000,
+      "batch_size": 32,
+      "grad_accum": 1,
+      "lr": 0.0003,
+      "min_lr_frac": 0.1,
+      "warmup": 300,
+      "weight_decay": 0.1,
+      "grad_clip": 1.0,
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "router_lr_mult": 1.0,
+      "log_every": 50,
+      "amp": "bf16",
+      "compile": false,
+      "device": "cuda"
+    },
+    "checkpointing": {
+      "run_name": "ali_ali_s0",
+      "runs_dir": "runs",
+      "ckpt_every": 200,
+      "keep": false,
+      "milestones": [
+        1000,
+        3000
+      ],
+      "resume": "auto"
+    },
+    "evaluation": {
+      "eval_every": 200,
+      "eval_batches": 8,
+      "probe_sents": 128,
+      "probe_bpb_batches": 4,
+      "log_spaced_probe": true,
+      "dense_early_every": 0,
+      "dense_early_until": 0,
+      "dump_reps": true,
+      "dump_reps_thresholds": [
+        0.3,
+        0.4,
+        0.5,
+        0.6,
+        0.7
+      ]
+    }
+  },
+  "format": "beetle-hnet step branch v1 (src/checkpointing/hub.py)"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e1b5c038341e12cd92798fb4f9efcc3f6c3e2eddeba1f739af0110c9f1b2d0
+size 91151512

training_config.yaml

@@ -0,0 +1,117 @@
+model:
+  architecture: ali
+  d_enc: 256
+  d_main: 512
+  n_enc: 2
+  n_dec: 2
+  n_main: 6
+  n_heads_enc: 4
+  n_heads_main: 8
+  seq_len: 1024
+  dropout: 0.0
+  target_ratio: 6.0
+  max_chunk_mult: 3.0
+  dechunk: ema
+  router_noise: 0.0
+  rope_base: 10000.0
+memory:
+  ali:
+    budget_B: 65536
+    staging_capacity: 262144
+    policy: utility
+    candidate_types:
+    - surface
+    - split
+    - merge
+    - skel
+    write_rule: mean
+    eta_min: 0.0
+    memory_scope: persistent
+    consolidate_every: 50
+    max_span: 64
+    gate: vector
+    retrieval: true
+    read_dropout: 0.1
+    gate_hidden: 0
+    type_dim: 16
+    gate_bias_init: -2.0
+    probe_seqs: 2
+    probe_reps: 8
+    horizon: 32
+    predictor: mlp
+    lambda_util: 0.1
+    enabled: true
+    inventory_kwargs: {}
+    probe_mix: null
+    util_batch_slots: 1024
+  pkm:
+    n_keys: 256
+    topk: 8
+    heads: 1
+    d_key: null
+    value_lr_mult: 1.0
+objective:
+  name: ratio
+  schedule: null
+  alpha: 0.03
+  alpha_parity_cap: 0.06
+  global_anchor: 0.0
+  premium_clip: 4.0
+  premium_kind: flores_per_sent
+  tau1: 0.4
+  tau2: 0.7
+  freeze_scope: router
+  cpc_decay: 0.9
+  genreg_target_entropy: null
+  genreg_eta: 0.05
+data:
+  corpus: corpus
+  langs:
+  - en
+  - tr
+  - fi
+  mixture: null
+  doc_filters: null
+  byte_classes: scratch/data/byte_classes.json
+training:
+  seed: 0
+  max_steps: 6000
+  batch_size: 32
+  grad_accum: 1
+  lr: 0.0003
+  min_lr_frac: 0.1
+  warmup: 300
+  weight_decay: 0.1
+  grad_clip: 1.0
+  betas:
+  - 0.9
+  - 0.95
+  router_lr_mult: 1.0
+  log_every: 50
+  amp: bf16
+  compile: false
+  device: cuda
+checkpointing:
+  run_name: ali_ali_s0
+  runs_dir: runs
+  ckpt_every: 200
+  keep: false
+  milestones:
+  - 1000
+  - 3000
+  resume: auto
+evaluation:
+  eval_every: 200
+  eval_batches: 8
+  probe_sents: 128
+  probe_bpb_batches: 4
+  log_spaced_probe: true
+  dense_early_every: 0
+  dense_early_until: 0
+  dump_reps: true
+  dump_reps_thresholds:
+  - 0.3
+  - 0.4
+  - 0.5
+  - 0.6
+  - 0.7