run records: final.json, log, probes

config.yaml

@@ -0,0 +1,72 @@
+model: {architecture: ali, d_enc: 256, d_main: 512, n_enc: 2, n_dec: 2, n_main: 6,
+  n_heads_enc: 4, n_heads_main: 8, seq_len: 1024, dropout: 0.0, target_ratio: 6.0,
+  max_chunk_mult: 3.0, dechunk: ema, router_noise: 0.0, rope_base: 10000.0}
+memory:
+  ali:
+    budget_B: 65536
+    staging_capacity: 262144
+    policy: utility
+    candidate_types: [surface, split, merge, skel]
+    write_rule: mean
+    eta_min: 0.0
+    memory_scope: persistent
+    consolidate_every: 50
+    max_span: 64
+    gate: vector
+    retrieval: true
+    read_dropout: 0.1
+    gate_hidden: 0
+    type_dim: 16
+    gate_bias_init: -2.0
+    probe_seqs: 2
+    probe_reps: 8
+    horizon: 32
+    predictor: mlp
+    lambda_util: 0.1
+    enabled: true
+    inventory_kwargs: {}
+    probe_mix: null
+    util_batch_slots: 1024
+  pkm: {n_keys: 256, topk: 8, heads: 1, d_key: null, value_lr_mult: 1.0}
+objective: {name: ratio, schedule: null, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
+  premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
+  cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05}
+data:
+  corpus: corpus
+  langs: [en, tr, fi]
+  mixture: null
+  doc_filters: null
+  byte_classes: scratch/data/byte_classes.json
+training:
+  seed: 2
+  max_steps: 6000
+  batch_size: 32
+  grad_accum: 1
+  lr: 0.0003
+  min_lr_frac: 0.1
+  warmup: 300
+  weight_decay: 0.1
+  grad_clip: 1.0
+  betas: [0.9, 0.95]
+  router_lr_mult: 1.0
+  log_every: 50
+  amp: bf16
+  compile: false
+  device: cuda
+checkpointing:
+  run_name: ali_ali_s2
+  runs_dir: runs
+  ckpt_every: 200
+  keep: false
+  milestones: [1000, 3000]
+  resume: auto
+evaluation:
+  eval_every: 200
+  eval_batches: 8
+  probe_sents: 128
+  probe_bpb_batches: 4
+  log_spaced_probe: true
+  dense_early_every: 0
+  dense_early_until: 0
+  dump_reps: true
+  dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]

final.json

@@ -0,0 +1,307 @@
+{
+  "event": "final",
+  "run": "ali_ali_s2",
+  "arch": "ali",
+  "objective": "ratio",
+  "condition": "A",
+  "step": 6000,
+  "params": 22785088,
+  "seed": 2,
+  "per_lang": {
+    "en": {
+      "cpc": 0.198931218755387,
+      "cps": 27.046875,
+      "bpb": 1.8996927689640364,
+      "bytes_per_chunk": 5.033825237871208,
+      "reads_per_chunk": 1.2592244102426238
+    },
+    "tr": {
+      "cpc": 0.21278241175495774,
+      "cps": 30.09375,
+      "bpb": 2.000258989870039,
+      "bytes_per_chunk": 5.287986525058751,
+      "reads_per_chunk": 1.1604385407526199
+    },
+    "fi": {
+      "cpc": 0.2420923927489235,
+      "cps": 35.578125,
+      "bpb": 2.1298347928691213,
+      "bytes_per_chunk": 4.516648144797939,
+      "reads_per_chunk": 1.0113801807389795
+    }
+  },
+  "bpb_mean": 2.009928850567732,
+  "gini_cpc": 0.04401017269961849,
+  "gini_cps": 0.061341422312099825,
+  "bpic": 4.780249410178632,
+  "mean_boundary_entropy": 0.1135382205247879,
+  "dechunk": "ema",
+  "router_noise": 0.0,
+  "transitions": [
+    {
+      "step": 0,
+      "phase": 1,
+      "objective": "global",
+      "router_trainable": true,
+      "encoder_trainable": true,
+      "frozen_router_params": 0
+    },
+    {
+      "step": 2400,
+      "phase": 2,
+      "objective": "global",
+      "router_trainable": true,
+      "encoder_trainable": true,
+      "frozen_router_params": 0
+    },
+    {
+      "step": 4200,
+      "phase": 3,
+      "objective": "global",
+      "router_trainable": true,
+      "encoder_trainable": true,
+      "frozen_router_params": 0
+    }
+  ],
+  "probe_overhead_sec_total": 472.7523412704468,
+  "probe_overhead_seq_frac": 0.49990625,
+  "wall_sec": 4825.126570940018,
+  "inventory": {
+    "size": 327666,
+    "admitted": 65536,
+    "staged": 262130,
+    "budget_B": 65536,
+    "staging_capacity": 262144,
+    "collisions": 0,
+    "dropped": 0,
+    "type_rejected": 0,
+    "config_keys": 1115,
+    "admitted_surface": 25955,
+    "staged_surface": 58566,
+    "admitted_split": 5226,
+    "staged_split": 43400,
+    "admitted_merge": 1953,
+    "staged_merge": 113391,
+    "admitted_skel": 32402,
+    "staged_skel": 46773,
+    "mean_count": 354.78802490234375,
+    "n_util_obs": 400,
+    "util_count_spearman": -0.09193389570565129
+  },
+  "ali_params": 268096,
+  "ali_config": {
+    "enabled": true,
+    "budget_B": 65536,
+    "staging_capacity": 262144,
+    "policy": "utility",
+    "candidate_types": [
+      "surface",
+      "split",
+      "merge",
+      "skel"
+    ],
+    "gate": "vector",
+    "retrieval": true,
+    "memory_scope": "persistent",
+    "read_dropout": 0.1,
+    "write_rule": "mean",
+    "eta_min": 0.0,
+    "consolidate_every": 50,
+    "probe_seqs": 2,
+    "probe_reps": 8,
+    "horizon": 32,
+    "predictor": "mlp",
+    "lambda_util": 0.1,
+    "max_span": 64,
+    "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json",
+    "gate_hidden": 0,
+    "type_dim": 16,
+    "gate_bias_init": -2.0
+  },
+  "util_vs_frequency_spearman": -0.09193389570565129,
+  "paired_vs_dropout": {
+    "n": 786,
+    "pearson": 0.024077608116929692,
+    "spearman": 0.015192791635274499,
+    "mean_abs_discrepancy": 0.45565709471702576,
+    "mean_paired": 0.008434545248746872,
+    "mean_dropout": 0.015219463035464287
+  },
+  "predictor_calibration": {
+    "n": 1760,
+    "all": {
+      "n": 1760,
+      "pearson": 0.27194086356134745,
+      "spearman": 0.17446890753785668,
+      "mae": 0.035816490650177,
+      "reliability": [
+        {
+          "pred": -0.016677947714924812,
+          "obs": -0.017740517854690552,
+          "n": 176
+        },
+        {
+          "pred": -0.004937878809869289,
+          "obs": -0.0037392256781458855,
+          "n": 176
+        },
+        {
+          "pred": -0.0017805221723392606,
+          "obs": 0.0012831037165597081,
+          "n": 176
+        },
+        {
+          "pred": 0.0007518658530898392,
+          "obs": -0.0013531275326386094,
+          "n": 176
+        },
+        {
+          "pred": 0.002950140042230487,
+          "obs": -0.0027285749092698097,
+          "n": 176
+        },
+        {
+          "pred": 0.004957593511790037,
+          "obs": 0.002060173312202096,
+          "n": 176
+        },
+        {
+          "pred": 0.006592847872525454,
+          "obs": 0.008401456288993359,
+          "n": 176
+        },
+        {
+          "pred": 0.00861427653580904,
+          "obs": 0.009444334544241428,
+          "n": 176
+        },
+        {
+          "pred": 0.012249228544533253,
+          "obs": 0.004808991681784391,
+          "n": 176
+        },
+        {
+          "pred": 0.030282508581876755,
+          "obs": 0.032979901880025864,
+          "n": 176
+        }
+      ]
+    },
+    "admitted": {
+      "n": 1664,
+      "pearson": 0.27126384192197844,
+      "spearman": 0.14323910119391775,
+      "mae": 0.03694799169898033,
+      "reliability": [
+        {
+          "pred": -0.014275510795414448,
+          "obs": -0.01375713013112545,
+          "n": 167
+        },
+        {
+          "pred": -0.004109010100364685,
+          "obs": 0.002235351363196969,
+          "n": 166
+        },
+        {
+          "pred": -0.0010778754949569702,
+          "obs": -0.0022600579541176558,
+          "n": 166
+        },
+        {
+          "pred": 0.0013819066807627678,
+          "obs": 0.0005889079766348004,
+          "n": 167
+        },
+        {
+          "pred": 0.003456159494817257,
+          "obs": -0.004050337243825197,
+          "n": 166
+        },
+        {
+          "pred": 0.005341973155736923,
+          "obs": 0.0051288180984556675,
+          "n": 166
+        },
+        {
+          "pred": 0.00684454757720232,
+          "obs": 0.010526491329073906,
+          "n": 167
+        },
+        {
+          "pred": 0.008928881026804447,
+          "obs": 0.007826470769941807,
+          "n": 166
+        },
+        {
+          "pred": 0.012642021290957928,
+          "obs": 0.005376995541155338,
+          "n": 166
+        },
+        {
+          "pred": 0.03106619231402874,
+          "obs": 0.03403934836387634,
+          "n": 167
+        }
+      ]
+    },
+    "staged": {
+      "n": 96,
+      "pearson": 0.2526075665089079,
+      "spearman": 0.31759359739555076,
+      "mae": 0.016203748062253,
+      "reliability": [
+        {
+          "pred": -0.04305326193571091,
+          "obs": -0.022864140570163727,
+          "n": 10
+        },
+        {
+          "pred": -0.014916949905455112,
+          "obs": -0.019766394048929214,
+          "n": 9
+        },
+        {
+          "pred": -0.010655229911208153,
+          "obs": -0.035392455756664276,
+          "n": 10
+        },
+        {
+          "pred": -0.0075031304731965065,
+          "obs": -0.01895267516374588,
+          "n": 9
+        },
+        {
+          "pred": -0.006007531192153692,
+          "obs": -0.016960132867097855,
+          "n": 10
+        },
+        {
+          "pred": -0.004678408615291119,
+          "obs": -0.014496909454464912,
+          "n": 10
+        },
+        {
+          "pred": -0.002913302043452859,
+          "obs": -0.013845675624907017,
+          "n": 9
+        },
+        {
+          "pred": -0.0008945651352405548,
+          "obs": -0.018120767548680305,
+          "n": 9
+        },
+        {
+          "pred": 0.001555976690724492,
+          "obs": -0.008646379224956036,
+          "n": 10
+        },
+        {
+          "pred": 0.007470910437405109,
+          "obs": -0.010902724228799343,
+          "n": 10
+        }
+      ]
+    }
+  }
+}

probe/000000__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb0b6634cc4b45f928c76202d90bdabfaab6e9250cfa6e2435c8be6418caf152
+size 258145

probe/000000__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b646c430a3196036ae4e60d0eede797d26eb98a11aa180037f78798fae99653
+size 294036

probe/000000__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fe8326a3316548dc94ef8ec37d5e1d8a69df2d544d6888dc9b5419691d6fbc0
+size 281520

probe/000001__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d406a1fec9b063e1fbf9abb78579d79a1630dafd6b80db9aeb64923a844b34f
+size 258081

probe/000001__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e89b93d2b52609a047b6fd5cc1e7a6ce67670bd9ca56601bb075714a3063ab
+size 294255

probe/000001__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91d8418d06b8140818dbb0d840a8a12a1b62e1077d575e15a82fac65afb72f90
+size 281320

probe/000002__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13e72f1c4d04b3cc7c7b85a601b390e2ef490c98a870b7c03a6b4c8ecdbef28a
+size 258175

probe/000002__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:905ebf19f8b05c31a326064a786bdc02707ef5ddc0b0e084bc7b9ce90bb4fc18
+size 294114

probe/000002__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e4ddcef5636db2929d871e59d306011a6e2ca9dceea679b8f55d29f3ac2f429
+size 281430

probe/000004__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b0ceca0393d0f3d5982ae233e046d381a0d6532dc0c42cfa87a9558b321b1e
+size 258424

probe/000004__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d4e4912c90a0653d83c09155e64aa85b199fb6b09612d4d637c35fcf80ecbb3
+size 294593

probe/000004__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c146b1d4eb1980ec8dc19db3d628b752e82057c4cfebea2f859382fa32fa9ea
+size 281479

probe/000008__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:582b44db35f08fb878745cb7a8a378ed7ea644dab5d74784957e4662c323418a
+size 259184

probe/000008__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bfd6d2c774324988cb2c1f00c5d853fcc4a977e8fe48c8864a99400471bb093
+size 295752

probe/000008__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d583322184d890135b0dbe66af6a2daaf3f855b8ea6799ab557220f2ae8df8d4
+size 281817

probe/000016__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efdae59ae0f4008450f5b6f38b43df4ad20d2acb7b6702b0e21416b12aab4eb6
+size 257856

probe/000016__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5185bc622afbd26dc5df10cb0419f7c1e8c75d68b9689946e485a19116adcc96
+size 294648

probe/000016__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23eee7e67c921df64701476497c820cc3a5dcc0e7374ec31437bbc32e426fdc4
+size 281081

probe/000032__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4fbfb9f736bbbb533fe1fabca95945c39092441e767f50470b908ebf4eccb2a
+size 252191

probe/000032__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:420a4e375b28920d0a62bbf7d5dde1e2fed3da560d174f0b73e8dad01925ae37
+size 285122

probe/000032__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ee3c4148b6100c6b7ea4f846e7e57d76de9885a739a768ac2a2a8e7240e38f7
+size 275753

probe/000064__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5d595c0374b8df6a81b71112a55f08fa4dc8c9c36a2cc50f070d39348c245b7
+size 261763

probe/000064__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9917335c928f8d3e2b792d4b68981c4c054214dd1d9c49cf1c7a34cd0fa0aa8c
+size 298039

probe/000064__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6d1941be45360c8a766d7eec6f00988aa59c5449a9b05a1cb1e1135d9a53058
+size 281409

probe/000128__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76369027ac8f68aacf7b0b379dba39e6019c48dc890c13d434f1037f6326c984
+size 266274

probe/000128__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd3178ba826effafc5bf678483877a171dc3f00cbb9031e0e46081fed11c58b9
+size 304163

probe/000128__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c61f5b6e7adac711f677e6fd1676c071ca785354f606d6d781e10e56de04cc2
+size 287754

probe/000200__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d20edf26da1da99ef5bd5ae29d395d3b0c1b2b6c18a069558f8a528f664eb99c
+size 270707

probe/000200__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a77a086e88d81a55ca9391e032f74b2d6bf958d1a13ea851e203d665059f4e1
+size 306948

probe/000200__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e21b08e41a7159cf0e0be0e26d8897c79495fc672766778b573db7a0b03faf60
+size 289681

probe/000256__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fb5cfef48077c4fb170cb8390ed72f5c289ab9eea1f9790f33f3057bccb1086
+size 271407

probe/000256__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7864c4c0ee4f16513adb7532eacbfdd1bfca269b6ae2f6f11ffae237cb2486d3
+size 311175

probe/000256__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9419fac12109ec3296c4a59f43b5d519bd7165254c1ac2ba55779a045c77cf12
+size 294062

probe/000400__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a2ee15860fb61a20b540be4466a01e421695e9ff11e57e56d313af591931af
+size 272698

probe/000400__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79a3100852ddd5666249333872a6b6758ddc9d95b0bf2e6c9517f28c23249300
+size 313190

probe/000400__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80469c98dded7da42122c69435ca9a1441c9b46a87c6cf46059a08f966d76dc3
+size 293275

probe/000600__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99352a530ae1c8502c5a554b28a832a0a39aaa2f584568e9e8b09afb5db3ff77
+size 273985

probe/000600__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5cc225e61623e5ec6ef555f4ae08cf32e355cf7f1c64de5a8c2c0af7a8f0842
+size 313073

probe/000600__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b82a8a3f766d6e68cb0690dabfcc314753be67e0d030e4c07e8eb9e9094e4fa
+size 293701

probe/000800__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bba8c08eae685d898f1007f18fc56993495b4b1d9e6b64eef4ae9046e1b9e7e0
+size 275389

probe/000800__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64173a6805a5e6c36eaf93cd3063eb05f6801dca026424b18b162392128b2615
+size 314370

probe/000800__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:925a7a76a00e017394a747363037ef7d330fd5d00599f10f53c7358663c476b0
+size 295029

probe/001000__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fd65f8d7bdb71a79fd3867d50b1da07d3aff07891257d75223de93e28b201f0
+size 274698

probe/001000__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df2cc2a5f681d37f4ac9e9df7157ec68306a0a87bb084f740841ef524cac7834
+size 314183

probe/001000__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5b9872f641b9b280e68dfafbb4c097f349771baa028d981f70c58d3ccbb9217
+size 294240

probe/001200__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53392986bf3dc2b91afb32e7f3cf4cff7564d787bf53fff7caf9043b0338fb11
+size 274988

probe/001200__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e75fc702ce9855e81c9a78202ebdcf2d138d5bf862813f1d5a5cfd1f912422b2
+size 314446