run records: final.json, log, probes

config.yaml

@@ -0,0 +1,72 @@
+model: {architecture: ali, d_enc: 256, d_main: 512, n_enc: 2, n_dec: 2, n_main: 6,
+  n_heads_enc: 4, n_heads_main: 8, seq_len: 1024, dropout: 0.0, target_ratio: 6.0,
+  max_chunk_mult: 3.0, dechunk: ema, router_noise: 0.0, rope_base: 10000.0}
+memory:
+  ali:
+    budget_B: 65536
+    staging_capacity: 262144
+    policy: utility
+    candidate_types: [surface]
+    write_rule: mean
+    eta_min: 0.0
+    memory_scope: persistent
+    consolidate_every: 50
+    max_span: 64
+    gate: vector
+    retrieval: true
+    read_dropout: 0.1
+    gate_hidden: 0
+    type_dim: 16
+    gate_bias_init: -2.0
+    probe_seqs: 2
+    probe_reps: 8
+    horizon: 32
+    predictor: mlp
+    lambda_util: 0.1
+    enabled: true
+    inventory_kwargs: {}
+    probe_mix: null
+    util_batch_slots: 1024
+  pkm: {n_keys: 256, topk: 8, heads: 1, d_key: null, value_lr_mult: 1.0}
+objective: {name: ratio, schedule: null, alpha: 0.03, alpha_parity_cap: 0.06, global_anchor: 0.0,
+  premium_clip: 4.0, premium_kind: flores_per_sent, tau1: 0.4, tau2: 0.7, freeze_scope: router,
+  cpc_decay: 0.9, genreg_target_entropy: null, genreg_eta: 0.05}
+data:
+  corpus: corpus
+  langs: [en, tr, fi]
+  mixture: null
+  doc_filters: null
+  byte_classes: scratch/data/byte_classes.json
+training:
+  seed: 0
+  max_steps: 6000
+  batch_size: 32
+  grad_accum: 1
+  lr: 0.0003
+  min_lr_frac: 0.1
+  warmup: 300
+  weight_decay: 0.1
+  grad_clip: 1.0
+  betas: [0.9, 0.95]
+  router_lr_mult: 1.0
+  log_every: 50
+  amp: bf16
+  compile: false
+  device: cuda
+checkpointing:
+  run_name: ali_ali-surface-only_s0
+  runs_dir: runs
+  ckpt_every: 200
+  keep: false
+  milestones: [1000, 3000]
+  resume: auto
+evaluation:
+  eval_every: 200
+  eval_batches: 8
+  probe_sents: 128
+  probe_bpb_batches: 4
+  log_spaced_probe: true
+  dense_early_every: 0
+  dense_early_until: 0
+  dump_reps: true
+  dump_reps_thresholds: [0.3, 0.4, 0.5, 0.6, 0.7]

final.json

@@ -0,0 +1,304 @@
+{
+  "event": "final",
+  "run": "ali_ali-surface-only_s0",
+  "arch": "ali",
+  "objective": "ratio",
+  "condition": "A",
+  "step": 6000,
+  "params": 22785088,
+  "seed": 0,
+  "per_lang": {
+    "en": {
+      "cpc": 0.19025455381256104,
+      "cps": 25.8671875,
+      "bpb": 1.877237854154134,
+      "bytes_per_chunk": 4.9039668509320835,
+      "reads_per_chunk": 0.5906595205357728
+    },
+    "tr": {
+      "cpc": 0.2050488869248191,
+      "cps": 29.0,
+      "bpb": 1.9836637604511769,
+      "bytes_per_chunk": 5.3378401767442805,
+      "reads_per_chunk": 0.5821667464187903
+    },
+    "fi": {
+      "cpc": 0.259901121684121,
+      "cps": 38.1953125,
+      "bpb": 2.1132871381510543,
+      "bytes_per_chunk": 4.068600518384008,
+      "reads_per_chunk": 0.637146715090562
+    }
+  },
+  "bpb_mean": 1.9913962509187886,
+  "gini_cpc": 0.07086496021350519,
+  "gini_cps": 0.08831430490261938,
+  "bpic": 4.762592343854936,
+  "mean_boundary_entropy": 0.08327320218086243,
+  "dechunk": "ema",
+  "router_noise": 0.0,
+  "transitions": [
+    {
+      "step": 0,
+      "phase": 1,
+      "objective": "global",
+      "router_trainable": true,
+      "encoder_trainable": true,
+      "frozen_router_params": 0
+    },
+    {
+      "step": 2400,
+      "phase": 2,
+      "objective": "global",
+      "router_trainable": true,
+      "encoder_trainable": true,
+      "frozen_router_params": 0
+    },
+    {
+      "step": 4200,
+      "phase": 3,
+      "objective": "global",
+      "router_trainable": true,
+      "encoder_trainable": true,
+      "frozen_router_params": 0
+    }
+  ],
+  "probe_overhead_sec_total": 486.7636468410492,
+  "probe_overhead_seq_frac": 0.4999166666666667,
+  "wall_sec": 4367.319061756134,
+  "inventory": {
+    "size": 327673,
+    "admitted": 65536,
+    "staged": 262137,
+    "budget_B": 65536,
+    "staging_capacity": 262144,
+    "collisions": 0,
+    "dropped": 0,
+    "type_rejected": 0,
+    "config_keys": 0,
+    "admitted_surface": 65536,
+    "staged_surface": 262137,
+    "admitted_split": 0,
+    "staged_split": 0,
+    "admitted_merge": 0,
+    "staged_merge": 0,
+    "admitted_skel": 0,
+    "staged_skel": 0,
+    "mean_count": 321.78704833984375,
+    "n_util_obs": 927,
+    "util_count_spearman": 0.05787692414288092
+  },
+  "ali_params": 268096,
+  "ali_config": {
+    "enabled": true,
+    "budget_B": 65536,
+    "staging_capacity": 262144,
+    "policy": "utility",
+    "candidate_types": [
+      "surface"
+    ],
+    "gate": "vector",
+    "retrieval": true,
+    "memory_scope": "persistent",
+    "read_dropout": 0.1,
+    "write_rule": "mean",
+    "eta_min": 0.0,
+    "consolidate_every": 50,
+    "probe_seqs": 2,
+    "probe_reps": 8,
+    "horizon": 32,
+    "predictor": "mlp",
+    "lambda_util": 0.1,
+    "max_span": 64,
+    "byte_classes_path": "/local/scratch/sas245/beetle-hnet/scratch/data/byte_classes.json",
+    "gate_hidden": 0,
+    "type_dim": 16,
+    "gate_bias_init": -2.0
+  },
+  "util_vs_frequency_spearman": 0.05787692414288092,
+  "paired_vs_dropout": {
+    "n": 1879,
+    "pearson": 0.03951643275230993,
+    "spearman": 0.03682888787124551,
+    "mean_abs_discrepancy": 0.7358910441398621,
+    "mean_paired": 0.11356548964977264,
+    "mean_dropout": 0.14920870959758759
+  },
+  "predictor_calibration": {
+    "n": 4216,
+    "all": {
+      "n": 4216,
+      "pearson": 0.1731671184310671,
+      "spearman": 0.21986157444398777,
+      "mae": 0.15914390981197357,
+      "reliability": [
+        {
+          "pred": 0.009084357880055904,
+          "obs": 0.0202009379863739,
+          "n": 422
+        },
+        {
+          "pred": 0.021293800324201584,
+          "obs": 0.05406903848052025,
+          "n": 421
+        },
+        {
+          "pred": 0.028047166764736176,
+          "obs": 0.061708394438028336,
+          "n": 422
+        },
+        {
+          "pred": 0.03469708189368248,
+          "obs": 0.05650480464100838,
+          "n": 421
+        },
+        {
+          "pred": 0.04238409921526909,
+          "obs": 0.05355525761842728,
+          "n": 422
+        },
+        {
+          "pred": 0.0506453812122345,
+          "obs": 0.12060724198818207,
+          "n": 421
+        },
+        {
+          "pred": 0.06036072224378586,
+          "obs": 0.11969655752182007,
+          "n": 422
+        },
+        {
+          "pred": 0.07203800976276398,
+          "obs": 0.12473899126052856,
+          "n": 421
+        },
+        {
+          "pred": 0.08788411319255829,
+          "obs": 0.1447857916355133,
+          "n": 422
+        },
+        {
+          "pred": 0.11606185138225555,
+          "obs": 0.16831223666667938,
+          "n": 422
+        }
+      ]
+    },
+    "admitted": {
+      "n": 4195,
+      "pearson": 0.16744744354721075,
+      "spearman": 0.21406817798821218,
+      "mae": 0.15933582186698914,
+      "reliability": [
+        {
+          "pred": 0.010852980427443981,
+          "obs": 0.027071459218859673,
+          "n": 420
+        },
+        {
+          "pred": 0.021461596712470055,
+          "obs": 0.056002579629421234,
+          "n": 419
+        },
+        {
+          "pred": 0.02820117585361004,
+          "obs": 0.06036455184221268,
+          "n": 420
+        },
+        {
+          "pred": 0.03486092761158943,
+          "obs": 0.058677759021520615,
+          "n": 419
+        },
+        {
+          "pred": 0.042565956711769104,
+          "obs": 0.052283115684986115,
+          "n": 419
+        },
+        {
+          "pred": 0.05083423852920532,
+          "obs": 0.1217113733291626,
+          "n": 420
+        },
+        {
+          "pred": 0.060542166233062744,
+          "obs": 0.11931566148996353,
+          "n": 419
+        },
+        {
+          "pred": 0.07221594452857971,
+          "obs": 0.12506148219108582,
+          "n": 420
+        },
+        {
+          "pred": 0.08802869915962219,
+          "obs": 0.14649838209152222,
+          "n": 419
+        },
+        {
+          "pred": 0.11615332961082458,
+          "obs": 0.16771550476551056,
+          "n": 420
+        }
+      ]
+    },
+    "staged": {
+      "n": 21,
+      "pearson": 0.21218906617925304,
+      "spearman": 0.27792207792207796,
+      "mae": 0.12080670893192291,
+      "reliability": [
+        {
+          "pred": -0.11282563209533691,
+          "obs": -0.14851582050323486,
+          "n": 2
+        },
+        {
+          "pred": -0.08935044705867767,
+          "obs": -0.022793347015976906,
+          "n": 2
+        },
+        {
+          "pred": -0.04074031487107277,
+          "obs": -0.23829758167266846,
+          "n": 3
+        },
+        {
+          "pred": -0.01928403042256832,
+          "obs": -0.17395782470703125,
+          "n": 1
+        },
+        {
+          "pred": -0.011489411816000938,
+          "obs": -0.14177291095256805,
+          "n": 2
+        },
+        {
+          "pred": 0.014031370170414448,
+          "obs": -0.11138230562210083,
+          "n": 3
+        },
+        {
+          "pred": 0.024435658007860184,
+          "obs": -0.09640923887491226,
+          "n": 2
+        },
+        {
+          "pred": 0.03347030654549599,
+          "obs": -0.12943308055400848,
+          "n": 1
+        },
+        {
+          "pred": 0.034968361258506775,
+          "obs": -0.055959004908800125,
+          "n": 2
+        },
+        {
+          "pred": 0.045220475643873215,
+          "obs": -0.0703248456120491,
+          "n": 3
+        }
+      ]
+    }
+  }
+}

probe/000000__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:246e33ccf60e09d4ee9867bdab9fb7a888920df25c1f712f527ffd299adfd882
+size 258090

probe/000000__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7b2abab1e9398b5a4d85c23c7d67562a82e0e04c94ebe64c6bfde72bb87705a
+size 295839

probe/000000__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4facad77fe7e6325fc302615cac5f448983bd45eb3902b924d29bae86113e448
+size 279476

probe/000001__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afd1491bc80c104393ebdce1e0f9b99b771b1d0d477d84cdf0f5643a3b212cce
+size 258158

probe/000001__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29b3ff5e025f20017a35fa98473dfc75cb6d87d6996ca8eb9954eaf66e7e054b
+size 295879

probe/000001__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c9b5bdf1e1e008ffbca625c1824294117e7a1151b77897303d76d9a88e76c6
+size 279413

probe/000002__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be386ffc7be3dea826641b32b15e68ba9d107ed136c65141d7d14f945d728869
+size 258183

probe/000002__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0823d4271a785880951fc5c05c7ccb0445a3aca1e2b3acd995f3991b9bff2068
+size 295590

probe/000002__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d304196efc7d4c4fda00d0b50b93f6d1bca91b1192c6222258ab6936bac119d1
+size 279368

probe/000004__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96726a125a397f0896998011abd3ad567192223c236f221f5d60e4ccc68cf645
+size 257931

probe/000004__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29a7b5634436a7ef7572e04f3548a86c6191658cb0197e386009123153d32b1d
+size 295452

probe/000004__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52327d015ae5dc6889fe51ef19679a323c1e2582da439a09b843e0db0d9d5a25
+size 278986

probe/000008__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d25e7b4f45dc24f2a2daaaffbbd9aee1d55ceaaa5c4a8417ef1d0d94ca27c187
+size 257184

probe/000008__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecc61201b8e8d67a9f8be181427607bcdeb4f75386f5288bc7e99a7a9a4c5d3d
+size 294703

probe/000008__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b19e52f8219a46b344ad5a66ecc63546b0237b6567be6be9821bddde1c93e6fe
+size 277639

probe/000016__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89dafda628a40c212950cb5ad972f34ae770cf2447fb14d77a0484b76fb77f4d
+size 254918

probe/000016__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d04d14a0f5fea1e4032f37c6983715fee28baa383f602d12af03d700a3a1b346
+size 294440

probe/000016__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2850d66424455dd8167e7f87c99080790b1d2324f5bc161bda75138df0580b24
+size 276212

probe/000032__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:985ba2e5f2b687afcda4f5469c86a94354718f021504b52b89559cfa7331f6bc
+size 252141

probe/000032__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b366c8f84ed07bd9957ce927b4b284465a6b658250b6b51145c167f6d8b639d
+size 290322

probe/000032__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a641a008592c27c6fb391eea7523a0f88fb5754fd2a656f83d704e30abc48404
+size 272645

probe/000064__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37767bcfd71b9fc87b60bbde42859563dc5e7843ac96910d623370472d88e16d
+size 256833

probe/000064__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e4dcd5eb85b464bf2589922f3537ec1d8f2f5416d46956a042f37d9489de9b2
+size 294193

probe/000064__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e342f2a4482c21788f012c2f59fb858a15a12d7340a2b375b823dc6fba39af63
+size 273959

probe/000128__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312e94f02151694a71a83d312c05e69a15727cd8dad59acc837b03653c8d8a37
+size 268039

probe/000128__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c9ae7a02182dcb086782edfe15b89b339344c5b4c577026a141203c35f55f48
+size 304783

probe/000128__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88de54514f83e42874821c42aa217fe9ec30d5ea4b8b029d7e0e76162c410220
+size 284432

probe/000200__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54c4f6166d7983ed3aa85808cc068c706eaaae54ee52b1b9767e1ffb9a892da
+size 267576

probe/000200__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbcc6c94b8ebb6a6ac8dd2afccc8e9cf2797941ef4eee38397c289eb1ff0ead3
+size 304530

probe/000200__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:608be63a9f9892477d0ab88ed868996fbf1b3e3b072be25bf8815ec359ccd9b7
+size 286017

probe/000256__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:337614a49ff0a756f0d4a6d81e69a9d5266e433d3e20b3d348c08babc031d30c
+size 271415

probe/000256__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0184b48d8c6b0ec181f27b54dcc9f65bf2160f2395d0509be7d5a826fdb2b853
+size 310708

probe/000256__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:359506225d5d57b196a2ab12f0ceb5756a0c032d1b6b78f42dbe6243e60b7f63
+size 290967

probe/000400__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79b067619040828f66a408080a9a4ba2d33c01f38ac5b204254a441acdcf9dc4
+size 274306

probe/000400__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f92054ae507a0fa14384e5ce86856ff8bbc414a19538e1c1d9e66d160b97aac
+size 315285

probe/000400__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:710a13f3ffb0e0d0c9ace27600e609c7addd24e052183c76771e782a7c1770b0
+size 295432

probe/000600__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf5cc73e4d209d5d2920dc8a90a9c081536474474953f68023cdd99b47d27459
+size 272404

probe/000600__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7a4c7b698a89a1a5abfeb0fd4171d07444930213962fb1b15bedc2b46bf1061
+size 310165

probe/000600__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc0875d78386665465971d0d59d3fabd991f02526cb383f2fb5e1b7567119f2
+size 293504

probe/000800__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ebf5808d74606c39bd0a84bacf0e4e1b5a9cb407cde5f9db7e44349fee4068c
+size 271209

probe/000800__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b18eef14136fac028626038f6190be72efbaacc3f6e501822c197e20275ed048
+size 309071

probe/000800__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2455262741d17474a53305ddb716a28b42d5a67c353b18125aefdd4d9a65e88d
+size 293198

probe/001000__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43572de0c6b263747e5ba1b5bdfdd6817909dce353d79f8ce3dc85396d3a5c9c
+size 269817

probe/001000__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e84b6a032ee06595e7e5d8e3c49e533d4bc50909563b896a9c377f3cb54a51
+size 308020

probe/001000__tr.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f83ab07ec56862ac391b8b1d6fab1043ac14cf7756dddd78a780982001d6694f
+size 292391

probe/001200__en.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:828231220719b578c498cfac009d249be625150f14eafc0f72dfbcdb8a2b8ca5
+size 270006

probe/001200__fi.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a682fe2532e1c2abe78104146788806aa450d85499ef8708d0671db447b26d
+size 310200