caption pack: lexicon difficulty and surprise (bytelex atlas, 11 tokenizers, the diet table)

mini-beatrix-3/caption_pack/lexicon_difficulty.json

@@ -1,7 +1,7 @@
 {
- "utc": "2026-10-06T04:12:56Z",
+ "utc": "2026-10-06T04:13:34Z",
  "pack": "/root/pack_v2",
- "rows_per_source": 300,
+ "rows_per_source": 30000,
  "seed": 20261005,
  "fleet": [
   "gpt2",
@@ -463,550 +463,1425 @@
   }
  },
  "sources": {
+  "dff": {
+   "words_total": 6582510,
+   "words_distinct": 5298,
+   "hapax_share_of_types": 0.169,
+   "mean_word_len": 5.36,
+   "difficulty_fleet_consensus": {
+    "top_share_q10_50_90_types": [
+     0.207,
+     0.355,
+     0.62
+    ],
+    "top_share_q10_50_90_tokens": [
+     0.211,
+     0.351,
+     0.552
+    ],
+    "mean_top_types": 0.39,
+    "mean_top_tokens": 0.37,
+    "entropy_bits_q10_50_90_types": [
+     1.665,
+     2.808,
+     3.457
+    ],
+    "closability_q10_50_90_types": [
+     0.209,
+     0.52,
+     0.916
+    ],
+    "band_share_types": {
+     "easy": 0.673,
+     "mid": 0.17,
+     "trap": 0.156,
+     "novel": 0.001
+    },
+    "band_share_tokens": {
+     "easy": 0.711,
+     "mid": 0.184,
+     "trap": 0.105,
+     "novel": 0.0
+    },
+    "novel_word_ends_types": 5,
+    "novel_word_ends_tokens": 7
+   },
+   "surprise_per_tokenizer_tokens": {
+    "gpt2": {
+     "dead_cell_share": 0.0018,
+     "mean_surprisal_bits_witnessed_steps": 1.975,
+     "unwitnessed_step_share": 0.0186
+    },
+    "smollm2": {
+     "dead_cell_share": 0.0019,
+     "mean_surprisal_bits_witnessed_steps": 2.021,
+     "unwitnessed_step_share": 0.0203
+    },
+    "cl100k_base": {
+     "dead_cell_share": 0.0008,
+     "mean_surprisal_bits_witnessed_steps": 2.026,
+     "unwitnessed_step_share": 0.0132
+    },
+    "o200k_base": {
+     "dead_cell_share": 0.0004,
+     "mean_surprisal_bits_witnessed_steps": 2.173,
+     "unwitnessed_step_share": 0.0091
+    },
+    "llama3": {
+     "dead_cell_share": 0.0007,
+     "mean_surprisal_bits_witnessed_steps": 2.042,
+     "unwitnessed_step_share": 0.013
+    },
+    "qwen25": {
+     "dead_cell_share": 0.0008,
+     "mean_surprisal_bits_witnessed_steps": 2.035,
+     "unwitnessed_step_share": 0.013
+    },
+    "deepseek-v3": {
+     "dead_cell_share": 0.0011,
+     "mean_surprisal_bits_witnessed_steps": 2.066,
+     "unwitnessed_step_share": 0.016
+    },
+    "mistral-v03": {
+     "dead_cell_share": 0.0087,
+     "mean_surprisal_bits_witnessed_steps": 1.794,
+     "unwitnessed_step_share": 0.0671
+    },
+    "xlmr": {
+     "dead_cell_share": 0.0023,
+     "mean_surprisal_bits_witnessed_steps": 2.751,
+     "unwitnessed_step_share": 0.0585
+    },
+    "t5-unigram": {
+     "dead_cell_share": 0.0018,
+     "mean_surprisal_bits_witnessed_steps": 2.012,
+     "unwitnessed_step_share": 0.0257
+    },
+    "bert-wordpiece": {
+     "dead_cell_share": 0.001,
+     "mean_surprisal_bits_witnessed_steps": 2.023,
+     "unwitnessed_step_share": 0.0158
+    }
+   },
+   "diet_tokens": {
+    "cells": 22140311,
+    "under_floor_share": 0.0007,
+    "never_seen_share": 0.0,
+    "mean_entropy_bits_known_cells": 1.968
+   },
+   "rows_sampled": 30274,
+   "rows_total": 121732,
+   "bytes_per_row": 1517.3,
+   "words_per_row": 217.4,
+   "registers": {
+    "tags": {
+     "words_total": 937621,
+     "words_distinct": 1189,
+     "hapax_share_of_types": 0.189,
+     "mean_word_len": 5.4,
+     "difficulty_fleet_consensus": {
+      "top_share_q10_50_90_types": [
+       0.21,
+       0.351,
+       0.59
+      ],
+      "top_share_q10_50_90_tokens": [
+       0.224,
+       0.355,
+       0.615
+      ],
+      "mean_top_types": 0.377,
+      "mean_top_tokens": 0.385,
+      "entropy_bits_q10_50_90_types": [
+       1.837,
+       2.868,
+       3.479
+      ],
+      "closability_q10_50_90_types": [
+       0.161,
+       0.423,
+       0.86
+      ],
+      "band_share_types": {
+       "easy": 0.704,
+       "mid": 0.16,
+       "trap": 0.136,
+       "novel": 0.0
+      },
+      "band_share_tokens": {
+       "easy": 0.716,
+       "mid": 0.16,
+       "trap": 0.124,
+       "novel": 0.0
+      },
+      "novel_word_ends_types": 0,
+      "novel_word_ends_tokens": 0
+     },
+     "surprise_per_tokenizer_tokens": {
+      "gpt2": {
+       "dead_cell_share": 0.0021,
+       "mean_surprisal_bits_witnessed_steps": 2.054,
+       "unwitnessed_step_share": 0.0164
+      },
+      "smollm2": {
+       "dead_cell_share": 0.0027,
+       "mean_surprisal_bits_witnessed_steps": 2.124,
+       "unwitnessed_step_share": 0.0209
+      },
+      "cl100k_base": {
+       "dead_cell_share": 0.0018,
+       "mean_surprisal_bits_witnessed_steps": 2.148,
+       "unwitnessed_step_share": 0.016
+      },
+      "o200k_base": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.303,
+       "unwitnessed_step_share": 0.0083
+      },
+      "llama3": {
+       "dead_cell_share": 0.0018,
+       "mean_surprisal_bits_witnessed_steps": 2.169,
+       "unwitnessed_step_share": 0.0159
+      },
+      "qwen25": {
+       "dead_cell_share": 0.0018,
+       "mean_surprisal_bits_witnessed_steps": 2.157,
+       "unwitnessed_step_share": 0.0157
+      },
+      "deepseek-v3": {
+       "dead_cell_share": 0.0023,
+       "mean_surprisal_bits_witnessed_steps": 2.205,
+       "unwitnessed_step_share": 0.0166
+      },
+      "mistral-v03": {
+       "dead_cell_share": 0.0089,
+       "mean_surprisal_bits_witnessed_steps": 1.881,
+       "unwitnessed_step_share": 0.0782
+      },
+      "xlmr": {
+       "dead_cell_share": 0.0009,
+       "mean_surprisal_bits_witnessed_steps": 2.888,
+       "unwitnessed_step_share": 0.0669
+      },
+      "t5-unigram": {
+       "dead_cell_share": 0.0022,
+       "mean_surprisal_bits_witnessed_steps": 2.109,
+       "unwitnessed_step_share": 0.0232
+      },
+      "bert-wordpiece": {
+       "dead_cell_share": 0.0017,
+       "mean_surprisal_bits_witnessed_steps": 2.045,
+       "unwitnessed_step_share": 0.0319
+      }
+     },
+     "diet_tokens": {
+      "cells": 3188381,
+      "under_floor_share": 0.0012,
+      "never_seen_share": 0.0,
+      "mean_entropy_bits_known_cells": 2.031
+     }
+    },
+    "english": {
+     "words_total": 4155139,
+     "words_distinct": 4958,
+     "hapax_share_of_types": 0.188,
+     "mean_word_len": 5.3,
+     "difficulty_fleet_consensus": {
+      "top_share_q10_50_90_types": [
+       0.207,
+       0.355,
+       0.615
+      ],
+      "top_share_q10_50_90_tokens": [
+       0.19,
+       0.341,
+       0.552
+      ],
+      "mean_top_types": 0.39,
+      "mean_top_tokens": 0.366,
+      "entropy_bits_q10_50_90_types": [
+       1.665,
+       2.808,
+       3.457
+      ],
+      "closability_q10_50_90_types": [
+       0.219,
+       0.545,
+       0.918
+      ],
+      "band_share_types": {
+       "easy": 0.674,
+       "mid": 0.171,
+       "trap": 0.154,
+       "novel": 0.001
+      },
+      "band_share_tokens": {
+       "easy": 0.712,
+       "mid": 0.187,
+       "trap": 0.102,
+       "novel": 0.0
+      },
+      "novel_word_ends_types": 5,
+      "novel_word_ends_tokens": 5
+     },
+     "surprise_per_tokenizer_tokens": {
+      "gpt2": {
+       "dead_cell_share": 0.0018,
+       "mean_surprisal_bits_witnessed_steps": 1.958,
+       "unwitnessed_step_share": 0.0189
+      },
+      "smollm2": {
+       "dead_cell_share": 0.0019,
+       "mean_surprisal_bits_witnessed_steps": 1.996,
+       "unwitnessed_step_share": 0.0205
+      },
+      "cl100k_base": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.004,
+       "unwitnessed_step_share": 0.0127
+      },
+      "o200k_base": {
+       "dead_cell_share": 0.0003,
+       "mean_surprisal_bits_witnessed_steps": 2.143,
+       "unwitnessed_step_share": 0.0092
+      },
+      "llama3": {
+       "dead_cell_share": 0.0005,
+       "mean_surprisal_bits_witnessed_steps": 2.02,
+       "unwitnessed_step_share": 0.0124
+      },
+      "qwen25": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.013,
+       "unwitnessed_step_share": 0.0126
+      },
+      "deepseek-v3": {
+       "dead_cell_share": 0.001,
+       "mean_surprisal_bits_witnessed_steps": 2.035,
+       "unwitnessed_step_share": 0.0158
+      },
+      "mistral-v03": {
+       "dead_cell_share": 0.0096,
+       "mean_surprisal_bits_witnessed_steps": 1.774,
+       "unwitnessed_step_share": 0.0664
+      },
+      "xlmr": {
+       "dead_cell_share": 0.0029,
+       "mean_surprisal_bits_witnessed_steps": 2.726,
+       "unwitnessed_step_share": 0.0571
+      },
+      "t5-unigram": {
+       "dead_cell_share": 0.0019,
+       "mean_surprisal_bits_witnessed_steps": 1.986,
+       "unwitnessed_step_share": 0.0265
+      },
+      "bert-wordpiece": {
+       "dead_cell_share": 0.001,
+       "mean_surprisal_bits_witnessed_steps": 2.02,
+       "unwitnessed_step_share": 0.013
+      }
+     },
+     "diet_tokens": {
+      "cells": 13730230,
+      "under_floor_share": 0.0006,
+      "never_seen_share": 0.0,
+      "mean_entropy_bits_known_cells": 1.927
+     }
+    }
+   }
+  },
+  "scf": {
+   "words_total": 8481824,
+   "words_distinct": 6363,
+   "hapax_share_of_types": 0.2,
+   "mean_word_len": 5.44,
+   "difficulty_fleet_consensus": {
+    "top_share_q10_50_90_types": [
+     0.208,
+     0.355,
+     0.597
+    ],
+    "top_share_q10_50_90_tokens": [
+     0.199,
+     0.352,
+     0.552
+    ],
+    "mean_top_types": 0.386,
+    "mean_top_tokens": 0.368,
+    "entropy_bits_q10_50_90_types": [
+     1.741,
+     2.814,
+     3.458
+    ],
+    "closability_q10_50_90_types": [
+     0.213,
+     0.514,
+     0.912
+    ],
+    "band_share_types": {
+     "easy": 0.676,
+     "mid": 0.177,
+     "trap": 0.146,
+     "novel": 0.001
+    },
+    "band_share_tokens": {
+     "easy": 0.708,
+     "mid": 0.182,
+     "trap": 0.109,
+     "novel": 0.001
+    },
+    "novel_word_ends_types": 4,
+    "novel_word_ends_tokens": 4987
+   },
+   "surprise_per_tokenizer_tokens": {
+    "gpt2": {
+     "dead_cell_share": 0.0015,
+     "mean_surprisal_bits_witnessed_steps": 2.016,
+     "unwitnessed_step_share": 0.017
+    },
+    "smollm2": {
+     "dead_cell_share": 0.0014,
+     "mean_surprisal_bits_witnessed_steps": 2.048,
+     "unwitnessed_step_share": 0.0195
+    },
+    "cl100k_base": {
+     "dead_cell_share": 0.0008,
+     "mean_surprisal_bits_witnessed_steps": 2.056,
+     "unwitnessed_step_share": 0.0125
+    },
+    "o200k_base": {
+     "dead_cell_share": 0.0002,
+     "mean_surprisal_bits_witnessed_steps": 2.206,
+     "unwitnessed_step_share": 0.0085
+    },
+    "llama3": {
+     "dead_cell_share": 0.0007,
+     "mean_surprisal_bits_witnessed_steps": 2.08,
+     "unwitnessed_step_share": 0.0124
+    },
+    "qwen25": {
+     "dead_cell_share": 0.0008,
+     "mean_surprisal_bits_witnessed_steps": 2.069,
+     "unwitnessed_step_share": 0.0121
+    },
+    "deepseek-v3": {
+     "dead_cell_share": 0.0005,
+     "mean_surprisal_bits_witnessed_steps": 2.097,
+     "unwitnessed_step_share": 0.0158
+    },
+    "mistral-v03": {
+     "dead_cell_share": 0.0042,
+     "mean_surprisal_bits_witnessed_steps": 1.824,
+     "unwitnessed_step_share": 0.0661
+    },
+    "xlmr": {
+     "dead_cell_share": 0.0016,
+     "mean_surprisal_bits_witnessed_steps": 2.744,
+     "unwitnessed_step_share": 0.0457
+    },
+    "t5-unigram": {
+     "dead_cell_share": 0.0016,
+     "mean_surprisal_bits_witnessed_steps": 2.056,
+     "unwitnessed_step_share": 0.0211
+    },
+    "bert-wordpiece": {
+     "dead_cell_share": 0.001,
+     "mean_surprisal_bits_witnessed_steps": 2.04,
+     "unwitnessed_step_share": 0.0107
+    }
+   },
+   "diet_tokens": {
+    "cells": 29161727,
+    "under_floor_share": 0.0003,
+    "never_seen_share": 0.0002,
+    "mean_entropy_bits_known_cells": 1.982
+   },
+   "rows_sampled": 42080,
+   "rows_total": 42257,
+   "bytes_per_row": 1437.5,
+   "words_per_row": 201.6,
+   "registers": {
+    "tags": {
+     "words_total": 1257465,
+     "words_distinct": 1218,
+     "hapax_share_of_types": 0.214,
+     "mean_word_len": 5.62,
+     "difficulty_fleet_consensus": {
+      "top_share_q10_50_90_types": [
+       0.211,
+       0.355,
+       0.574
+      ],
+      "top_share_q10_50_90_tokens": [
+       0.219,
+       0.355,
+       0.522
+      ],
+      "mean_top_types": 0.378,
+      "mean_top_tokens": 0.372,
+      "entropy_bits_q10_50_90_types": [
+       1.849,
+       2.838,
+       3.458
+      ],
+      "closability_q10_50_90_types": [
+       0.171,
+       0.424,
+       0.86
+      ],
+      "band_share_types": {
+       "easy": 0.699,
+       "mid": 0.172,
+       "trap": 0.13,
+       "novel": 0.0
+      },
+      "band_share_tokens": {
+       "easy": 0.701,
+       "mid": 0.208,
+       "trap": 0.091,
+       "novel": 0.0
+      },
+      "novel_word_ends_types": 0,
+      "novel_word_ends_tokens": 0
+     },
+     "surprise_per_tokenizer_tokens": {
+      "gpt2": {
+       "dead_cell_share": 0.0022,
+       "mean_surprisal_bits_witnessed_steps": 2.078,
+       "unwitnessed_step_share": 0.0219
+      },
+      "smollm2": {
+       "dead_cell_share": 0.0023,
+       "mean_surprisal_bits_witnessed_steps": 2.102,
+       "unwitnessed_step_share": 0.0262
+      },
+      "cl100k_base": {
+       "dead_cell_share": 0.0007,
+       "mean_surprisal_bits_witnessed_steps": 2.16,
+       "unwitnessed_step_share": 0.0173
+      },
+      "o200k_base": {
+       "dead_cell_share": 0.0,
+       "mean_surprisal_bits_witnessed_steps": 2.267,
+       "unwitnessed_step_share": 0.0129
+      },
+      "llama3": {
+       "dead_cell_share": 0.0007,
+       "mean_surprisal_bits_witnessed_steps": 2.195,
+       "unwitnessed_step_share": 0.0173
+      },
+      "qwen25": {
+       "dead_cell_share": 0.0007,
+       "mean_surprisal_bits_witnessed_steps": 2.176,
+       "unwitnessed_step_share": 0.0165
+      },
+      "deepseek-v3": {
+       "dead_cell_share": 0.0002,
+       "mean_surprisal_bits_witnessed_steps": 2.175,
+       "unwitnessed_step_share": 0.0212
+      },
+      "mistral-v03": {
+       "dead_cell_share": 0.0063,
+       "mean_surprisal_bits_witnessed_steps": 1.884,
+       "unwitnessed_step_share": 0.0834
+      },
+      "xlmr": {
+       "dead_cell_share": 0.0003,
+       "mean_surprisal_bits_witnessed_steps": 2.79,
+       "unwitnessed_step_share": 0.0488
+      },
+      "t5-unigram": {
+       "dead_cell_share": 0.0031,
+       "mean_surprisal_bits_witnessed_steps": 2.107,
+       "unwitnessed_step_share": 0.0225
+      },
+      "bert-wordpiece": {
+       "dead_cell_share": 0.0028,
+       "mean_surprisal_bits_witnessed_steps": 2.062,
+       "unwitnessed_step_share": 0.0218
+      }
+     },
+     "diet_tokens": {
+      "cells": 4556370,
+      "under_floor_share": 0.0001,
+      "never_seen_share": 0.0,
+      "mean_entropy_bits_known_cells": 2.044
+     }
+    },
+    "english": {
+     "words_total": 4858976,
+     "words_distinct": 6111,
+     "hapax_share_of_types": 0.235,
+     "mean_word_len": 5.34,
+     "difficulty_fleet_consensus": {
+      "top_share_q10_50_90_types": [
+       0.208,
+       0.355,
+       0.597
+      ],
+      "top_share_q10_50_90_tokens": [
+       0.19,
+       0.343,
+       0.559
+      ],
+      "mean_top_types": 0.386,
+      "mean_top_tokens": 0.364,
+      "entropy_bits_q10_50_90_types": [
+       1.726,
+       2.814,
+       3.459
+      ],
+      "closability_q10_50_90_types": [
+       0.218,
+       0.521,
+       0.914
+      ],
+      "band_share_types": {
+       "easy": 0.677,
+       "mid": 0.177,
+       "trap": 0.145,
+       "novel": 0.001
+      },
+      "band_share_tokens": {
+       "easy": 0.714,
+       "mid": 0.171,
+       "trap": 0.114,
+       "novel": 0.001
+      },
+      "novel_word_ends_types": 4,
+      "novel_word_ends_tokens": 4984
+     },
+     "surprise_per_tokenizer_tokens": {
+      "gpt2": {
+       "dead_cell_share": 0.0015,
+       "mean_surprisal_bits_witnessed_steps": 1.983,
+       "unwitnessed_step_share": 0.0159
+      },
+      "smollm2": {
+       "dead_cell_share": 0.0014,
+       "mean_surprisal_bits_witnessed_steps": 2.016,
+       "unwitnessed_step_share": 0.0178
+      },
+      "cl100k_base": {
+       "dead_cell_share": 0.001,
+       "mean_surprisal_bits_witnessed_steps": 2.019,
+       "unwitnessed_step_share": 0.0115
+      },
+      "o200k_base": {
+       "dead_cell_share": 0.0004,
+       "mean_surprisal_bits_witnessed_steps": 2.167,
+       "unwitnessed_step_share": 0.0079
+      },
+      "llama3": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.04,
+       "unwitnessed_step_share": 0.0114
+      },
+      "qwen25": {
+       "dead_cell_share": 0.001,
+       "mean_surprisal_bits_witnessed_steps": 2.031,
+       "unwitnessed_step_share": 0.0112
+      },
+      "deepseek-v3": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.058,
+       "unwitnessed_step_share": 0.0145
+      },
+      "mistral-v03": {
+       "dead_cell_share": 0.0043,
+       "mean_surprisal_bits_witnessed_steps": 1.8,
+       "unwitnessed_step_share": 0.0587
+      },
+      "xlmr": {
+       "dead_cell_share": 0.0019,
+       "mean_surprisal_bits_witnessed_steps": 2.714,
+       "unwitnessed_step_share": 0.045
+      },
+      "t5-unigram": {
+       "dead_cell_share": 0.0017,
+       "mean_surprisal_bits_witnessed_steps": 2.017,
+       "unwitnessed_step_share": 0.0215
+      },
+      "bert-wordpiece": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.015,
+       "unwitnessed_step_share": 0.009
+      }
+     },
+     "diet_tokens": {
+      "cells": 16235587,
+      "under_floor_share": 0.0004,
+      "never_seen_share": 0.0003,
+      "mean_entropy_bits_known_cells": 1.922
+     }
+    }
+   }
+  },
+  "cc12m": {
+   "words_total": 2543745,
+   "words_distinct": 35439,
+   "hapax_share_of_types": 0.276,
+   "mean_word_len": 5.56,
+   "difficulty_fleet_consensus": {
+    "top_share_q10_50_90_types": [
+     0.207,
+     0.355,
+     0.631
+    ],
+    "top_share_q10_50_90_tokens": [
+     0.19,
+     0.351,
+     0.552
+    ],
+    "mean_top_types": 0.388,
+    "mean_top_tokens": 0.364,
+    "entropy_bits_q10_50_90_types": [
+     1.579,
+     2.805,
+     3.462
+    ],
+    "closability_q10_50_90_types": [
+     0.151,
+     0.466,
+     0.905
+    ],
+    "band_share_types": {
+     "easy": 0.664,
+     "mid": 0.171,
+     "trap": 0.157,
+     "novel": 0.007
+    },
+    "band_share_tokens": {
+     "easy": 0.691,
+     "mid": 0.208,
+     "trap": 0.101,
+     "novel": 0.0
+    },
+    "novel_word_ends_types": 252,
+    "novel_word_ends_tokens": 587
+   },
+   "surprise_per_tokenizer_tokens": {
+    "gpt2": {
+     "dead_cell_share": 0.003,
+     "mean_surprisal_bits_witnessed_steps": 1.945,
+     "unwitnessed_step_share": 0.0212
+    },
+    "smollm2": {
+     "dead_cell_share": 0.0028,
+     "mean_surprisal_bits_witnessed_steps": 1.959,
+     "unwitnessed_step_share": 0.0185
+    },
+    "cl100k_base": {
+     "dead_cell_share": 0.0016,
+     "mean_surprisal_bits_witnessed_steps": 1.985,
+     "unwitnessed_step_share": 0.0162
+    },
+    "o200k_base": {
+     "dead_cell_share": 0.0007,
+     "mean_surprisal_bits_witnessed_steps": 2.119,
+     "unwitnessed_step_share": 0.0095
+    },
+    "llama3": {
+     "dead_cell_share": 0.0014,
+     "mean_surprisal_bits_witnessed_steps": 2.001,
+     "unwitnessed_step_share": 0.0157
+    },
+    "qwen25": {
+     "dead_cell_share": 0.0016,
+     "mean_surprisal_bits_witnessed_steps": 1.995,
+     "unwitnessed_step_share": 0.016
+    },
+    "deepseek-v3": {
+     "dead_cell_share": 0.0017,
+     "mean_surprisal_bits_witnessed_steps": 2.008,
+     "unwitnessed_step_share": 0.0156
+    },
+    "mistral-v03": {
+     "dead_cell_share": 0.0101,
+     "mean_surprisal_bits_witnessed_steps": 1.783,
+     "unwitnessed_step_share": 0.0664
+    },
+    "xlmr": {
+     "dead_cell_share": 0.0029,
+     "mean_surprisal_bits_witnessed_steps": 2.684,
+     "unwitnessed_step_share": 0.0444
+    },
+    "t5-unigram": {
+     "dead_cell_share": 0.0032,
+     "mean_surprisal_bits_witnessed_steps": 1.981,
+     "unwitnessed_step_share": 0.0249
+    },
+    "bert-wordpiece": {
+     "dead_cell_share": 0.0019,
+     "mean_surprisal_bits_witnessed_steps": 2.0,
+     "unwitnessed_step_share": 0.0154
+    }
+   },
+   "diet_tokens": {
+    "cells": 9061436,
+    "under_floor_share": 0.0012,
+    "never_seen_share": 0.0003,
+    "mean_entropy_bits_known_cells": 1.914
+   },
+   "rows_sampled": 27604,
+   "rows_total": 1362918,
+   "bytes_per_row": 693.7,
+   "words_per_row": 92.2,
+   "registers": {
+    "tags": null,
+    "english": {
+     "words_total": 2460933,
+     "words_distinct": 35439,
+     "hapax_share_of_types": 0.276,
+     "mean_word_len": 5.57,
+     "difficulty_fleet_consensus": {
+      "top_share_q10_50_90_types": [
+       0.207,
+       0.355,
+       0.631
+      ],
+      "top_share_q10_50_90_tokens": [
+       0.19,
+       0.352,
+       0.536
+      ],
+      "mean_top_types": 0.388,
+      "mean_top_tokens": 0.366,
+      "entropy_bits_q10_50_90_types": [
+       1.579,
+       2.805,
+       3.462
+      ],
+      "closability_q10_50_90_types": [
+       0.151,
+       0.466,
+       0.905
+      ],
+      "band_share_types": {
+       "easy": 0.664,
+       "mid": 0.171,
+       "trap": 0.157,
+       "novel": 0.007
+      },
+      "band_share_tokens": {
+       "easy": 0.692,
+       "mid": 0.215,
+       "trap": 0.093,
+       "novel": 0.0
+      },
+      "novel_word_ends_types": 252,
+      "novel_word_ends_tokens": 587
+     },
+     "surprise_per_tokenizer_tokens": {
+      "gpt2": {
+       "dead_cell_share": 0.0031,
+       "mean_surprisal_bits_witnessed_steps": 1.96,
+       "unwitnessed_step_share": 0.0219
+      },
+      "smollm2": {
+       "dead_cell_share": 0.0029,
+       "mean_surprisal_bits_witnessed_steps": 1.978,
+       "unwitnessed_step_share": 0.019
+      },
+      "cl100k_base": {
+       "dead_cell_share": 0.0017,
+       "mean_surprisal_bits_witnessed_steps": 2.006,
+       "unwitnessed_step_share": 0.0167
+      },
+      "o200k_base": {
+       "dead_cell_share": 0.0008,
+       "mean_surprisal_bits_witnessed_steps": 2.141,
+       "unwitnessed_step_share": 0.0097
+      },
+      "llama3": {
+       "dead_cell_share": 0.0014,
+       "mean_surprisal_bits_witnessed_steps": 2.022,
+       "unwitnessed_step_share": 0.0162
+      },
+      "qwen25": {
+       "dead_cell_share": 0.0016,
+       "mean_surprisal_bits_witnessed_steps": 2.015,
+       "unwitnessed_step_share": 0.0165
+      },
+      "deepseek-v3": {
+       "dead_cell_share": 0.0017,
+       "mean_surprisal_bits_witnessed_steps": 2.026,
+       "unwitnessed_step_share": 0.0161
+      },
+      "mistral-v03": {
+       "dead_cell_share": 0.0104,
+       "mean_surprisal_bits_witnessed_steps": 1.791,
+       "unwitnessed_step_share": 0.0684
+      },
+      "xlmr": {
+       "dead_cell_share": 0.003,
+       "mean_surprisal_bits_witnessed_steps": 2.698,
+       "unwitnessed_step_share": 0.0458
+      },
+      "t5-unigram": {
+       "dead_cell_share": 0.0033,
+       "mean_surprisal_bits_witnessed_steps": 1.997,
+       "unwitnessed_step_share": 0.0256
+      },
+      "bert-wordpiece": {
+       "dead_cell_share": 0.002,
+       "mean_surprisal_bits_witnessed_steps": 2.016,
+       "unwitnessed_step_share": 0.0159
+      }
+     },
+     "diet_tokens": {
+      "cells": 8785396,
+      "under_floor_share": 0.0012,
+      "never_seen_share": 0.0003,
+      "mean_entropy_bits_known_cells": 1.903
+     }
+    }
+   }
+  },
   "coco": {
-   "words_total": 7941,
-   "words_distinct": 785,
-   "hapax_share_of_types": 0.031,
-   "mean_word_len": 6.13,
+   "words_total": 508840,
+   "words_distinct": 5941,
+   "hapax_share_of_types": 0.027,
+   "mean_word_len": 6.11,
    "difficulty_fleet_consensus": {
     "top_share_q10_50_90_types": [
-     0.229,
+     0.211,
      0.355,
-     0.566
+     0.609
     ],
     "top_share_q10_50_90_tokens": [
      0.23,
      0.371,
      0.797
     ],
-    "mean_top_types": 0.384,
-    "mean_top_tokens": 0.44,
+    "mean_top_types": 0.386,
+    "mean_top_tokens": 0.441,
     "entropy_bits_q10_50_90_types": [
-     1.814,
-     2.884,
-     3.437
+     1.667,
+     2.851,
+     3.444
     ],
     "closability_q10_50_90_types": [
-     0.233,
-     0.506,
-     0.86
+     0.23,
+     0.558,
+     0.905
     ],
     "band_share_types": {
-     "easy": 0.722,
-     "mid": 0.161,
-     "trap": 0.117,
-     "novel": 0.0
+     "easy": 0.684,
+     "mid": 0.173,
+     "trap": 0.143,
+     "novel": 0.001
     },
     "band_share_tokens": {
-     "easy": 0.659,
-     "mid": 0.116,
-     "trap": 0.225,
+     "easy": 0.652,
+     "mid": 0.119,
+     "trap": 0.23,
      "novel": 0.0
     },
-    "novel_word_ends_types": 0,
-    "novel_word_ends_tokens": 0
+    "novel_word_ends_types": 5,
+    "novel_word_ends_tokens": 10
    },
    "surprise_per_tokenizer_tokens": {
     "gpt2": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 1.598,
-     "unwitnessed_step_share": 0.0115
+     "dead_cell_share": 0.0014,
+     "mean_surprisal_bits_witnessed_steps": 1.601,
+     "unwitnessed_step_share": 0.0109
     },
     "smollm2": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 1.572,
-     "unwitnessed_step_share": 0.0084
+     "dead_cell_share": 0.0014,
+     "mean_surprisal_bits_witnessed_steps": 1.578,
+     "unwitnessed_step_share": 0.0087
     },
     "cl100k_base": {
-     "dead_cell_share": 0.0018,
-     "mean_surprisal_bits_witnessed_steps": 1.587,
-     "unwitnessed_step_share": 0.0102
+     "dead_cell_share": 0.0015,
+     "mean_surprisal_bits_witnessed_steps": 1.594,
+     "unwitnessed_step_share": 0.0091
     },
     "o200k_base": {
-     "dead_cell_share": 0.0011,
-     "mean_surprisal_bits_witnessed_steps": 1.734,
-     "unwitnessed_step_share": 0.0052
+     "dead_cell_share": 0.0008,
+     "mean_surprisal_bits_witnessed_steps": 1.733,
+     "unwitnessed_step_share": 0.0045
     },
     "llama3": {
-     "dead_cell_share": 0.0018,
-     "mean_surprisal_bits_witnessed_steps": 1.598,
-     "unwitnessed_step_share": 0.0102
+     "dead_cell_share": 0.0014,
+     "mean_surprisal_bits_witnessed_steps": 1.605,
+     "unwitnessed_step_share": 0.009
     },
     "qwen25": {
-     "dead_cell_share": 0.0018,
-     "mean_surprisal_bits_witnessed_steps": 1.595,
-     "unwitnessed_step_share": 0.0102
+     "dead_cell_share": 0.0014,
+     "mean_surprisal_bits_witnessed_steps": 1.602,
+     "unwitnessed_step_share": 0.0091
     },
     "deepseek-v3": {
-     "dead_cell_share": 0.0013,
-     "mean_surprisal_bits_witnessed_steps": 1.652,
-     "unwitnessed_step_share": 0.0093
+     "dead_cell_share": 0.001,
+     "mean_surprisal_bits_witnessed_steps": 1.658,
+     "unwitnessed_step_share": 0.0085
     },
     "mistral-v03": {
-     "dead_cell_share": 0.0052,
-     "mean_surprisal_bits_witnessed_steps": 1.375,
-     "unwitnessed_step_share": 0.0339
+     "dead_cell_share": 0.0049,
+     "mean_surprisal_bits_witnessed_steps": 1.379,
+     "unwitnessed_step_share": 0.0325
     },
     "xlmr": {
-     "dead_cell_share": 0.0043,
-     "mean_surprisal_bits_witnessed_steps": 2.566,
-     "unwitnessed_step_share": 0.0337
+     "dead_cell_share": 0.0042,
+     "mean_surprisal_bits_witnessed_steps": 2.557,
+     "unwitnessed_step_share": 0.0336
     },
     "t5-unigram": {
-     "dead_cell_share": 0.002,
+     "dead_cell_share": 0.0016,
      "mean_surprisal_bits_witnessed_steps": 1.733,
-     "unwitnessed_step_share": 0.0148
+     "unwitnessed_step_share": 0.0129
     },
     "bert-wordpiece": {
-     "dead_cell_share": 0.0013,
-     "mean_surprisal_bits_witnessed_steps": 1.678,
-     "unwitnessed_step_share": 0.0097
+     "dead_cell_share": 0.0011,
+     "mean_surprisal_bits_witnessed_steps": 1.685,
+     "unwitnessed_step_share": 0.0093
     }
    },
    "diet_tokens": {
-    "cells": 32822,
-    "under_floor_share": 0.0011,
-    "never_seen_share": 0.0001,
-    "mean_entropy_bits_known_cells": 1.786
+    "cells": 2092837,
+    "under_floor_share": 0.0009,
+    "never_seen_share": 0.0,
+    "mean_entropy_bits_known_cells": 1.794
    },
-   "rows_sampled": 300,
+   "rows_sampled": 19227,
    "rows_total": 19341,
-   "bytes_per_row": 250.3,
+   "bytes_per_row": 248.9,
    "words_per_row": 26.5,
    "registers": {
     "tags": null,
     "english": {
-     "words_total": 7341,
-     "words_distinct": 783,
-     "hapax_share_of_types": 0.031,
-     "mean_word_len": 5.98,
+     "words_total": 470386,
+     "words_distinct": 5941,
+     "hapax_share_of_types": 0.027,
+     "mean_word_len": 5.96,
      "difficulty_fleet_consensus": {
       "top_share_q10_50_90_types": [
-       0.229,
+       0.211,
        0.355,
-       0.566
+       0.609
       ],
       "top_share_q10_50_90_tokens": [
-       0.23,
+       0.226,
        0.371,
        0.797
       ],
-      "mean_top_types": 0.384,
-      "mean_top_tokens": 0.438,
+      "mean_top_types": 0.386,
+      "mean_top_tokens": 0.439,
       "entropy_bits_q10_50_90_types": [
-       1.814,
-       2.889,
-       3.437
+       1.667,
+       2.851,
+       3.444
       ],
       "closability_q10_50_90_types": [
-       0.232,
-       0.506,
-       0.86
+       0.23,
+       0.558,
+       0.905
       ],
       "band_share_types": {
-       "easy": 0.723,
-       "mid": 0.161,
-       "trap": 0.116,
-       "novel": 0.0
+       "easy": 0.684,
+       "mid": 0.173,
+       "trap": 0.143,
+       "novel": 0.001
       },
       "band_share_tokens": {
-       "easy": 0.673,
-       "mid": 0.125,
-       "trap": 0.202,
+       "easy": 0.664,
+       "mid": 0.129,
+       "trap": 0.207,
        "novel": 0.0
       },
-      "novel_word_ends_types": 0,
-      "novel_word_ends_tokens": 0
+      "novel_word_ends_types": 5,
+      "novel_word_ends_tokens": 10
      },
      "surprise_per_tokenizer_tokens": {
       "gpt2": {
-       "dead_cell_share": 0.0018,
-       "mean_surprisal_bits_witnessed_steps": 1.649,
-       "unwitnessed_step_share": 0.013
+       "dead_cell_share": 0.0016,
+       "mean_surprisal_bits_witnessed_steps": 1.654,
+       "unwitnessed_step_share": 0.0124
       },
       "smollm2": {
-       "dead_cell_share": 0.0018,
-       "mean_surprisal_bits_witnessed_steps": 1.64,
-       "unwitnessed_step_share": 0.0095
+       "dead_cell_share": 0.0016,
+       "mean_surprisal_bits_witnessed_steps": 1.647,
+       "unwitnessed_step_share": 0.0099
       },
       "cl100k_base": {
-       "dead_cell_share": 0.002,
-       "mean_surprisal_bits_witnessed_steps": 1.652,
-       "unwitnessed_step_share": 0.0116
+       "dead_cell_share": 0.0016,
+       "mean_surprisal_bits_witnessed_steps": 1.66,
+       "unwitnessed_step_share": 0.0104
       },
       "o200k_base": {
-       "dead_cell_share": 0.0012,
-       "mean_surprisal_bits_witnessed_steps": 1.808,
-       "unwitnessed_step_share": 0.0059
+       "dead_cell_share": 0.0009,
+       "mean_surprisal_bits_witnessed_steps": 1.807,
+       "unwitnessed_step_share": 0.0051
       },
       "llama3": {
-       "dead_cell_share": 0.002,
-       "mean_surprisal_bits_witnessed_steps": 1.663,
-       "unwitnessed_step_share": 0.0116
+       "dead_cell_share": 0.0016,
+       "mean_surprisal_bits_witnessed_steps": 1.672,
+       "unwitnessed_step_share": 0.0103
       },
       "qwen25": {
-       "dead_cell_share": 0.002,
-       "mean_surprisal_bits_witnessed_steps": 1.659,
-       "unwitnessed_step_share": 0.0116
+       "dead_cell_share": 0.0016,
+       "mean_surprisal_bits_witnessed_steps": 1.667,
+       "unwitnessed_step_share": 0.0104
       },
       "deepseek-v3": {
-       "dead_cell_share": 0.0015,
-       "mean_surprisal_bits_witnessed_steps": 1.725,
-       "unwitnessed_step_share": 0.0106
+       "dead_cell_share": 0.0012,
+       "mean_surprisal_bits_witnessed_steps": 1.732,
+       "unwitnessed_step_share": 0.0097
       },
       "mistral-v03": {
-       "dead_cell_share": 0.0058,
-       "mean_surprisal_bits_witnessed_steps": 1.416,
-       "unwitnessed_step_share": 0.0385
+       "dead_cell_share": 0.0055,
+       "mean_surprisal_bits_witnessed_steps": 1.421,
+       "unwitnessed_step_share": 0.037
       },
       "xlmr": {
-       "dead_cell_share": 0.0049,
-       "mean_surprisal_bits_witnessed_steps": 2.655,
-       "unwitnessed_step_share": 0.0383
+       "dead_cell_share": 0.0047,
+       "mean_surprisal_bits_witnessed_steps": 2.645,
+       "unwitnessed_step_share": 0.0382
       },
       "t5-unigram": {
-       "dead_cell_share": 0.0022,
-       "mean_surprisal_bits_witnessed_steps": 1.796,
-       "unwitnessed_step_share": 0.0169
+       "dead_cell_share": 0.0018,
+       "mean_surprisal_bits_witnessed_steps": 1.797,
+       "unwitnessed_step_share": 0.0147
       },
       "bert-wordpiece": {
-       "dead_cell_share": 0.0015,
-       "mean_surprisal_bits_witnessed_steps": 1.739,
-       "unwitnessed_step_share": 0.0111
+       "dead_cell_share": 0.0012,
+       "mean_surprisal_bits_witnessed_steps": 1.747,
+       "unwitnessed_step_share": 0.0106
       }
      },
      "diet_tokens": {
-      "cells": 29222,
-      "under_floor_share": 0.0013,
-      "never_seen_share": 0.0001,
-      "mean_entropy_bits_known_cells": 1.766
+      "cells": 1862113,
+      "under_floor_share": 0.001,
+      "never_seen_share": 0.0,
+      "mean_entropy_bits_known_cells": 1.775
      }
     }
    }
   },
-  "scf": {
-   "words_total": 60909,
-   "words_distinct": 1579,
-   "hapax_share_of_types": 0.218,
-   "mean_word_len": 5.44,
+  "danbooru": {
+   "words_total": 7009965,
+   "words_distinct": 40110,
+   "hapax_share_of_types": 0.184,
+   "mean_word_len": 5.46,
    "difficulty_fleet_consensus": {
     "top_share_q10_50_90_types": [
-     0.219,
+     0.205,
      0.355,
-     0.598
+     0.638
     ],
     "top_share_q10_50_90_tokens": [
-     0.199,
-     0.352,
-     0.559
+     0.19,
+     0.341,
+     0.548
     ],
     "mean_top_types": 0.388,
-    "mean_top_tokens": 0.369,
+    "mean_top_tokens": 0.359,
     "entropy_bits_q10_50_90_types": [
-     1.673,
-     2.812,
-     3.448
+     1.564,
+     2.803,
+     3.457
     ],
     "closability_q10_50_90_types": [
-     0.222,
-     0.506,
-     0.912
+     0.096,
+     0.365,
+     0.86
     ],
     "band_share_types": {
-     "easy": 0.679,
-     "mid": 0.177,
-     "trap": 0.144,
-     "novel": 0.001
+     "easy": 0.667,
+     "mid": 0.158,
+     "trap": 0.165,
+     "novel": 0.01
     },
     "band_share_tokens": {
-     "easy": 0.71,
-     "mid": 0.18,
-     "trap": 0.11,
+     "easy": 0.732,
+     "mid": 0.168,
+     "trap": 0.1,
      "novel": 0.0
     },
-    "novel_word_ends_types": 1,
-    "novel_word_ends_tokens": 25
+    "novel_word_ends_types": 382,
+    "novel_word_ends_tokens": 2904
    },
    "surprise_per_tokenizer_tokens": {
     "gpt2": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 2.023,
-     "unwitnessed_step_share": 0.0181
+     "dead_cell_share": 0.0106,
+     "mean_surprisal_bits_witnessed_steps": 2.071,
+     "unwitnessed_step_share": 0.0613
     },
     "smollm2": {
-     "dead_cell_share": 0.0014,
-     "mean_surprisal_bits_witnessed_steps": 2.052,
-     "unwitnessed_step_share": 0.0206
+     "dead_cell_share": 0.0126,
+     "mean_surprisal_bits_witnessed_steps": 2.102,
+     "unwitnessed_step_share": 0.0672
     },
     "cl100k_base": {
-     "dead_cell_share": 0.0008,
-     "mean_surprisal_bits_witnessed_steps": 2.062,
-     "unwitnessed_step_share": 0.0135
+     "dead_cell_share": 0.0072,
+     "mean_surprisal_bits_witnessed_steps": 2.127,
+     "unwitnessed_step_share": 0.0545
     },
     "o200k_base": {
-     "dead_cell_share": 0.0002,
-     "mean_surprisal_bits_witnessed_steps": 2.211,
-     "unwitnessed_step_share": 0.0091
+     "dead_cell_share": 0.0026,
+     "mean_surprisal_bits_witnessed_steps": 2.295,
+     "unwitnessed_step_share": 0.0369
     },
     "llama3": {
-     "dead_cell_share": 0.0007,
-     "mean_surprisal_bits_witnessed_steps": 2.086,
-     "unwitnessed_step_share": 0.0135
+     "dead_cell_share": 0.0058,
+     "mean_surprisal_bits_witnessed_steps": 2.146,
+     "unwitnessed_step_share": 0.0516
     },
     "qwen25": {
-     "dead_cell_share": 0.0008,
-     "mean_surprisal_bits_witnessed_steps": 2.075,
-     "unwitnessed_step_share": 0.0131
+     "dead_cell_share": 0.0071,
+     "mean_surprisal_bits_witnessed_steps": 2.137,
+     "unwitnessed_step_share": 0.0538
     },
     "deepseek-v3": {
-     "dead_cell_share": 0.0005,
-     "mean_surprisal_bits_witnessed_steps": 2.102,
-     "unwitnessed_step_share": 0.0168
+     "dead_cell_share": 0.0062,
+     "mean_surprisal_bits_witnessed_steps": 2.157,
+     "unwitnessed_step_share": 0.0531
     },
     "mistral-v03": {
-     "dead_cell_share": 0.0042,
-     "mean_surprisal_bits_witnessed_steps": 1.828,
-     "unwitnessed_step_share": 0.0674
+     "dead_cell_share": 0.0222,
+     "mean_surprisal_bits_witnessed_steps": 1.878,
+     "unwitnessed_step_share": 0.1253
     },
     "xlmr": {
-     "dead_cell_share": 0.0015,
-     "mean_surprisal_bits_witnessed_steps": 2.753,
-     "unwitnessed_step_share": 0.0466
+     "dead_cell_share": 0.0062,
+     "mean_surprisal_bits_witnessed_steps": 2.833,
+     "unwitnessed_step_share": 0.0869
     },
     "t5-unigram": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 2.059,
-     "unwitnessed_step_share": 0.0223
+     "dead_cell_share": 0.0104,
+     "mean_surprisal_bits_witnessed_steps": 2.128,
+     "unwitnessed_step_share": 0.075
     },
     "bert-wordpiece": {
-     "dead_cell_share": 0.0011,
-     "mean_surprisal_bits_witnessed_steps": 2.046,
-     "unwitnessed_step_share": 0.0119
+     "dead_cell_share": 0.0075,
+     "mean_surprisal_bits_witnessed_steps": 2.109,
+     "unwitnessed_step_share": 0.0496
     }
    },
    "diet_tokens": {
-    "cells": 209568,
-    "under_floor_share": 0.0002,
-    "never_seen_share": 0.0001,
-    "mean_entropy_bits_known_cells": 1.986
+    "cells": 24230571,
+    "under_floor_share": 0.0051,
+    "never_seen_share": 0.0007,
+    "mean_entropy_bits_known_cells": 1.934
    },
-   "rows_sampled": 301,
-   "rows_total": 42257,
-   "bytes_per_row": 1442.8,
-   "words_per_row": 202.4,
+   "rows_sampled": 30160,
+   "rows_total": 606017,
+   "bytes_per_row": 1657.0,
+   "words_per_row": 232.4,
    "registers": {
     "tags": {
-     "words_total": 9014,
-     "words_distinct": 323,
-     "hapax_share_of_types": 0.241,
-     "mean_word_len": 5.64,
+     "words_total": 1681110,
+     "words_distinct": 23869,
+     "hapax_share_of_types": 0.377,
+     "mean_word_len": 5.59,
      "difficulty_fleet_consensus": {
       "top_share_q10_50_90_types": [
-       0.207,
-       0.34,
-       0.564
+       0.201,
+       0.342,
+       0.626
       ],
       "top_share_q10_50_90_tokens": [
        0.211,
        0.355,
-       0.522
+       0.583
       ],
-      "mean_top_types": 0.364,
-      "mean_top_tokens": 0.371,
+      "mean_top_types": 0.379,
+      "mean_top_tokens": 0.377,
       "entropy_bits_q10_50_90_types": [
-       1.887,
-       2.969,
-       3.475
+       1.687,
+       2.819,
+       3.487
       ],
       "closability_q10_50_90_types": [
-       0.188,
-       0.425,
-       0.86
+       0.086,
+       0.32,
+       0.741
       ],
       "band_share_types": {
-       "easy": 0.759,
-       "mid": 0.13,
-       "trap": 0.111,
-       "novel": 0.0
+       "easy": 0.693,
+       "mid": 0.149,
+       "trap": 0.153,
+       "novel": 0.005
       },
       "band_share_tokens": {
-       "easy": 0.705,
-       "mid": 0.203,
-       "trap": 0.092,
-       "novel": 0.0
+       "easy": 0.7,
+       "mid": 0.178,
+       "trap": 0.121,
+       "novel": 0.001
       },
-      "novel_word_ends_types": 0,
-      "novel_word_ends_tokens": 0
+      "novel_word_ends_types": 123,
+      "novel_word_ends_tokens": 1093
      },
      "surprise_per_tokenizer_tokens": {
       "gpt2": {
-       "dead_cell_share": 0.0021,
-       "mean_surprisal_bits_witnessed_steps": 2.081,
-       "unwitnessed_step_share": 0.0226
+       "dead_cell_share": 0.0175,
+       "mean_surprisal_bits_witnessed_steps": 2.175,
+       "unwitnessed_step_share": 0.097
       },
       "smollm2": {
-       "dead_cell_share": 0.0023,
-       "mean_surprisal_bits_witnessed_steps": 2.1,
-       "unwitnessed_step_share": 0.0275
+       "dead_cell_share": 0.0211,
+       "mean_surprisal_bits_witnessed_steps": 2.215,
+       "unwitnessed_step_share": 0.11
       },
       "cl100k_base": {
-       "dead_cell_share": 0.0005,
-       "mean_surprisal_bits_witnessed_steps": 2.163,
-       "unwitnessed_step_share": 0.0182
+       "dead_cell_share": 0.0127,
+       "mean_surprisal_bits_witnessed_steps": 2.265,
+       "unwitnessed_step_share": 0.0881
       },
       "o200k_base": {
-       "dead_cell_share": 0.0,
-       "mean_surprisal_bits_witnessed_steps": 2.27,
-       "unwitnessed_step_share": 0.0135
+       "dead_cell_share": 0.0055,
+       "mean_surprisal_bits_witnessed_steps": 2.441,
+       "unwitnessed_step_share": 0.0596
       },
       "llama3": {
-       "dead_cell_share": 0.0005,
-       "mean_surprisal_bits_witnessed_steps": 2.199,
-       "unwitnessed_step_share": 0.0182
+       "dead_cell_share": 0.0105,
+       "mean_surprisal_bits_witnessed_steps": 2.291,
+       "unwitnessed_step_share": 0.0826
       },
       "qwen25": {
-       "dead_cell_share": 0.0005,
-       "mean_surprisal_bits_witnessed_steps": 2.179,
-       "unwitnessed_step_share": 0.0174
+       "dead_cell_share": 0.0125,
+       "mean_surprisal_bits_witnessed_steps": 2.276,
+       "unwitnessed_step_share": 0.0867
       },
       "deepseek-v3": {
-       "dead_cell_share": 0.0001,
-       "mean_surprisal_bits_witnessed_steps": 2.176,
-       "unwitnessed_step_share": 0.0222
+       "dead_cell_share": 0.0118,
+       "mean_surprisal_bits_witnessed_steps": 2.294,
+       "unwitnessed_step_share": 0.0871
       },
       "mistral-v03": {
-       "dead_cell_share": 0.0066,
-       "mean_surprisal_bits_witnessed_steps": 1.882,
-       "unwitnessed_step_share": 0.0863
+       "dead_cell_share": 0.0337,
+       "mean_surprisal_bits_witnessed_steps": 1.966,
+       "unwitnessed_step_share": 0.1765
       },
       "xlmr": {
-       "dead_cell_share": 0.0002,
-       "mean_surprisal_bits_witnessed_steps": 2.791,
-       "unwitnessed_step_share": 0.0501
+       "dead_cell_share": 0.0094,
+       "mean_surprisal_bits_witnessed_steps": 2.936,
+       "unwitnessed_step_share": 0.115
       },
       "t5-unigram": {
-       "dead_cell_share": 0.0031,
-       "mean_surprisal_bits_witnessed_steps": 2.106,
-       "unwitnessed_step_share": 0.0233
+       "dead_cell_share": 0.0176,
+       "mean_surprisal_bits_witnessed_steps": 2.239,
+       "unwitnessed_step_share": 0.1144
       },
       "bert-wordpiece": {
-       "dead_cell_share": 0.0028,
-       "mean_surprisal_bits_witnessed_steps": 2.063,
-       "unwitnessed_step_share": 0.023
+       "dead_cell_share": 0.0133,
+       "mean_surprisal_bits_witnessed_steps": 2.202,
+       "unwitnessed_step_share": 0.0836
       }
      },
      "diet_tokens": {
-      "cells": 32791,
-      "under_floor_share": 0.0,
-      "never_seen_share": 0.0,
-      "mean_entropy_bits_known_cells": 2.041
+      "cells": 6031919,
+      "under_floor_share": 0.0085,
+      "never_seen_share": 0.0012,
+      "mean_entropy_bits_known_cells": 2.005
      }
     },
     "english": {
-     "words_total": 34773,
-     "words_distinct": 1484,
-     "hapax_share_of_types": 0.282,
-     "mean_word_len": 5.34,
+     "words_total": 5147935,
+     "words_distinct": 31940,
+     "hapax_share_of_types": 0.081,
+     "mean_word_len": 5.41,
      "difficulty_fleet_consensus": {
       "top_share_q10_50_90_types": [
-       0.22,
+       0.205,
        0.355,
-       0.6
+       0.638
       ],
       "top_share_q10_50_90_tokens": [
        0.19,
-       0.343,
-       0.562
+       0.333,
+       0.537
       ],
       "mean_top_types": 0.389,
-      "mean_top_tokens": 0.365,
+      "mean_top_tokens": 0.354,
       "entropy_bits_q10_50_90_types": [
-       1.665,
-       2.812,
-       3.444
+       1.541,
+       2.803,
+       3.457
       ],
       "closability_q10_50_90_types": [
-       0.223,
-       0.507,
-       0.916
+       0.1,
+       0.393,
+       0.86
       ],
       "band_share_types": {
-       "easy": 0.675,
-       "mid": 0.181,
-       "trap": 0.144,
-       "novel": 0.001
+       "easy": 0.663,
+       "mid": 0.161,
+       "trap": 0.166,
+       "novel": 0.01
       },
       "band_share_tokens": {
-       "easy": 0.715,
+       "easy": 0.735,
        "mid": 0.168,
-       "trap": 0.115,
-       "novel": 0.001
+       "trap": 0.096,
+       "novel": 0.0
       },
-      "novel_word_ends_types": 1,
-      "novel_word_ends_tokens": 25
+      "novel_word_ends_types": 312,
+      "novel_word_ends_tokens": 1811
      },
      "surprise_per_tokenizer_tokens": {
       "gpt2": {
-       "dead_cell_share": 0.0015,
-       "mean_surprisal_bits_witnessed_steps": 1.991,
-       "unwitnessed_step_share": 0.0169
+       "dead_cell_share": 0.0086,
+       "mean_surprisal_bits_witnessed_steps": 2.039,
+       "unwitnessed_step_share": 0.0511
       },
       "smollm2": {
-       "dead_cell_share": 0.0012,
-       "mean_surprisal_bits_witnessed_steps": 2.023,
-       "unwitnessed_step_share": 0.0187
+       "dead_cell_share": 0.0102,
+       "mean_surprisal_bits_witnessed_steps": 2.069,
+       "unwitnessed_step_share": 0.0548
       },
       "cl100k_base": {
-       "dead_cell_share": 0.0009,
-       "mean_surprisal_bits_witnessed_steps": 2.027,
-       "unwitnessed_step_share": 0.0123
+       "dead_cell_share": 0.0056,
+       "mean_surprisal_bits_witnessed_steps": 2.09,
+       "unwitnessed_step_share": 0.0447
       },
       "o200k_base": {
-       "dead_cell_share": 0.0003,
-       "mean_surprisal_bits_witnessed_steps": 2.174,
-       "unwitnessed_step_share": 0.0085
+       "dead_cell_share": 0.0017,
+       "mean_surprisal_bits_witnessed_steps": 2.255,
+       "unwitnessed_step_share": 0.0303
       },
       "llama3": {
-       "dead_cell_share": 0.0008,
-       "mean_surprisal_bits_witnessed_steps": 2.047,
-       "unwitnessed_step_share": 0.0122
+       "dead_cell_share": 0.0044,
+       "mean_surprisal_bits_witnessed_steps": 2.107,
+       "unwitnessed_step_share": 0.0426
       },
       "qwen25": {
-       "dead_cell_share": 0.0009,
-       "mean_surprisal_bits_witnessed_steps": 2.039,
-       "unwitnessed_step_share": 0.012
+       "dead_cell_share": 0.0055,
+       "mean_surprisal_bits_witnessed_steps": 2.1,
+       "unwitnessed_step_share": 0.0443
       },
       "deepseek-v3": {
-       "dead_cell_share": 0.0007,
-       "mean_surprisal_bits_witnessed_steps": 2.066,
-       "unwitnessed_step_share": 0.0153
+       "dead_cell_share": 0.0046,
+       "mean_surprisal_bits_witnessed_steps": 2.118,
+       "unwitnessed_step_share": 0.0432
       },
       "mistral-v03": {
-       "dead_cell_share": 0.0041,
-       "mean_surprisal_bits_witnessed_steps": 1.806,
-       "unwitnessed_step_share": 0.0594
+       "dead_cell_share": 0.0191,
+       "mean_surprisal_bits_witnessed_steps": 1.845,
+       "unwitnessed_step_share": 0.112
       },
       "xlmr": {
-       "dead_cell_share": 0.0018,
-       "mean_surprisal_bits_witnessed_steps": 2.725,
-       "unwitnessed_step_share": 0.0456
+       "dead_cell_share": 0.0053,
+       "mean_surprisal_bits_witnessed_steps": 2.806,
+       "unwitnessed_step_share": 0.0803
       },
       "t5-unigram": {
-       "dead_cell_share": 0.0015,
-       "mean_surprisal_bits_witnessed_steps": 2.021,
-       "unwitnessed_step_share": 0.0227
+       "dead_cell_share": 0.0084,
+       "mean_surprisal_bits_witnessed_steps": 2.098,
+       "unwitnessed_step_share": 0.064
       },
       "bert-wordpiece": {
-       "dead_cell_share": 0.0009,
-       "mean_surprisal_bits_witnessed_steps": 2.024,
-       "unwitnessed_step_share": 0.0102
+       "dead_cell_share": 0.0058,
+       "mean_surprisal_bits_witnessed_steps": 2.074,
+       "unwitnessed_step_share": 0.0396
       }
      },
      "diet_tokens": {
-      "cells": 116040,
-      "under_floor_share": 0.0003,
-      "never_seen_share": 0.0002,
-      "mean_entropy_bits_known_cells": 1.928
+      "cells": 17547574,
+      "under_floor_share": 0.0041,
+      "never_seen_share": 0.0006,
+      "mean_entropy_bits_known_cells": 1.907
      }
     }
    }
@@ -1014,326 +1889,330 @@
  },
  "registers": {
   "tags": {
-   "words_total": 9014,
-   "words_distinct": 323,
-   "hapax_share_of_types": 0.241,
-   "mean_word_len": 5.64,
+   "words_total": 3876196,
+   "words_distinct": 23874,
+   "hapax_share_of_types": 0.377,
+   "mean_word_len": 5.55,
    "difficulty_fleet_consensus": {
     "top_share_q10_50_90_types": [
-     0.207,
-     0.34,
-     0.564
+     0.201,
+     0.342,
+     0.626
     ],
     "top_share_q10_50_90_tokens": [
-     0.211,
+     0.219,
      0.355,
-     0.522
+     0.583
     ],
-    "mean_top_types": 0.364,
-    "mean_top_tokens": 0.371,
+    "mean_top_types": 0.379,
+    "mean_top_tokens": 0.377,
     "entropy_bits_q10_50_90_types": [
-     1.887,
-     2.969,
-     3.475
+     1.687,
+     2.819,
+     3.487
     ],
     "closability_q10_50_90_types": [
-     0.188,
-     0.425,
-     0.86
+     0.086,
+     0.32,
+     0.741
     ],
     "band_share_types": {
-     "easy": 0.759,
-     "mid": 0.13,
-     "trap": 0.111,
-     "novel": 0.0
+     "easy": 0.693,
+     "mid": 0.149,
+     "trap": 0.153,
+     "novel": 0.005
     },
     "band_share_tokens": {
-     "easy": 0.705,
-     "mid": 0.203,
-     "trap": 0.092,
+     "easy": 0.704,
+     "mid": 0.183,
+     "trap": 0.112,
      "novel": 0.0
     },
-    "novel_word_ends_types": 0,
-    "novel_word_ends_tokens": 0
+    "novel_word_ends_types": 123,
+    "novel_word_ends_tokens": 1093
    },
    "surprise_per_tokenizer_tokens": {
     "gpt2": {
-     "dead_cell_share": 0.0021,
-     "mean_surprisal_bits_witnessed_steps": 2.081,
-     "unwitnessed_step_share": 0.0226
+     "dead_cell_share": 0.0089,
+     "mean_surprisal_bits_witnessed_steps": 2.113,
+     "unwitnessed_step_share": 0.0537
     },
     "smollm2": {
-     "dead_cell_share": 0.0023,
-     "mean_surprisal_bits_witnessed_steps": 2.1,
-     "unwitnessed_step_share": 0.0275
+     "dead_cell_share": 0.0106,
+     "mean_surprisal_bits_witnessed_steps": 2.154,
+     "unwitnessed_step_share": 0.0618
     },
     "cl100k_base": {
-     "dead_cell_share": 0.0005,
-     "mean_surprisal_bits_witnessed_steps": 2.163,
-     "unwitnessed_step_share": 0.0182
+     "dead_cell_share": 0.0062,
+     "mean_surprisal_bits_witnessed_steps": 2.201,
+     "unwitnessed_step_share": 0.0481
     },
     "o200k_base": {
-     "dead_cell_share": 0.0,
-     "mean_surprisal_bits_witnessed_steps": 2.27,
-     "unwitnessed_step_share": 0.0135
+     "dead_cell_share": 0.0026,
+     "mean_surprisal_bits_witnessed_steps": 2.35,
+     "unwitnessed_step_share": 0.0324
     },
     "llama3": {
-     "dead_cell_share": 0.0005,
-     "mean_surprisal_bits_witnessed_steps": 2.199,
-     "unwitnessed_step_share": 0.0182
+     "dead_cell_share": 0.0052,
+     "mean_surprisal_bits_witnessed_steps": 2.229,
+     "unwitnessed_step_share": 0.0457
     },
     "qwen25": {
-     "dead_cell_share": 0.0005,
-     "mean_surprisal_bits_witnessed_steps": 2.179,
-     "unwitnessed_step_share": 0.0174
+     "dead_cell_share": 0.0061,
+     "mean_surprisal_bits_witnessed_steps": 2.213,
+     "unwitnessed_step_share": 0.0472
     },
     "deepseek-v3": {
-     "dead_cell_share": 0.0001,
-     "mean_surprisal_bits_witnessed_steps": 2.176,
-     "unwitnessed_step_share": 0.0222
+     "dead_cell_share": 0.0058,
+     "mean_surprisal_bits_witnessed_steps": 2.232,
+     "unwitnessed_step_share": 0.0491
     },
     "mistral-v03": {
-     "dead_cell_share": 0.0066,
-     "mean_surprisal_bits_witnessed_steps": 1.882,
-     "unwitnessed_step_share": 0.0863
+     "dead_cell_share": 0.0189,
+     "mean_surprisal_bits_witnessed_steps": 1.917,
+     "unwitnessed_step_share": 0.1231
     },
     "xlmr": {
-     "dead_cell_share": 0.0002,
-     "mean_surprisal_bits_witnessed_steps": 2.791,
-     "unwitnessed_step_share": 0.0501
+     "dead_cell_share": 0.0045,
+     "mean_surprisal_bits_witnessed_steps": 2.875,
+     "unwitnessed_step_share": 0.082
     },
     "t5-unigram": {
-     "dead_cell_share": 0.0031,
-     "mean_surprisal_bits_witnessed_steps": 2.106,
-     "unwitnessed_step_share": 0.0233
+     "dead_cell_share": 0.0092,
+     "mean_surprisal_bits_witnessed_steps": 2.162,
+     "unwitnessed_step_share": 0.0631
     },
     "bert-wordpiece": {
-     "dead_cell_share": 0.0028,
-     "mean_surprisal_bits_witnessed_steps": 2.063,
-     "unwitnessed_step_share": 0.023
+     "dead_cell_share": 0.0071,
+     "mean_surprisal_bits_witnessed_steps": 2.118,
+     "unwitnessed_step_share": 0.0512
     }
    },
    "diet_tokens": {
-    "cells": 32791,
-    "under_floor_share": 0.0,
-    "never_seen_share": 0.0,
-    "mean_entropy_bits_known_cells": 2.041
+    "cells": 13776670,
+    "under_floor_share": 0.004,
+    "never_seen_share": 0.0005,
+    "mean_entropy_bits_known_cells": 2.024
    }
   },
   "english": {
-   "words_total": 42114,
-   "words_distinct": 1962,
-   "hapax_share_of_types": 0.196,
-   "mean_word_len": 5.45,
+   "words_total": 17093369,
+   "words_distinct": 54480,
+   "hapax_share_of_types": 0.182,
+   "mean_word_len": 5.4,
    "difficulty_fleet_consensus": {
     "top_share_q10_50_90_types": [
-     0.222,
+     0.207,
      0.355,
-     0.598
+     0.638
     ],
     "top_share_q10_50_90_tokens": [
      0.19,
-     0.352,
-     0.643
+     0.342,
+     0.552
     ],
     "mean_top_types": 0.39,
-    "mean_top_tokens": 0.378,
+    "mean_top_tokens": 0.364,
     "entropy_bits_q10_50_90_types": [
-     1.677,
-     2.82,
-     3.437
+     1.541,
+     2.789,
+     3.457
     ],
     "closability_q10_50_90_types": [
-     0.227,
-     0.51,
-     0.901
+     0.112,
+     0.413,
+     0.87
     ],
     "band_share_types": {
-     "easy": 0.682,
-     "mid": 0.177,
-     "trap": 0.14,
-     "novel": 0.001
+     "easy": 0.66,
+     "mid": 0.166,
+     "trap": 0.164,
+     "novel": 0.01
     },
     "band_share_tokens": {
-     "easy": 0.708,
-     "mid": 0.161,
-     "trap": 0.131,
-     "novel": 0.001
+     "easy": 0.715,
+     "mid": 0.179,
+     "trap": 0.105,
+     "novel": 0.0
     },
-    "novel_word_ends_types": 1,
-    "novel_word_ends_tokens": 25
+    "novel_word_ends_types": 554,
+    "novel_word_ends_tokens": 7397
    },
    "surprise_per_tokenizer_tokens": {
     "gpt2": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 1.918,
-     "unwitnessed_step_share": 0.0161
+     "dead_cell_share": 0.0039,
+     "mean_surprisal_bits_witnessed_steps": 1.979,
+     "unwitnessed_step_share": 0.0281
     },
     "smollm2": {
-     "dead_cell_share": 0.0014,
-     "mean_surprisal_bits_witnessed_steps": 1.941,
-     "unwitnessed_step_share": 0.0167
+     "dead_cell_share": 0.0044,
+     "mean_surprisal_bits_witnessed_steps": 2.008,
+     "unwitnessed_step_share": 0.0295
     },
     "cl100k_base": {
-     "dead_cell_share": 0.0011,
-     "mean_surprisal_bits_witnessed_steps": 1.947,
-     "unwitnessed_step_share": 0.0121
+     "dead_cell_share": 0.0025,
+     "mean_surprisal_bits_witnessed_steps": 2.022,
+     "unwitnessed_step_share": 0.0226
     },
     "o200k_base": {
-     "dead_cell_share": 0.0005,
-     "mean_surprisal_bits_witnessed_steps": 2.096,
-     "unwitnessed_step_share": 0.0079
+     "dead_cell_share": 0.0008,
+     "mean_surprisal_bits_witnessed_steps": 2.171,
+     "unwitnessed_step_share": 0.0152
     },
     "llama3": {
-     "dead_cell_share": 0.001,
-     "mean_surprisal_bits_witnessed_steps": 1.966,
-     "unwitnessed_step_share": 0.0121
+     "dead_cell_share": 0.0019,
+     "mean_surprisal_bits_witnessed_steps": 2.04,
+     "unwitnessed_step_share": 0.0218
     },
     "qwen25": {
-     "dead_cell_share": 0.0011,
-     "mean_surprisal_bits_witnessed_steps": 1.958,
-     "unwitnessed_step_share": 0.0119
+     "dead_cell_share": 0.0024,
+     "mean_surprisal_bits_witnessed_steps": 2.032,
+     "unwitnessed_step_share": 0.0223
     },
     "deepseek-v3": {
-     "dead_cell_share": 0.0008,
-     "mean_surprisal_bits_witnessed_steps": 1.994,
-     "unwitnessed_step_share": 0.0143
+     "dead_cell_share": 0.0021,
+     "mean_surprisal_bits_witnessed_steps": 2.054,
+     "unwitnessed_step_share": 0.0235
     },
     "mistral-v03": {
-     "dead_cell_share": 0.0044,
-     "mean_surprisal_bits_witnessed_steps": 1.722,
-     "unwitnessed_step_share": 0.0549
+     "dead_cell_share": 0.011,
+     "mean_surprisal_bits_witnessed_steps": 1.792,
+     "unwitnessed_step_share": 0.0774
     },
     "xlmr": {
-     "dead_cell_share": 0.0024,
-     "mean_surprisal_bits_witnessed_steps": 2.71,
-     "unwitnessed_step_share": 0.0441
+     "dead_cell_share": 0.0034,
+     "mean_surprisal_bits_witnessed_steps": 2.739,
+     "unwitnessed_step_share": 0.0584
     },
     "t5-unigram": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 1.973,
-     "unwitnessed_step_share": 0.0214
+     "dead_cell_share": 0.004,
+     "mean_surprisal_bits_witnessed_steps": 2.023,
+     "unwitnessed_step_share": 0.0359
     },
     "bert-wordpiece": {
-     "dead_cell_share": 0.001,
-     "mean_surprisal_bits_witnessed_steps": 1.964,
-     "unwitnessed_step_share": 0.0104
+     "dead_cell_share": 0.0026,
+     "mean_surprisal_bits_witnessed_steps": 2.025,
+     "unwitnessed_step_share": 0.0203
     }
    },
    "diet_tokens": {
-    "cells": 145262,
-    "under_floor_share": 0.0005,
-    "never_seen_share": 0.0002,
-    "mean_entropy_bits_known_cells": 1.895
+    "cells": 58160900,
+    "under_floor_share": 0.0017,
+    "never_seen_share": 0.0003,
+    "mean_entropy_bits_known_cells": 1.911
    }
   },
   "whole_pack_sample": {
-   "words_total": 68850,
-   "words_distinct": 2039,
-   "hapax_share_of_types": 0.153,
-   "mean_word_len": 5.52,
+   "words_total": 25126884,
+   "words_distinct": 61394,
+   "hapax_share_of_types": 0.226,
+   "mean_word_len": 5.45,
    "difficulty_fleet_consensus": {
     "top_share_q10_50_90_types": [
-     0.222,
+     0.206,
      0.355,
-     0.599
+     0.638
     ],
     "top_share_q10_50_90_tokens": [
-     0.207,
-     0.355,
-     0.585
+     0.194,
+     0.352,
+     0.552
     ],
-    "mean_top_types": 0.39,
-    "mean_top_tokens": 0.377,
+    "mean_top_types": 0.389,
+    "mean_top_tokens": 0.367,
     "entropy_bits_q10_50_90_types": [
-     1.677,
-     2.819,
-     3.437
+     1.541,
+     2.791,
+     3.459
     ],
     "closability_q10_50_90_types": [
-     0.223,
-     0.509,
-     0.89
+     0.103,
+     0.4,
+     0.86
     ],
     "band_share_types": {
-     "easy": 0.684,
-     "mid": 0.175,
-     "trap": 0.141,
-     "novel": 0.0
+     "easy": 0.663,
+     "mid": 0.164,
+     "trap": 0.163,
+     "novel": 0.01
     },
     "band_share_tokens": {
-     "easy": 0.704,
-     "mid": 0.172,
-     "trap": 0.123,
+     "easy": 0.712,
+     "mid": 0.18,
+     "trap": 0.107,
      "novel": 0.0
     },
-    "novel_word_ends_types": 1,
-    "novel_word_ends_tokens": 25
+    "novel_word_ends_types": 617,
+    "novel_word_ends_tokens": 8495
    },
    "surprise_per_tokenizer_tokens": {
     "gpt2": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 1.962,
-     "unwitnessed_step_share": 0.0171
+     "dead_cell_share": 0.0043,
+     "mean_surprisal_bits_witnessed_steps": 2.002,
+     "unwitnessed_step_share": 0.0301
     },
     "smollm2": {
-     "dead_cell_share": 0.0015,
-     "mean_surprisal_bits_witnessed_steps": 1.983,
-     "unwitnessed_step_share": 0.0188
+     "dead_cell_share": 0.0048,
+     "mean_surprisal_bits_witnessed_steps": 2.034,
+     "unwitnessed_step_share": 0.0327
     },
     "cl100k_base": {
-     "dead_cell_share": 0.0009,
-     "mean_surprisal_bits_witnessed_steps": 1.993,
-     "unwitnessed_step_share": 0.0131
+     "dead_cell_share": 0.0027,
+     "mean_surprisal_bits_witnessed_steps": 2.048,
+     "unwitnessed_step_share": 0.0247
     },
     "o200k_base": {
-     "dead_cell_share": 0.0003,
-     "mean_surprisal_bits_witnessed_steps": 2.143,
-     "unwitnessed_step_share": 0.0085
+     "dead_cell_share": 0.001,
+     "mean_surprisal_bits_witnessed_steps": 2.2,
+     "unwitnessed_step_share": 0.0166
     },
     "llama3": {
-     "dead_cell_share": 0.0009,
-     "mean_surprisal_bits_witnessed_steps": 2.016,
-     "unwitnessed_step_share": 0.013
+     "dead_cell_share": 0.0022,
+     "mean_surprisal_bits_witnessed_steps": 2.068,
+     "unwitnessed_step_share": 0.0238
     },
     "qwen25": {
-     "dead_cell_share": 0.0009,
-     "mean_surprisal_bits_witnessed_steps": 2.006,
-     "unwitnessed_step_share": 0.0127
+     "dead_cell_share": 0.0027,
+     "mean_surprisal_bits_witnessed_steps": 2.059,
+     "unwitnessed_step_share": 0.0243
     },
     "deepseek-v3": {
-     "dead_cell_share": 0.0006,
-     "mean_surprisal_bits_witnessed_steps": 2.037,
-     "unwitnessed_step_share": 0.0157
+     "dead_cell_share": 0.0024,
+     "mean_surprisal_bits_witnessed_steps": 2.084,
+     "unwitnessed_step_share": 0.0261
     },
     "mistral-v03": {
-     "dead_cell_share": 0.0043,
-     "mean_surprisal_bits_witnessed_steps": 1.761,
-     "unwitnessed_step_share": 0.0626
+     "dead_cell_share": 0.011,
+     "mean_surprisal_bits_witnessed_steps": 1.814,
+     "unwitnessed_step_share": 0.0821
     },
     "xlmr": {
-     "dead_cell_share": 0.0018,
-     "mean_surprisal_bits_witnessed_steps": 2.726,
-     "unwitnessed_step_share": 0.0448
+     "dead_cell_share": 0.0032,
+     "mean_surprisal_bits_witnessed_steps": 2.758,
+     "unwitnessed_step_share": 0.06
     },
     "t5-unigram": {
-     "dead_cell_share": 0.0016,
-     "mean_surprisal_bits_witnessed_steps": 2.012,
-     "unwitnessed_step_share": 0.0212
+     "dead_cell_share": 0.0043,
+     "mean_surprisal_bits_witnessed_steps": 2.048,
+     "unwitnessed_step_share": 0.0375
     },
     "bert-wordpiece": {
-     "dead_cell_share": 0.0011,
-     "mean_surprisal_bits_witnessed_steps": 1.994,
-     "unwitnessed_step_share": 0.0116
+     "dead_cell_share": 0.0029,
+     "mean_surprisal_bits_witnessed_steps": 2.041,
+     "unwitnessed_step_share": 0.0233
     }
    },
    "diet_tokens": {
-    "cells": 242390,
-    "under_floor_share": 0.0003,
-    "never_seen_share": 0.0001,
-    "mean_entropy_bits_known_cells": 1.959
+    "cells": 86686882,
+    "under_floor_share": 0.0018,
+    "never_seen_share": 0.0003,
+    "mean_entropy_bits_known_cells": 1.953
    }
   }
  },
- "implanted_lexicon_hits": {}
+ "implanted_lexicon_hits": {
+  "scf:passag": 1,
+  "cc12m:cades": 1,
+  "danbooru:raye": 1
+ }
 }