tokenizer/meta.json

{
  "id": "hf_byte_bpe_32k_identity",
  "status": "trained",
  "normalization": "identity",
  "normalization_note": "identity leaves original code points unchanged; nfc recomposes canonical equivalents",
  "training_manifest_hash": "1abe78dd7c9dc9f93ac65bce02fb0c5d8ea501c65c12d4f3d660024935174319",
  "n_train_lines": 24683,
  "config": {
    "id": "hf_byte_bpe_32k_identity",
    "family": "hf_byte_bpe",
    "vocab_size": 32000,
    "normalization": "identity",
    "seed": 42
  },
  "embedding_cost": {
    "vocab_size": 32000,
    "parameters": {
      "384": 12288000,
      "768": 24576000,
      "1024": 32768000
    },
    "fp32_bytes": {
      "384": 49152000,
      "768": 98304000,
      "1024": 131072000
    },
    "bf16_bytes": {
      "384": 24576000,
      "768": 49152000,
      "1024": 65536000
    }
  },
  "artifact_bytes": 2884675,
  "reload_reproduced_first_example": true,
  "vocab_collapsed": false,
  "vocab_collapse_note": null,
  "family": "hf_byte_bpe",
  "model_path": "tokenizer\\artifacts\\open_release\\hf_byte_bpe_32k_identity\\tokenizer.json",
  "requested_vocab_size": 32000,
  "actual_vocab_size": 32000,
  "artifact_hash": "0f55e47920211588e6985172cd71338c8aff6226fe912c9668055b44bacc7ce7",
  "byte_fallback": true,
  "seed": 42
}