{
  "model_type": "exp_train_hybrid",
  "architectures": [
    "HybridLMForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "patch.HybridLMConfig",
    "AutoModelForCausalLM": "patch.HybridLMForCausalLM"
  },
  "format": "exp_train checkpoint: model.safetensors (dense FP32 weights, tied embedding stored under both names), ngram.safetensors (BF16 table + FP32 row second moments), optim_rank*/state_rank* (.safetensors + .json structure; resume only), meta.json (step, tokens, checksums)",
  "architecture": "hybrid LM: Gated DeltaNet + gated softmax attention, hashed n-gram memory",
  "preset": "700m",
  "ngram_params": 1000000000,
  "vocab_size": 60800,
  "bos_token_id": 2,
  "eos_token_id": 3,
  "pad_token_id": 0,
  "max_doc_len": 900,
  "torch_dtype": "float32",
  "model_config": {
    "vocab_size": 60800,
    "d_model": 768,
    "n_layers": 56,
    "full_attn_every": 4,
    "n_heads": 6,
    "n_kv_heads": 2,
    "head_dim": 128,
    "rope_theta": 1000000.0,
    "gdn_heads": 6,
    "gdn_head_dim": 128,
    "gdn_expand_v": 1.0,
    "gdn_conv_size": 4,
    "gdn_variant": "gdn",
    "gdn2_gate_rank": 0,
    "gdn2_lower_bound": 0.0,
    "ffn_dim": 2304,
    "norm_eps": 1e-06,
    "gated_residual": true,
    "tie_embeddings": false,
    "ngram": {
      "enabled": true,
      "orders": [
        2,
        3
      ],
      "heads_per_order": 4,
      "dim": 32,
      "total_params": 1000000000,
      "inject_layers": [
        1,
        27
      ],
      "table_dtype": "bfloat16",
      "lr": 0.02,
      "beta2": 0.99,
      "eps": 1e-08,
      "canonical_tokens": true,
      "stochastic_round": true,
      "sign_pool": 8192
    }
  },
  "modifications": {
    "delta_block": true,
    "delta_blocks": 4,
    "delta_variant": "repo",
    "depth_attn": "off",
    "ngram_canonical_sha256": "0e9f25836e7da68ad664bb0908c7acba962458643ca92a9e2f2d75c54e8ebb2c",
    "layer_drop": 0.25,
    "layer_drop_final": 0.05,
    "gdn_variant": "gdn",
    "tst_bag": 1,
    "tst_steps": 0
  }
}