config.json

{
  "model_type": "nucengram",
  "architectures": [
    "NucEngramModel"
  ],
  "auto_map": {
    "AutoConfig": "configuration_nucengram.NucEngramConfig",
    "AutoModel": "modeling_nucengram.NucEngramModel",
    "AutoModelForMaskedLM": "modeling_nucengram.NucEngramForMaskedLM"
  },
  "backbone": {
    "vocab_size": 9,
    "hidden_size": 512,
    "intermediate_size": 2048,
    "num_hidden_layers": 22,
    "num_attention_heads": 16,
    "max_position_embeddings": 8192,
    "local_attention": 128,
    "rope_theta_global": 160000.0,
    "rope_theta_local": 10000.0,
    "attn_implementation": "sdpa",
    "tie_word_embeddings": true
  },
  "engram": {
    "ngram_orders": [
      3,
      4,
      5,
      6,
      8
    ],
    "n_heads_per_order": 4,
    "d_mem": 48,
    "layer_inject_ids": [
      1,
      4
    ],
    "use_conv": true,
    "gate_temp": 1.0,
    "seed": 0,
    "hidden_size": 512,
    "vocab_size": 9,
    "pad_id": 0
  },
  "max_length": 8192,
  "pad_token_id": 0,
  "hidden_size": 512,
  "torch_dtype": "float32",
  "transformers_version": "5.7.0"
}