config.json

{
  "architectures": [
    "MicroDistilleryBigramForCausalLM"
  ],
  "model_type": "custom",
  "custom_model_type": "micro-distillery-bigram",
  "vocab_size": 256,
  "max_position_embeddings": 1024,
  "hidden_size": 512,
  "num_hidden_layers": 0,
  "num_attention_heads": 0,
  "intermediate_size": 0,
  "activation_function": "gelu_new",
  "layer_norm_epsilon": 0.00001,
  "use_cache": true,
  "bos_token_id": 0,
  "eos_token_id": 1,
  "transformers_version": "4.44.0",
  "library_name": "micro-distillery",
  "micro_distillery": {
    "format": "bigram-distilled",
    "dialect": "character-level",
    "reference_arch": "tiny-bigram",
    "distill_from": [
      "smollm2-135m",
      "qwen2.5-0.5b",
      "olmo2-1b",
      "tinyllama-1.1b"
    ],
    "harmonic_seed": 1251963776,
    "harmonic_frequencies": [
      1,
      1.618033988749895,
      2,
      3.141592653589793
    ],
    "grpo_config": {
      "group_size": 8,
      "kl_penalty": 0.1
    },
    "vae_config": {
      "latent_dim": 32,
      "filter_threshold": 0.7
    }
  }
}