{
  "model_name": "cortex-dev-1",
  "status": "development-config-not-trained",
  "_comment": [
    "CORTEX dev-scale training configuration.",
    "This describes a model Frankenstein-Labs can actually TRAIN on realistic hardware.",
    "It does NOT describe the distributed checkpoint in this repository, which is a",
    "1.65T-parameter redistribution and is untouched by this pipeline.",
    "Any checkpoint produced from this config is a new, separately trained model."
  ],

  "architecture": "cortex_dense_decoder",
  "hidden_size": 768,
  "num_hidden_layers": 12,
  "num_attention_heads": 12,
  "num_key_value_heads": 4,
  "intermediate_size": 2048,
  "hidden_act": "silu",
  "max_position_embeddings": 2048,
  "rope_theta": 10000.0,
  "rms_norm_eps": 1e-05,
  "attention_bias": false,
  "attention_dropout": 0.0,
  "tie_word_embeddings": true,
  "initializer_range": 0.02,
  "torch_dtype": "float32",
  "use_cache": true,

  "vocab_size": 129280,
  "bos_token_id": 0,
  "eos_token_id": 1,
  "pad_token_id": 2,

  "tokenizer_source": {
    "repo": "Frankenstein-Labs/Cortex-ai",
    "file": "tokenizer.json",
    "license": "MIT",
    "note": "Reused as-is from the distributed repository. The tokenizer is not modified."
  },

  "training": {
    "precision": "float32",
    "micro_batch_size": 2,
    "gradient_accumulation_steps": 8,
    "effective_batch_size": 16,
    "learning_rate": 0.0003,
    "min_learning_rate": 0.00003,
    "weight_decay": 0.1,
    "beta1": 0.9,
    "beta2": 0.95,
    "grad_clip": 1.0,
    "warmup_steps": 100,
    "max_steps": 20000,
    "lr_schedule": "cosine",
    "seed": 1337,
    "log_every": 10,
    "eval_every": 500,
    "save_every": 1000,
    "sequence_length": 512
  },

  "hardware_expectation": {
    "minimum": "1 CPU core, ~2 GiB RAM, runs but very slowly",
    "recommended": "1 GPU with >= 8 GiB VRAM",
    "verified_on": "4 CPU cores, ~15 GiB RAM, no GPU",
    "note": "Smoke-tested on CPU. Full training is not claimed to have been run."
  },

  "provenance": {
    "developer": "Frankenstein-Labs",
    "weights": "randomly initialised by this pipeline, then trained by Frankenstein-Labs",
    "base_model": null,
    "base_model_note": [
      "Deliberately null: no parent model applies to weights produced from this config.",
      "This field must stay null unless the weights are in fact derived from another model."
    ],
    "license": "mit"
  }
}