| { |
| "model_name": "cortex-dev-1", |
| "status": "development-config-not-trained", |
| "_comment": [ |
| "CORTEX dev-scale training configuration.", |
| "This describes a model Frankenstein-Labs can actually TRAIN on realistic hardware.", |
| "It does NOT describe the distributed checkpoint in this repository, which is a", |
| "1.65T-parameter redistribution and is untouched by this pipeline.", |
| "Any checkpoint produced from this config is a new, separately trained model." |
| ], |
|
|
| "architecture": "cortex_dense_decoder", |
| "hidden_size": 768, |
| "num_hidden_layers": 12, |
| "num_attention_heads": 12, |
| "num_key_value_heads": 4, |
| "intermediate_size": 2048, |
| "hidden_act": "silu", |
| "max_position_embeddings": 2048, |
| "rope_theta": 10000.0, |
| "rms_norm_eps": 1e-05, |
| "attention_bias": false, |
| "attention_dropout": 0.0, |
| "tie_word_embeddings": true, |
| "initializer_range": 0.02, |
| "torch_dtype": "float32", |
| "use_cache": true, |
|
|
| "vocab_size": 129280, |
| "bos_token_id": 0, |
| "eos_token_id": 1, |
| "pad_token_id": 2, |
|
|
| "tokenizer_source": { |
| "repo": "Frankenstein-Labs/Cortex-ai", |
| "file": "tokenizer.json", |
| "license": "MIT", |
| "note": "Reused as-is from the distributed repository. The tokenizer is not modified." |
| }, |
|
|
| "training": { |
| "precision": "float32", |
| "micro_batch_size": 2, |
| "gradient_accumulation_steps": 8, |
| "effective_batch_size": 16, |
| "learning_rate": 0.0003, |
| "min_learning_rate": 0.00003, |
| "weight_decay": 0.1, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "warmup_steps": 100, |
| "max_steps": 20000, |
| "lr_schedule": "cosine", |
| "seed": 1337, |
| "log_every": 10, |
| "eval_every": 500, |
| "save_every": 1000, |
| "sequence_length": 512 |
| }, |
|
|
| "hardware_expectation": { |
| "minimum": "1 CPU core, ~2 GiB RAM, runs but very slowly", |
| "recommended": "1 GPU with >= 8 GiB VRAM", |
| "verified_on": "4 CPU cores, ~15 GiB RAM, no GPU", |
| "note": "Smoke-tested on CPU. Full training is not claimed to have been run." |
| }, |
|
|
| "provenance": { |
| "developer": "Frankenstein-Labs", |
| "weights": "randomly initialised by this pipeline, then trained by Frankenstein-Labs", |
| "base_model": null, |
| "base_model_note": [ |
| "Deliberately null: no parent model applies to weights produced from this config.", |
| "This field must stay null unless the weights are in fact derived from another model." |
| ], |
| "license": "mit" |
| } |
| } |
|
|
