File size: 2,416 Bytes
6fbe100 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | {
"model_name": "cortex-dev-1",
"status": "development-config-not-trained",
"_comment": [
"CORTEX dev-scale training configuration.",
"This describes a model Frankenstein-Labs can actually TRAIN on realistic hardware.",
"It does NOT describe the distributed checkpoint in this repository, which is a",
"1.65T-parameter redistribution and is untouched by this pipeline.",
"Any checkpoint produced from this config is a new, separately trained model."
],
"architecture": "cortex_dense_decoder",
"hidden_size": 768,
"num_hidden_layers": 12,
"num_attention_heads": 12,
"num_key_value_heads": 4,
"intermediate_size": 2048,
"hidden_act": "silu",
"max_position_embeddings": 2048,
"rope_theta": 10000.0,
"rms_norm_eps": 1e-05,
"attention_bias": false,
"attention_dropout": 0.0,
"tie_word_embeddings": true,
"initializer_range": 0.02,
"torch_dtype": "float32",
"use_cache": true,
"vocab_size": 129280,
"bos_token_id": 0,
"eos_token_id": 1,
"pad_token_id": 2,
"tokenizer_source": {
"repo": "Frankenstein-Labs/Cortex-ai",
"file": "tokenizer.json",
"license": "MIT",
"note": "Reused as-is from the distributed repository. The tokenizer is not modified."
},
"training": {
"precision": "float32",
"micro_batch_size": 2,
"gradient_accumulation_steps": 8,
"effective_batch_size": 16,
"learning_rate": 0.0003,
"min_learning_rate": 0.00003,
"weight_decay": 0.1,
"beta1": 0.9,
"beta2": 0.95,
"grad_clip": 1.0,
"warmup_steps": 100,
"max_steps": 20000,
"lr_schedule": "cosine",
"seed": 1337,
"log_every": 10,
"eval_every": 500,
"save_every": 1000,
"sequence_length": 512
},
"hardware_expectation": {
"minimum": "1 CPU core, ~2 GiB RAM, runs but very slowly",
"recommended": "1 GPU with >= 8 GiB VRAM",
"verified_on": "4 CPU cores, ~15 GiB RAM, no GPU",
"note": "Smoke-tested on CPU. Full training is not claimed to have been run."
},
"provenance": {
"developer": "Frankenstein-Labs",
"weights": "randomly initialised by this pipeline, then trained by Frankenstein-Labs",
"base_model": null,
"base_model_note": [
"Deliberately null: no parent model applies to weights produced from this config.",
"This field must stay null unless the weights are in fact derived from another model."
],
"license": "mit"
}
}
|