File size: 1,858 Bytes
bf64b03 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """
Vortex-7B model configuration.
Optimized for 8GB VRAM (4060 laptop) and MacBook Pro M2/M3.
"""
VORTEX_7B_CONFIG = {
# Model dimensions
"d_model": 4096,
"num_layers": 32,
"num_heads": 32,
"head_dim": 128, # d_model // num_heads
# State-space layer parameters
"d_state": 16, # SSM state dimension
"d_conv": 4, # SSM convolution width
# Attention parameters
"window_size": 512, # Local attention window
"use_flash_attention": True, # CUDA only
# Feed-forward parameters
"ffn_expansion": 4, # Hidden dim = d_model * expansion
"num_domains": 7, # Physics, Math, Chemistry, Biology, Earth, Space, Zoology
# Tokenizer parameters
"vocab_size": 50000,
"max_seq_len": 16384,
# Layer ratio: 60% SSM, 40% attention
"ssm_ratio": 0.6,
# Data types
"dtype": "bfloat16",
# Special tokens
"special_tokens": {
"[PAD]": 0,
"[UNK]": 1,
"[BOS]": 2,
"[EOS]": 3,
"[EQUATION]": 4,
"[/EQUATION]": 5,
"[CITATION]": 6,
"[/CITATION]": 7,
"[MOLECULE]": 8,
"[/MOLECULE]": 9,
"[FIGURE]": 10,
"[TABLE]": 11,
"[MATH]": 12,
"[CHEM]": 13,
"[BIO]": 14,
"[PHYS]": 15,
"[EARTH]": 16,
"[SPACE]": 17,
"[ZOO]": 18,
},
# Domain tags
"domain_tags": ["[MATH]", "[CHEM]", "[BIO]", "[PHYS]", "[EARTH]", "[SPACE]", "[ZOO]"],
# Science module flags (enable/disable for ablation)
"enable_equation_module": True,
"enable_numerical_module": True,
"enable_citation_module": True,
"enable_molecular_module": True,
}
def get_config():
"""Return the 7B configuration dictionary."""
return VORTEX_7B_CONFIG
|