File size: 2,148 Bytes
1c07ac0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | from __future__ import annotations
from dataclasses import dataclass, field
from typing import Optional, List
@dataclass
class ModelConfig:
# Core architecture
vocab_size: int
dim: int
n_layers: int
n_heads: int
n_kv_heads: int
ff_mult: float
max_seq_len: int = 1024 # INCREASED: reasoning needs 1024+
# Regularisation
dropout: float = 0.0
# Positional encoding
rope_theta: float = 10000.0
# Normalisation
rms_norm_eps: float = 1e-5
use_qk_norm: bool = False
use_post_norm: bool = False
# Embeddings
tie_embeddings: bool = True
# Attention
use_flash_if_available: bool = True
sliding_window_attention: bool = False
window_size: int = 2048
residual_scale: float = 1.0
# MoE (Mixture of Experts) — future expansion
use_moe: bool = False
moe_every: int = 2
n_experts: int = 8
moe_top_k: int = 2
aux_loss_alpha: float = 1e-2
# MLA (Multi-head Latent Attention) — future expansion
use_mla: bool = False
mla_latent_dim: int = 512
# ── SELF-LEARNING & EVOLUTION ─────────────────────────────
# MOTHER CORE is a self-evolving sovereign AI system
# These flags enable continual learning capabilities
# Continual learning — model can continue learning after deployment
continual_learning: bool = True
# Self-correction — model checks and corrects its own outputs
self_correction: bool = True
# Knowledge consolidation — periodic replay of core facts
knowledge_consolidation: bool = True
# Adaptive learning rate — adjusts per-layer based on gradient norms
adaptive_lr: bool = True
# Memory augmentation — external memory for long-term knowledge
use_external_memory: bool = False
memory_size: int = 10000
# Identity — sovereign AI metadata
model_name: str = "MOTHER CORE"
model_version: str = "v11"
architect: str = "Christopher Kenna"
organisation: str = "MediaStream AI Limited"
sovereign: bool = True
sovereign_nation: str = "United Kingdom"
|