the-net / configuration_omni.py
JacobLinCool's picture
The Net
93ebaf2
Raw History Blame Contribute Delete
3.47 kB
"""Concrete heterogeneous omni architecture; all parameter counts derive from its modules."""
from copy import deepcopy
from transformers import PretrainedConfig
# (width, block count, expert count, FFN expansion, selected experts)
STAGES = {
"text_encoder": [(2048, 6, 1, 4, 1), (4096, 12, 1, 4, 1), (8192, 6, 1, 4, 1)],
"audio_encoder": [(4096, 12, 1, 4, 1), (8192, 24, 1, 4, 1), (12288, 12, 1, 4, 1)],
"image_encoder": [(4096, 8, 1, 4, 1), (8192, 24, 1, 4, 1), (12288, 24, 1, 4, 1)],
"video_encoder": [(4096, 8, 1, 4, 1), (8192, 32, 1, 4, 1), (16384, 44, 1, 4, 1)],
"eeg_encoder": [(2048, 6, 1, 4, 1), (4096, 12, 1, 4, 1), (8192, 6, 1, 4, 1)],
"fusion": [(16384, 30, 8, 3, 2)],
"world_model": [(8192, 48, 128, 2, 2), (12288, 48, 128, 2, 2), (16384, 34, 128, 2, 2)],
"audio_decoder": [(4096, 12, 1, 4, 1), (8192, 84, 8, 2, 2)],
"image_decoder": [(4096, 8, 1, 4, 1), (12288, 52, 16, 2, 2)],
"video_decoder": [(8192, 8, 1, 4, 1), (16384, 88, 16, 2, 2)],
"memory_control": [(8192, 38, 32, 2, 2)],
}
class OmniConfig(PretrainedConfig):
model_type = "the_net_omni"
def __init__(
self,
stages=None,
head_dim=128,
vocab_size=259,
eeg_channels=64,
context_dim=256,
audio_frame_size=320,
patch_size=4,
audio_codebooks=8,
audio_codebook_size=1024,
**kwargs,
):
super().__init__(**kwargs)
self.stages = deepcopy(STAGES if stages is None else stages)
self.head_dim = head_dim
self.vocab_size = vocab_size
self.eeg_channels = eeg_channels
self.context_dim = context_dim
self.audio_frame_size = audio_frame_size
self.patch_size = patch_size
self.audio_codebooks = audio_codebooks
self.audio_codebook_size = audio_codebook_size
for name in (
"head_dim",
"vocab_size",
"eeg_channels",
"context_dim",
"audio_frame_size",
"patch_size",
"audio_codebooks",
"audio_codebook_size",
):
value = getattr(self, name)
if type(value) is not int or value <= 0:
raise ValueError(f"{name} must be a positive integer")
if set(self.stages) != set(STAGES):
raise ValueError("All eleven named components are required")
for component, rows in self.stages.items():
if not rows:
raise ValueError(f"Empty component: {component}")
for row in rows:
if len(row) != 5 or any(type(n) is not int or n < 1 for n in row):
raise ValueError(f"Invalid stage: {row}")
width, _, experts, _, selected = row
if width % head_dim or selected > experts:
raise ValueError(f"Invalid heads or routing: {row}")
@classmethod
def tiny(cls):
stages = {
name: [
(32 * (i + 1), 1, min(experts, 2), expansion, min(selected, 2))
for i, (_, _, experts, expansion, selected) in enumerate(rows)
]
for name, rows in STAGES.items()
}
return cls(
stages=stages,
head_dim=16,
context_dim=32,
audio_frame_size=32,
eeg_channels=8,
audio_codebooks=2,
audio_codebook_size=16,
)
OmniConfig.register_for_auto_class()