TheFireHacker's picture
Upload folder using huggingface_hub
27cd743 verified
Raw History Blame Contribute Delete
4.38 kB
from __future__ import annotations
import copy
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
import yaml
def _deep_merge(base: dict, override: dict) -> dict:
merged = copy.deepcopy(base)
for key, value in override.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
merged[key] = _deep_merge(merged[key], value)
else:
merged[key] = value
return merged
@dataclass
class AttentionConfig:
n_heads: int
head_dim: int
n_kv_heads: int
qk_norm: bool
rope_theta: float
output_gate: bool
bias: bool
@dataclass
class MoEConfig:
n_experts: int
top_k: int
expert_hidden: int
activation: str
router_init_std: float
backend: str
load_balance_coef: float
router_z_loss_coef: float
@dataclass
class InitConfig:
std: float
@dataclass
class ModelConfig:
n_layers: int
d_model: int
vocab_size: int
tie_embeddings: bool
max_seq_len: int
norm_eps: float
attention: AttentionConfig
moe: MoEConfig
init: InitConfig
@dataclass
class MuonConfig:
lr: float
momentum: float
nesterov: bool
ns_steps: int
weight_decay: float
@dataclass
class AdamWConfig:
lr: float
betas: list
eps: float
weight_decay: float
@dataclass
class OptimizerConfig:
muon: MuonConfig
adamw: AdamWConfig
grad_clip: float
@dataclass
class ScheduleConfig:
warmup_steps: int
decay: str
min_lr_ratio: float
wsd_decay_frac: float
@dataclass
class TrainingConfig:
dataset: str
dataset_config: str
total_tokens: int
seq_len: int
global_batch_tokens: int
micro_batch: int
precision: str
compile: bool
checkpoint_every: int
log_every: int
expert_hist_every: int
attn_logit_every: int
@property
def grad_accum_steps(self) -> int:
micro_tokens = self.micro_batch * self.seq_len
assert self.global_batch_tokens % micro_tokens == 0, (
f"global_batch_tokens ({self.global_batch_tokens}) must be a multiple "
f"of micro_batch*seq_len ({micro_tokens})"
)
return self.global_batch_tokens // micro_tokens
@dataclass
class DataConfig:
data_dir: str
checkpoint_dir: str
@dataclass
class TokenizerConfig:
path: str
doc_separator: str
@dataclass
class WandbConfig:
project: str
entity: Optional[str] = None
@dataclass
class RunConfig:
run_name: str
model: ModelConfig
optimizer: OptimizerConfig
schedule: ScheduleConfig
training: TrainingConfig
data: DataConfig
tokenizer: TokenizerConfig
wandb: WandbConfig
raw: dict = field(default_factory=dict, repr=False)
def _build_run_config(d: dict) -> RunConfig:
m = d["model"]
attn = AttentionConfig(**m["attention"])
moe = MoEConfig(**m["moe"])
init = InitConfig(**m["init"])
model = ModelConfig(
n_layers=m["n_layers"],
d_model=m["d_model"],
vocab_size=m["vocab_size"],
tie_embeddings=m["tie_embeddings"],
max_seq_len=m["max_seq_len"],
norm_eps=m["norm_eps"],
attention=attn,
moe=moe,
init=init,
)
o = d["optimizer"]
optimizer = OptimizerConfig(
muon=MuonConfig(**o["muon"]),
adamw=AdamWConfig(**o["adamw"]),
grad_clip=o["grad_clip"],
)
schedule = ScheduleConfig(**d["schedule"])
training = TrainingConfig(**d["training"])
data = DataConfig(**d["data"])
tokenizer = TokenizerConfig(**d["tokenizer"])
wandb = WandbConfig(**d["wandb"])
return RunConfig(
run_name=d["run_name"],
model=model,
optimizer=optimizer,
schedule=schedule,
training=training,
data=data,
tokenizer=tokenizer,
wandb=wandb,
raw=d,
)
def load_config(base_path: str | Path, debug: bool = False, debug_path: str | Path | None = None) -> RunConfig:
base_path = Path(base_path)
with open(base_path) as f:
merged = yaml.safe_load(f)
if debug:
debug_path = Path(debug_path) if debug_path else base_path.parent / "debug.yaml"
with open(debug_path) as f:
override = yaml.safe_load(f)
merged = _deep_merge(merged, override)
return _build_run_config(merged)