Download src/config.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 4.38 kB
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/config.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/config.py
-
curl -L -o config.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/config.py
4.38 kB
| from __future__ import annotations | |
| import copy | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Optional | |
| import yaml | |
| def _deep_merge(base: dict, override: dict) -> dict: | |
| merged = copy.deepcopy(base) | |
| for key, value in override.items(): | |
| if isinstance(value, dict) and isinstance(merged.get(key), dict): | |
| merged[key] = _deep_merge(merged[key], value) | |
| else: | |
| merged[key] = value | |
| return merged | |
| class AttentionConfig: | |
| n_heads: int | |
| head_dim: int | |
| n_kv_heads: int | |
| qk_norm: bool | |
| rope_theta: float | |
| output_gate: bool | |
| bias: bool | |
| class MoEConfig: | |
| n_experts: int | |
| top_k: int | |
| expert_hidden: int | |
| activation: str | |
| router_init_std: float | |
| backend: str | |
| load_balance_coef: float | |
| router_z_loss_coef: float | |
| class InitConfig: | |
| std: float | |
| class ModelConfig: | |
| n_layers: int | |
| d_model: int | |
| vocab_size: int | |
| tie_embeddings: bool | |
| max_seq_len: int | |
| norm_eps: float | |
| attention: AttentionConfig | |
| moe: MoEConfig | |
| init: InitConfig | |
| class MuonConfig: | |
| lr: float | |
| momentum: float | |
| nesterov: bool | |
| ns_steps: int | |
| weight_decay: float | |
| class AdamWConfig: | |
| lr: float | |
| betas: list | |
| eps: float | |
| weight_decay: float | |
| class OptimizerConfig: | |
| muon: MuonConfig | |
| adamw: AdamWConfig | |
| grad_clip: float | |
| class ScheduleConfig: | |
| warmup_steps: int | |
| decay: str | |
| min_lr_ratio: float | |
| wsd_decay_frac: float | |
| class TrainingConfig: | |
| dataset: str | |
| dataset_config: str | |
| total_tokens: int | |
| seq_len: int | |
| global_batch_tokens: int | |
| micro_batch: int | |
| precision: str | |
| compile: bool | |
| checkpoint_every: int | |
| log_every: int | |
| expert_hist_every: int | |
| attn_logit_every: int | |
| def grad_accum_steps(self) -> int: | |
| micro_tokens = self.micro_batch * self.seq_len | |
| assert self.global_batch_tokens % micro_tokens == 0, ( | |
| f"global_batch_tokens ({self.global_batch_tokens}) must be a multiple " | |
| f"of micro_batch*seq_len ({micro_tokens})" | |
| ) | |
| return self.global_batch_tokens // micro_tokens | |
| class DataConfig: | |
| data_dir: str | |
| checkpoint_dir: str | |
| class TokenizerConfig: | |
| path: str | |
| doc_separator: str | |
| class WandbConfig: | |
| project: str | |
| entity: Optional[str] = None | |
| class RunConfig: | |
| run_name: str | |
| model: ModelConfig | |
| optimizer: OptimizerConfig | |
| schedule: ScheduleConfig | |
| training: TrainingConfig | |
| data: DataConfig | |
| tokenizer: TokenizerConfig | |
| wandb: WandbConfig | |
| raw: dict = field(default_factory=dict, repr=False) | |
| def _build_run_config(d: dict) -> RunConfig: | |
| m = d["model"] | |
| attn = AttentionConfig(**m["attention"]) | |
| moe = MoEConfig(**m["moe"]) | |
| init = InitConfig(**m["init"]) | |
| model = ModelConfig( | |
| n_layers=m["n_layers"], | |
| d_model=m["d_model"], | |
| vocab_size=m["vocab_size"], | |
| tie_embeddings=m["tie_embeddings"], | |
| max_seq_len=m["max_seq_len"], | |
| norm_eps=m["norm_eps"], | |
| attention=attn, | |
| moe=moe, | |
| init=init, | |
| ) | |
| o = d["optimizer"] | |
| optimizer = OptimizerConfig( | |
| muon=MuonConfig(**o["muon"]), | |
| adamw=AdamWConfig(**o["adamw"]), | |
| grad_clip=o["grad_clip"], | |
| ) | |
| schedule = ScheduleConfig(**d["schedule"]) | |
| training = TrainingConfig(**d["training"]) | |
| data = DataConfig(**d["data"]) | |
| tokenizer = TokenizerConfig(**d["tokenizer"]) | |
| wandb = WandbConfig(**d["wandb"]) | |
| return RunConfig( | |
| run_name=d["run_name"], | |
| model=model, | |
| optimizer=optimizer, | |
| schedule=schedule, | |
| training=training, | |
| data=data, | |
| tokenizer=tokenizer, | |
| wandb=wandb, | |
| raw=d, | |
| ) | |
| def load_config(base_path: str | Path, debug: bool = False, debug_path: str | Path | None = None) -> RunConfig: | |
| base_path = Path(base_path) | |
| with open(base_path) as f: | |
| merged = yaml.safe_load(f) | |
| if debug: | |
| debug_path = Path(debug_path) if debug_path else base_path.parent / "debug.yaml" | |
| with open(debug_path) as f: | |
| override = yaml.safe_load(f) | |
| merged = _deep_merge(merged, override) | |
| return _build_run_config(merged) | |