Spaces:
Sleeping
Sleeping
File size: 2,109 Bytes
c407ed8 8098a3f c407ed8 8098a3f c407ed8 8098a3f c407ed8 8098a3f c407ed8 8098a3f c407ed8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | from __future__ import annotations
from dataclasses import asdict, dataclass
@dataclass(frozen=True)
class ModelConfig:
vocab_size: int
block_size: int = 128
n_layer: int = 4
n_head: int = 4
n_embd: int = 128
dropout: float = 0.1
def __post_init__(self) -> None:
if self.block_size <= 0:
raise ValueError("block_size must be positive")
if self.n_layer <= 0:
raise ValueError("n_layer must be positive")
if self.n_head <= 0:
raise ValueError("n_head must be positive")
if self.n_embd <= 0:
raise ValueError("n_embd must be positive")
if self.n_embd % self.n_head != 0:
raise ValueError("n_embd must be divisible by n_head")
if self.vocab_size <= 0:
raise ValueError("vocab_size must be positive")
if not 0.0 <= self.dropout < 1.0:
raise ValueError("dropout must be in the range [0, 1)")
def to_dict(self) -> dict[str, int | float]:
return asdict(self)
@dataclass(frozen=True)
class TrainConfig:
batch_size: int = 32
learning_rate: float = 3e-4
max_steps: int = 1_000
eval_interval: int = 100
eval_batches: int = 20
grad_accum_steps: int = 1
use_amp: bool = False
seed: int = 1337
output_path: str = "runs/tiny-transformer.pt"
loss_history_path: str | None = None
def __post_init__(self) -> None:
if self.batch_size <= 0:
raise ValueError("batch_size must be positive")
if self.learning_rate <= 0:
raise ValueError("learning_rate must be positive")
if self.max_steps <= 0:
raise ValueError("max_steps must be positive")
if self.eval_interval <= 0:
raise ValueError("eval_interval must be positive")
if self.eval_batches <= 0:
raise ValueError("eval_batches must be positive")
if self.grad_accum_steps <= 0:
raise ValueError("grad_accum_steps must be positive")
def to_dict(self) -> dict[str, bool | int | float | str | None]:
return asdict(self)
|