File size: 736 Bytes
ca0bb3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | from dataclasses import dataclass, field
from typing import Optional
@dataclass
class AmaiaConfig:
vocab_size: int = 32768
d_model: int = 128
n_heads: int = 4
n_layers: int = 4
max_seq: int = 256
dropout: float = 0.1
lr: float = 3e-4
lr_min: float = 1e-5
warmup_steps: int = 500
decay_steps: int = 50000
beta1: float = 0.9
beta2: float = 0.999
weight_decay: float = 0.1
grad_clip: float = 1.0
batch_size: int = 32
chunk_size: int = 128
num_workers: int = 2
ckpt_file: str = "amaia.pt"
sp_model: str = "amaia.model"
log_every: int = 100
ckpt_every: int = 1000
sample_every: int = 500
sample_prompt: str = "The"
max_new_tokens: int = 80
|