File size: 736 Bytes
ca0bb3f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
from dataclasses import dataclass, field
from typing import Optional


@dataclass
class AmaiaConfig:
    vocab_size: int = 32768
    d_model: int = 128
    n_heads: int = 4
    n_layers: int = 4
    max_seq: int = 256
    dropout: float = 0.1

    lr: float = 3e-4
    lr_min: float = 1e-5
    warmup_steps: int = 500
    decay_steps: int = 50000
    beta1: float = 0.9
    beta2: float = 0.999
    weight_decay: float = 0.1
    grad_clip: float = 1.0

    batch_size: int = 32
    chunk_size: int = 128
    num_workers: int = 2

    ckpt_file: str = "amaia.pt"
    sp_model: str = "amaia.model"
    log_every: int = 100
    ckpt_every: int = 1000
    sample_every: int = 500
    sample_prompt: str = "The"
    max_new_tokens: int = 80