Hugging Face's logo

OpenSML
/
OpenSML-Base-175M

Text Generation
MLX
Safetensors
English
base-model
pretrained
small-language-model
apple-silicon
from-scratch
research
Model card Files
xet
 Community
OpenSML-Base-175M
File size: 5,523 Bytes
8f2ec8e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
{
  "best_val_loss": 2.5563359551876785,
  "ema": 2.4005908938776055,
  "initialization": "normal-0.02-residual-scaled-v1",
  "lr_scheduler": {
    "bad_evals": 0,
    "best": 2.5563359551876785,
    "cap": 3.221225472000002e-05,
    "config_fingerprint": "4f636b3a8d4804bd7b10b26b34967c380d566268898be845bf324f8df4e1459f",
    "cooldown_remaining": 0,
    "evaluations": 242,
    "format": "sml-v2-plateau-v1",
    "last_loss": 2.5563359551876785,
    "last_step": 132005,
    "last_tokens": 17300111360,
    "reductions": 9
  },
  "recipe": {
    "batch_rebalance_transition": {
      "format": "sml-research-batch-rebalance-v1",
      "parent_batches": [
        6,
        4,
        2,
        2,
        2
      ],
      "parent_contract": {
        "code": "2c7e9a69271f9817017c2371ae5e874c9229da489b6c0128aadda943c109201d",
        "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
        "recipe": "bb8108e126d9752d656ad7f58d9b81db869e0a624341de57e8e02cbbaaf226ce",
        "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
      },
      "parent_replica_hash": "678b8e8364d8ce87951628aa4ad68022f378c1c5c122f2e7ef7a8e5c7731dca7",
      "parent_step": 963,
      "start_tokens": 124174336
    },
    "batch_transition": {
      "format": "sml-research-batch-transition-v1",
      "parent_batches": [
        4,
        4,
        2,
        2,
        2
      ],
      "parent_contract": {
        "code": "58f75d5d92576c268370e8f3dbb59f7cbc5ef58bcd1cf247a7cf6008c47e7e3c",
        "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
        "recipe": "0a2de034cbafd3bbc101d6ac151602d2ea4889b51deaf9bf9fc49e2c827a93da",
        "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
      },
      "parent_replica_hash": "6a63305e36520f376da3db369a11350c959485ed673f410c017bd6a79190ce5f",
      "parent_step": 125,
      "start_tokens": 14336000
    },
    "batches": [
      7,
      3,
      2,
      2,
      2
    ],
    "betas": [
      0.9,
      0.95
    ],
    "cache_gib_by_mac": [
      16,
      16,
      4,
      4,
      4
    ],
    "continuation": {
      "format": "sml-v2-continuation-v1",
      "parent_contract": {
        "code": "db42e945868bbf75a15d49a3fd95692493d1e1262b465834779ce40ee9f0e59a",
        "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
        "recipe": "e530ae78f2a7d2e70c3c55b0507ce74842d936825eedd27583802cf365a2a793",
        "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
      },
      "parent_replica_hash": "a7a6b1a5986e0b752e502b73a212a01a7c02c36d0d16b889a1ba255267eed819",
      "parent_step": 113313,
      "rewarm_tokens": 50000000,
      "start_lr": 0.0003,
      "start_tokens": 14850113536
    },
    "data_mode": "hf_stream",
    "decay_start_fraction": 0.8,
    "eps": 1e-08,
    "eval_batch_size": 2,
    "eval_batches_per_source": 32,
    "eval_every_tokens": 50000000,
    "grad_accum": 4,
    "grad_clip": 1.0,
    "keep_checkpoints": 3,
    "log_every": 10,
    "min_lr_ratio": 0.1,
    "model": {
      "attention_impl": "fast",
      "bias": false,
      "ce_impl": "metal",
      "d_model": 768,
      "ffn_impl": "packed-metal",
      "loss_dtype": "float32",
      "max_seq_len": 2048,
      "mlp_multiple_of": 256,
      "mlp_ratio": 4.0,
      "n_heads": 12,
      "n_kv_heads": 4,
      "n_layers": 24,
      "qk_norm": true,
      "vocab_size": 32000
    },
    "name": "research-20b-python-176m-continued-to-30000000000",
    "peak_lr": 0.0003,
    "plateau_scheduler": {
      "cooldown": 10,
      "factor": 0.8,
      "format": "sml-v2-plateau-v1",
      "initial_lr": 0.00024,
      "min_delta": 0.002,
      "min_lr": 3e-05,
      "patience": 12
    },
    "playground_publish_best": true,
    "reserve_gib": 16,
    "sample_every_tokens": 50000000,
    "sample_prompts_file": "sample_prompts.json",
    "save_every_tokens": 100000000,
    "scheduler_transition": {
      "baseline_best": 2.801170878969133,
      "format": "sml-v2-scheduler-transition-v1",
      "parent_contract": {
        "code": "db42e945868bbf75a15d49a3fd95692493d1e1262b465834779ce40ee9f0e59a",
        "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
        "recipe": "970e96ed6919947a45d3135a5210496650e7e0f177afe311064ca0532fc9ab0e",
        "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
      },
      "parent_replica_hash": "e7d76ee273de9c808e90454e4f2cc31228594ca68e4619b675ba0b4e6a451861",
      "parent_step": 39896,
      "start_tokens": 5227200512
    },
    "seed": 7337,
    "stream": {
      "dedup_window": 100000,
      "prefetch_batches": 4,
      "read_attempts": 5,
      "shuffle_buffer": 256,
      "tokenizer_corpus_policy": "reuse-frozen"
    },
    "target_tokens": 30000000000,
    "warmup_fraction": 0.01,
    "weight_decay": 0.1
  },
  "replica_hash": "c5b173e6cd9f07144528d9900608275c00be13c1ae0eb543503c05a6db40e8a5",
  "step": 132005,
  "tokens": 17300111360,
  "v2_contract": {
    "code": "6e444060dba8c63ba33e128a43da03df6d927d289318ec51a2c0026fb494ab9f",
    "data": "114cafddb71ce092c8ede1cdddcefe033f30bbab1c093b8593a9e23e36014a76",
    "recipe": "79ca696d2052326b32a71c4fc82729916312a9f97e61c358f5532131a9575d76",
    "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
  },
  "validation_fingerprint": "4ddd461c131d4d27d07bb07f5aab3fcb5a00d4b1202066e3d9b6ee6b3129087b"
}