Hugging Face's logo

legesher
/
language-decoded-lora

Text Generation
Transformers
Safetensors
lora
aya
tiny-aya
multilingual
code
legesher
tiny-aya-expedition
language-decoded
unsloth
arxiv:2603.11510
arxiv:2211.15533
arxiv:2510.09591
arxiv:1809.05053
arxiv:2308.16884
arxiv:2106.06937
arxiv:2210.03057
Model card Files
xet
 Community
22

phase-2-the-stack-v1-condition-3-zh-5k-seed123

#16
by rafaym - opened
base: refs/heads/main
←
from: refs/pr/16
Discussion Files changed
+278
-0
Files changed (3) hide show
  1. tiny-aya-base/phase-2-the-stack-v1-condition-3-zh-5k-seed123/adapter_config.json +54 -0
  2. tiny-aya-base/phase-2-the-stack-v1-condition-3-zh-5k-seed123/adapter_model.safetensors +3 -0
  3. tiny-aya-base/phase-2-the-stack-v1-condition-3-zh-5k-seed123/training_metrics.json +221 -0
tiny-aya-base/phase-2-the-stack-v1-condition-3-zh-5k-seed123/adapter_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": {
6
+ "base_model_class": "Cohere2ForCausalLM",
7
+ "parent_library": "unsloth_compiled_module_cohere2",
8
+ "unsloth_fixed": true
9
+ },
10
+ "base_model_name_or_path": "CohereLabs/tiny-aya-base",
11
+ "bias": "none",
12
+ "corda_config": null,
13
+ "ensure_weight_tying": false,
14
+ "eva_config": null,
15
+ "exclude_modules": null,
16
+ "fan_in_fan_out": false,
17
+ "inference_mode": true,
18
+ "init_lora_weights": true,
19
+ "layer_replication": null,
20
+ "layers_pattern": null,
21
+ "layers_to_transform": null,
22
+ "loftq_config": {},
23
+ "lora_alpha": 32,
24
+ "lora_bias": false,
25
+ "lora_dropout": 0.0,
26
+ "lora_ga_config": null,
27
+ "megatron_config": null,
28
+ "megatron_core": "megatron.core",
29
+ "modules_to_save": null,
30
+ "monteclora_config": null,
31
+ "peft_type": "LORA",
32
+ "peft_version": "0.20.0",
33
+ "qalora_group_size": 16,
34
+ "r": 16,
35
+ "rank_pattern": {},
36
+ "revision": null,
37
+ "target_modules": [
38
+ "v_proj",
39
+ "down_proj",
40
+ "q_proj",
41
+ "gate_proj",
42
+ "up_proj",
43
+ "k_proj",
44
+ "o_proj"
45
+ ],
46
+ "target_parameters": null,
47
+ "task_type": "CAUSAL_LM",
48
+ "trainable_token_indices": null,
49
+ "use_bdlora": null,
50
+ "use_dora": false,
51
+ "use_qalora": false,
52
+ "use_rslora": false,
53
+ "velora_config": null
54
+ }
tiny-aya-base/phase-2-the-stack-v1-condition-3-zh-5k-seed123/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e0088e80bf1f80a148b955fc79635b5ebb5db29045d732722b58ce930d70823
3
+ size 120981200
tiny-aya-base/phase-2-the-stack-v1-condition-3-zh-5k-seed123/training_metrics.json ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_config_name": "tiny-aya-base",
3
+ "condition_name": "phase-2-the-stack-v1-condition-3-zh-5k",
4
+ "seed": 123,
5
+ "output_name": "phase-2-the-stack-v1-condition-3-zh-5k-seed123",
6
+ "train_result": {
7
+ "train_runtime": 3624.0282,
8
+ "train_samples_per_second": 1.242,
9
+ "train_steps_per_second": 0.078,
10
+ "total_flos": 7.809827609142886e+16,
11
+ "train_loss": 1.1588138908359176,
12
+ "epoch": 1.0
13
+ },
14
+ "log_history": [
15
+ {
16
+ "loss": 1.4367,
17
+ "grad_norm": 0.11920547485351562,
18
+ "learning_rate": 0.00012,
19
+ "epoch": 0.03546099290780142,
20
+ "step": 10
21
+ },
22
+ {
23
+ "loss": 1.2996,
24
+ "grad_norm": 0.16951777040958405,
25
+ "learning_rate": 0.00019988926445681492,
26
+ "epoch": 0.07092198581560284,
27
+ "step": 20
28
+ },
29
+ {
30
+ "loss": 1.255,
31
+ "grad_norm": 0.13241389393806458,
32
+ "learning_rate": 0.0001986463043917528,
33
+ "epoch": 0.10638297872340426,
34
+ "step": 30
35
+ },
36
+ {
37
+ "loss": 1.151,
38
+ "grad_norm": 0.1258794665336609,
39
+ "learning_rate": 0.00019603921063437793,
40
+ "epoch": 0.14184397163120568,
41
+ "step": 40
42
+ },
43
+ {
44
+ "loss": 1.1592,
45
+ "grad_norm": 0.1519426554441452,
46
+ "learning_rate": 0.0001921040354671897,
47
+ "epoch": 0.1773049645390071,
48
+ "step": 50
49
+ },
50
+ {
51
+ "loss": 1.2344,
52
+ "grad_norm": 0.18602345883846283,
53
+ "learning_rate": 0.00018689519659051467,
54
+ "epoch": 0.2127659574468085,
55
+ "step": 60
56
+ },
57
+ {
58
+ "loss": 1.2181,
59
+ "grad_norm": 0.15296107530593872,
60
+ "learning_rate": 0.00018048472460553257,
61
+ "epoch": 0.24822695035460993,
62
+ "step": 70
63
+ },
64
+ {
65
+ "loss": 1.1438,
66
+ "grad_norm": 0.15809592604637146,
67
+ "learning_rate": 0.00017296126693671884,
68
+ "epoch": 0.28368794326241137,
69
+ "step": 80
70
+ },
71
+ {
72
+ "loss": 1.124,
73
+ "grad_norm": 0.1690121740102768,
74
+ "learning_rate": 0.00016442886196799464,
75
+ "epoch": 0.3191489361702128,
76
+ "step": 90
77
+ },
78
+ {
79
+ "loss": 1.1361,
80
+ "grad_norm": 0.15152955055236816,
81
+ "learning_rate": 0.00015500550034448413,
82
+ "epoch": 0.3546099290780142,
83
+ "step": 100
84
+ },
85
+ {
86
+ "loss": 1.1709,
87
+ "grad_norm": 0.15868514776229858,
88
+ "learning_rate": 0.00014482149333496454,
89
+ "epoch": 0.3900709219858156,
90
+ "step": 110
91
+ },
92
+ {
93
+ "loss": 1.1466,
94
+ "grad_norm": 0.1790330410003662,
95
+ "learning_rate": 0.0001340176708181637,
96
+ "epoch": 0.425531914893617,
97
+ "step": 120
98
+ },
99
+ {
100
+ "loss": 1.2095,
101
+ "grad_norm": 0.17775388062000275,
102
+ "learning_rate": 0.00012274343381211066,
103
+ "epoch": 0.46099290780141844,
104
+ "step": 130
105
+ },
106
+ {
107
+ "loss": 1.2383,
108
+ "grad_norm": 0.17202383279800415,
109
+ "learning_rate": 0.00011115468847720245,
110
+ "epoch": 0.49645390070921985,
111
+ "step": 140
112
+ },
113
+ {
114
+ "loss": 1.069,
115
+ "grad_norm": 0.13828101754188538,
116
+ "learning_rate": 9.941169016269379e-05,
117
+ "epoch": 0.5319148936170213,
118
+ "step": 150
119
+ },
120
+ {
121
+ "loss": 1.2857,
122
+ "grad_norm": 0.15986786782741547,
123
+ "learning_rate": 8.767682731028415e-05,
124
+ "epoch": 0.5673758865248227,
125
+ "step": 160
126
+ },
127
+ {
128
+ "loss": 1.0463,
129
+ "grad_norm": 0.1267540156841278,
130
+ "learning_rate": 7.611237586016557e-05,
131
+ "epoch": 0.6028368794326241,
132
+ "step": 170
133
+ },
134
+ {
135
+ "loss": 1.1668,
136
+ "grad_norm": 0.1478545218706131,
137
+ "learning_rate": 6.487825521280109e-05,
138
+ "epoch": 0.6382978723404256,
139
+ "step": 180
140
+ },
141
+ {
142
+ "loss": 1.0712,
143
+ "grad_norm": 0.15280702710151672,
144
+ "learning_rate": 5.4129816778190936e-05,
145
+ "epoch": 0.6737588652482269,
146
+ "step": 190
147
+ },
148
+ {
149
+ "loss": 1.1129,
150
+ "grad_norm": 0.1279517263174057,
151
+ "learning_rate": 4.401569569374668e-05,
152
+ "epoch": 0.7092198581560284,
153
+ "step": 200
154
+ },
155
+ {
156
+ "loss": 1.157,
157
+ "grad_norm": 0.2598387598991394,
158
+ "learning_rate": 3.467575541836305e-05,
159
+ "epoch": 0.7446808510638298,
160
+ "step": 210
161
+ },
162
+ {
163
+ "loss": 1.1301,
164
+ "grad_norm": 0.14336277544498444,
165
+ "learning_rate": 2.6239153625937784e-05,
166
+ "epoch": 0.7801418439716312,
167
+ "step": 220
168
+ },
169
+ {
170
+ "loss": 1.051,
171
+ "grad_norm": 0.15225663781166077,
172
+ "learning_rate": 1.882255614419376e-05,
173
+ "epoch": 0.8156028368794326,
174
+ "step": 230
175
+ },
176
+ {
177
+ "loss": 1.1094,
178
+ "grad_norm": 0.17789503931999207,
179
+ "learning_rate": 1.2528523637410838e-05,
180
+ "epoch": 0.851063829787234,
181
+ "step": 240
182
+ },
183
+ {
184
+ "loss": 1.0212,
185
+ "grad_norm": 0.17333965003490448,
186
+ "learning_rate": 7.4440933428779e-06,
187
+ "epoch": 0.8865248226950354,
188
+ "step": 250
189
+ },
190
+ {
191
+ "loss": 1.0941,
192
+ "grad_norm": 0.1316404789686203,
193
+ "learning_rate": 3.6395754735699894e-06,
194
+ "epoch": 0.9219858156028369,
195
+ "step": 260
196
+ },
197
+ {
198
+ "loss": 1.1369,
199
+ "grad_norm": 0.19937266409397125,
200
+ "learning_rate": 1.1675809310361497e-06,
201
+ "epoch": 0.9574468085106383,
202
+ "step": 270
203
+ },
204
+ {
205
+ "loss": 1.1022,
206
+ "grad_norm": 0.15226872265338898,
207
+ "learning_rate": 6.229377380218005e-08,
208
+ "epoch": 0.9929078014184397,
209
+ "step": 280
210
+ },
211
+ {
212
+ "train_runtime": 3624.0282,
213
+ "train_samples_per_second": 1.242,
214
+ "train_steps_per_second": 0.078,
215
+ "total_flos": 7.809827609142886e+16,
216
+ "train_loss": 1.1588138908359176,
217
+ "epoch": 1.0,
218
+ "step": 282
219
+ }
220
+ ]
221
+ }