PEFT
Safetensors
hugooguh commited on
Commit
bb90cf3
·
1 Parent(s): 1beeac6
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- base_model: meta-llama/Llama-3.2-1B
3
  library_name: peft
4
  ---
5
 
 
1
  ---
2
+ base_model: models-pretrained/llama/model
3
  library_name: peft
4
  ---
5
 
adapter_config.json CHANGED
@@ -27,6 +27,8 @@
27
  "revision": null,
28
  "target_modules": [
29
  "q_proj",
 
 
30
  "v_proj"
31
  ],
32
  "task_type": "SEQ_CLS",
 
27
  "revision": null,
28
  "target_modules": [
29
  "q_proj",
30
+ "k_proj",
31
+ "o_proj",
32
  "v_proj"
33
  ],
34
  "task_type": "SEQ_CLS",
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:75111924ea098a7e200c4543c8b5ccfaa0133c23433bccd627c1a7dadb769141
3
- size 1728768
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3ce34476b758e75d75a854560fecfa08ce69e937a8f4721ac9e376e03a8ab37
3
+ size 3441096
optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:602f69696573a44e412bd2b319f61d7802f812af0e268f5fb520d09bd879f131
3
- size 35042
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:daa610be6d42e1f69b37a753bd4047c1b8a0ac9de59b59f263249b2b45112e59
3
+ size 35170
rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:36be32085207ee8661d6a02bc289ea83fdc0647b4a501e6a07affe84fc98069c
3
  size 14244
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eebad48a7ae5057b385065938f885b4e877a13bd73d22167b119c325f133c3ba
3
  size 14244
scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4005bb983f6eabace62ffa591ad6ca6349df92ee7bf1d921dfe5363812305456
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4986462be4218f55b55b2398ae4eefa03343f0222855cddecd9b3e8e789340e
3
  size 1064
trainer_state.json CHANGED
@@ -1,46 +1,46 @@
1
  {
2
- "best_metric": 0.004436289891600609,
3
- "best_model_checkpoint": "meta-llama/Llama-3.2-1B-lora-text-classification/checkpoint-2529",
4
- "epoch": 1.0,
5
  "eval_steps": 500,
6
- "global_step": 2529,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.19770660340055357,
13
- "grad_norm": 0.010457451455295086,
14
- "learning_rate": 8.022933965994464e-05,
15
  "loss": 0.0094,
16
  "step": 500
17
  },
18
  {
19
  "epoch": 0.39541320680110714,
20
- "grad_norm": 0.19572089612483978,
21
- "learning_rate": 6.045867931988929e-05,
22
- "loss": 0.006,
23
  "step": 1000
24
  },
25
  {
26
  "epoch": 0.5931198102016607,
27
- "grad_norm": 0.0019002881599590182,
28
- "learning_rate": 4.0688018979833934e-05,
29
- "loss": 0.0103,
30
  "step": 1500
31
  },
32
  {
33
  "epoch": 0.7908264136022143,
34
- "grad_norm": 0.008688927628099918,
35
- "learning_rate": 2.091735863977857e-05,
36
- "loss": 0.0052,
37
  "step": 2000
38
  },
39
  {
40
  "epoch": 0.9885330170027679,
41
- "grad_norm": 0.011189923621714115,
42
- "learning_rate": 1.146698299723211e-06,
43
- "loss": 0.0076,
44
  "step": 2500
45
  },
46
  {
@@ -48,17 +48,63 @@
48
  "eval_accuracy": {
49
  "accuracy": 0.9993080781871648
50
  },
51
- "eval_loss": 0.004436289891600609,
52
- "eval_runtime": 13639.4658,
53
- "eval_samples_per_second": 1.272,
54
- "eval_steps_per_second": 0.318,
55
  "step": 2529
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  }
57
  ],
58
  "logging_steps": 500,
59
- "max_steps": 2529,
60
  "num_input_tokens_seen": 0,
61
- "num_train_epochs": 1,
62
  "save_steps": 500,
63
  "stateful_callbacks": {
64
  "TrainerControl": {
@@ -72,7 +118,7 @@
72
  "attributes": {}
73
  }
74
  },
75
- "total_flos": 1.18184858320896e+18,
76
  "train_batch_size": 4,
77
  "trial_name": null,
78
  "trial_params": null
 
1
  {
2
+ "best_metric": 0.0036397832445800304,
3
+ "best_model_checkpoint": "meta-llama/Llama-3.2-1B-lora-text-classification/checkpoint-5058",
4
+ "epoch": 2.0,
5
  "eval_steps": 500,
6
+ "global_step": 5058,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "epoch": 0.19770660340055357,
13
+ "grad_norm": 0.012822521850466728,
14
+ "learning_rate": 9.011466982997232e-05,
15
  "loss": 0.0094,
16
  "step": 500
17
  },
18
  {
19
  "epoch": 0.39541320680110714,
20
+ "grad_norm": 0.18435056507587433,
21
+ "learning_rate": 8.022933965994464e-05,
22
+ "loss": 0.0061,
23
  "step": 1000
24
  },
25
  {
26
  "epoch": 0.5931198102016607,
27
+ "grad_norm": 0.000889307411853224,
28
+ "learning_rate": 7.034400948991697e-05,
29
+ "loss": 0.0102,
30
  "step": 1500
31
  },
32
  {
33
  "epoch": 0.7908264136022143,
34
+ "grad_norm": 0.007775792386382818,
35
+ "learning_rate": 6.045867931988929e-05,
36
+ "loss": 0.0051,
37
  "step": 2000
38
  },
39
  {
40
  "epoch": 0.9885330170027679,
41
+ "grad_norm": 0.0005994698731228709,
42
+ "learning_rate": 5.05733491498616e-05,
43
+ "loss": 0.007,
44
  "step": 2500
45
  },
46
  {
 
48
  "eval_accuracy": {
49
  "accuracy": 0.9993080781871648
50
  },
51
+ "eval_loss": 0.005377906374633312,
52
+ "eval_runtime": 13938.8278,
53
+ "eval_samples_per_second": 1.244,
54
+ "eval_steps_per_second": 0.311,
55
  "step": 2529
56
+ },
57
+ {
58
+ "epoch": 1.1862396204033214,
59
+ "grad_norm": 0.011812306940555573,
60
+ "learning_rate": 4.0688018979833934e-05,
61
+ "loss": 0.0045,
62
+ "step": 3000
63
+ },
64
+ {
65
+ "epoch": 1.383946223803875,
66
+ "grad_norm": 0.08777114003896713,
67
+ "learning_rate": 3.080268880980625e-05,
68
+ "loss": 0.008,
69
+ "step": 3500
70
+ },
71
+ {
72
+ "epoch": 1.5816528272044286,
73
+ "grad_norm": 0.005097639746963978,
74
+ "learning_rate": 2.091735863977857e-05,
75
+ "loss": 0.0067,
76
+ "step": 4000
77
+ },
78
+ {
79
+ "epoch": 1.7793594306049823,
80
+ "grad_norm": 0.0066584269516170025,
81
+ "learning_rate": 1.1032028469750892e-05,
82
+ "loss": 0.0031,
83
+ "step": 4500
84
+ },
85
+ {
86
+ "epoch": 1.977066034005536,
87
+ "grad_norm": 0.0033356526400893927,
88
+ "learning_rate": 1.146698299723211e-06,
89
+ "loss": 0.0037,
90
+ "step": 5000
91
+ },
92
+ {
93
+ "epoch": 2.0,
94
+ "eval_accuracy": {
95
+ "accuracy": 0.9993080781871648
96
+ },
97
+ "eval_loss": 0.0036397832445800304,
98
+ "eval_runtime": 13938.1843,
99
+ "eval_samples_per_second": 1.244,
100
+ "eval_steps_per_second": 0.311,
101
+ "step": 5058
102
  }
103
  ],
104
  "logging_steps": 500,
105
+ "max_steps": 5058,
106
  "num_input_tokens_seen": 0,
107
+ "num_train_epochs": 2,
108
  "save_steps": 500,
109
  "stateful_callbacks": {
110
  "TrainerControl": {
 
118
  "attributes": {}
119
  }
120
  },
121
+ "total_flos": 2.36473138741248e+18,
122
  "train_batch_size": 4,
123
  "trial_name": null,
124
  "trial_params": null
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f617fa7c469e7120a3e44b9422f99aa8eb377155aa53d4fa5fa269d5ceb16995
3
  size 5368
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51ebc3457e58a760775864fda9ed33f4710660bfcb7b3e45d0864b3e2c5579ee
3
  size 5368