chakkakrishna commited on
Commit
9e48483
·
verified ·
1 Parent(s): 1727dcc

Upload folder using huggingface_hub

Browse files
Files changed (43) hide show
  1. README.md +20 -0
  2. adapter_config.json +26 -0
  3. adapter_model.bin +3 -0
  4. all_results.json +8 -0
  5. checkpoint-1000/README.md +20 -0
  6. checkpoint-1000/adapter_config.json +26 -0
  7. checkpoint-1000/adapter_model.bin +3 -0
  8. checkpoint-1000/optimizer.pt +3 -0
  9. checkpoint-1000/rng_state.pth +3 -0
  10. checkpoint-1000/scheduler.pt +3 -0
  11. checkpoint-1000/trainer_state.json +76 -0
  12. checkpoint-1000/training_args.bin +3 -0
  13. checkpoint-1500/README.md +20 -0
  14. checkpoint-1500/adapter_config.json +26 -0
  15. checkpoint-1500/adapter_model.bin +3 -0
  16. checkpoint-1500/optimizer.pt +3 -0
  17. checkpoint-1500/rng_state.pth +3 -0
  18. checkpoint-1500/scheduler.pt +3 -0
  19. checkpoint-1500/trainer_state.json +106 -0
  20. checkpoint-1500/training_args.bin +3 -0
  21. checkpoint-2000/README.md +20 -0
  22. checkpoint-2000/adapter_config.json +26 -0
  23. checkpoint-2000/adapter_model.bin +3 -0
  24. checkpoint-2000/optimizer.pt +3 -0
  25. checkpoint-2000/rng_state.pth +3 -0
  26. checkpoint-2000/scheduler.pt +3 -0
  27. checkpoint-2000/trainer_state.json +136 -0
  28. checkpoint-2000/training_args.bin +3 -0
  29. checkpoint-500/README.md +20 -0
  30. checkpoint-500/adapter_config.json +26 -0
  31. checkpoint-500/adapter_model.bin +3 -0
  32. checkpoint-500/optimizer.pt +3 -0
  33. checkpoint-500/rng_state.pth +3 -0
  34. checkpoint-500/scheduler.pt +3 -0
  35. checkpoint-500/trainer_state.json +46 -0
  36. checkpoint-500/training_args.bin +3 -0
  37. runs/Apr23_07-25-06_d6ff39b69086/events.out.tfevents.1713857106.d6ff39b69086.384.0 +3 -0
  38. special_tokens_map.json +24 -0
  39. tokenizer.json +0 -0
  40. tokenizer.model +3 -0
  41. tokenizer_config.json +40 -0
  42. train_results.json +8 -0
  43. trainer_state.json +157 -0
README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.4.0
adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 64,
11
+ "lora_dropout": 0.1,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 16,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "gate_proj",
18
+ "o_proj",
19
+ "down_proj",
20
+ "k_proj",
21
+ "q_proj",
22
+ "up_proj",
23
+ "v_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3270a404e8bec84f5118cb9c14ebde70f27cb3a6b97c8f16ec5440c613c0b587
3
+ size 50573530
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 1.7946134696558592e+16,
4
+ "train_loss": 0.6671193393433366,
5
+ "train_runtime": 6110.3906,
6
+ "train_samples_per_second": 5.981,
7
+ "train_steps_per_second": 0.374
8
+ }
checkpoint-1000/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.4.0
checkpoint-1000/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 64,
11
+ "lora_dropout": 0.1,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 16,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "gate_proj",
18
+ "o_proj",
19
+ "down_proj",
20
+ "k_proj",
21
+ "q_proj",
22
+ "up_proj",
23
+ "v_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
checkpoint-1000/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f87033379cfce8a91440d1d6654a78bd60cce06d73a4cab0577299b74e8709d
3
+ size 50573530
checkpoint-1000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cce5a5fe5453f1db71ad903bb50a7cbfe0f766d20ac2ff330261ce6a3e1cc96c
3
+ size 25635172
checkpoint-1000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:874db3c036c1fd60ab8c4bcb934b8818cc8d3fb5d269287db23e862c7f104fc9
3
+ size 14244
checkpoint-1000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41a8ca0acc05d184f3c2f29e3e82a85a9ab6983c30e0ad22ad4c95d6d42f273e
3
+ size 1064
checkpoint-1000/trainer_state.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.43787629994526545,
5
+ "global_step": 1000,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "learning_rate": 2e-05,
13
+ "loss": 1.5875,
14
+ "step": 100
15
+ },
16
+ {
17
+ "epoch": 0.09,
18
+ "learning_rate": 4e-05,
19
+ "loss": 0.9455,
20
+ "step": 200
21
+ },
22
+ {
23
+ "epoch": 0.13,
24
+ "learning_rate": 3.8080614203454896e-05,
25
+ "loss": 0.9132,
26
+ "step": 300
27
+ },
28
+ {
29
+ "epoch": 0.18,
30
+ "learning_rate": 3.6161228406909795e-05,
31
+ "loss": 0.8896,
32
+ "step": 400
33
+ },
34
+ {
35
+ "epoch": 0.22,
36
+ "learning_rate": 3.424184261036469e-05,
37
+ "loss": 0.8479,
38
+ "step": 500
39
+ },
40
+ {
41
+ "epoch": 0.26,
42
+ "learning_rate": 3.232245681381958e-05,
43
+ "loss": 0.8214,
44
+ "step": 600
45
+ },
46
+ {
47
+ "epoch": 0.31,
48
+ "learning_rate": 3.0403071017274476e-05,
49
+ "loss": 0.7844,
50
+ "step": 700
51
+ },
52
+ {
53
+ "epoch": 0.35,
54
+ "learning_rate": 2.8483685220729368e-05,
55
+ "loss": 0.7575,
56
+ "step": 800
57
+ },
58
+ {
59
+ "epoch": 0.39,
60
+ "learning_rate": 2.6564299424184264e-05,
61
+ "loss": 0.7334,
62
+ "step": 900
63
+ },
64
+ {
65
+ "epoch": 0.44,
66
+ "learning_rate": 2.4644913627639156e-05,
67
+ "loss": 0.6934,
68
+ "step": 1000
69
+ }
70
+ ],
71
+ "max_steps": 2284,
72
+ "num_train_epochs": 2,
73
+ "total_flos": 7867483983839232.0,
74
+ "trial_name": null,
75
+ "trial_params": null
76
+ }
checkpoint-1000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d103bb9dc68bc805d39193913fa2ce0e74adedf06cc7010a1886731a77b04945
3
+ size 4472
checkpoint-1500/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.4.0
checkpoint-1500/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 64,
11
+ "lora_dropout": 0.1,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 16,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "gate_proj",
18
+ "o_proj",
19
+ "down_proj",
20
+ "k_proj",
21
+ "q_proj",
22
+ "up_proj",
23
+ "v_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
checkpoint-1500/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29bb11c303698eea8c38a21501d5c13cfdb86ca1b148c32ebfbb95825485c2b8
3
+ size 50573530
checkpoint-1500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74944ca3accb88d26765376da076e6d3911c427f11b687a222e38ebe543708d9
3
+ size 25635172
checkpoint-1500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c592079f8ad13df2da74684c66f45a0e8a760e24577c403c9f04bd71761c13d3
3
+ size 14244
checkpoint-1500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89ab4b2c75cce66eab2c867fd029bf86edaa30884e3215407b9a877e8b73335a
3
+ size 1064
checkpoint-1500/trainer_state.json ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.6568144499178982,
5
+ "global_step": 1500,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "learning_rate": 2e-05,
13
+ "loss": 1.5875,
14
+ "step": 100
15
+ },
16
+ {
17
+ "epoch": 0.09,
18
+ "learning_rate": 4e-05,
19
+ "loss": 0.9455,
20
+ "step": 200
21
+ },
22
+ {
23
+ "epoch": 0.13,
24
+ "learning_rate": 3.8080614203454896e-05,
25
+ "loss": 0.9132,
26
+ "step": 300
27
+ },
28
+ {
29
+ "epoch": 0.18,
30
+ "learning_rate": 3.6161228406909795e-05,
31
+ "loss": 0.8896,
32
+ "step": 400
33
+ },
34
+ {
35
+ "epoch": 0.22,
36
+ "learning_rate": 3.424184261036469e-05,
37
+ "loss": 0.8479,
38
+ "step": 500
39
+ },
40
+ {
41
+ "epoch": 0.26,
42
+ "learning_rate": 3.232245681381958e-05,
43
+ "loss": 0.8214,
44
+ "step": 600
45
+ },
46
+ {
47
+ "epoch": 0.31,
48
+ "learning_rate": 3.0403071017274476e-05,
49
+ "loss": 0.7844,
50
+ "step": 700
51
+ },
52
+ {
53
+ "epoch": 0.35,
54
+ "learning_rate": 2.8483685220729368e-05,
55
+ "loss": 0.7575,
56
+ "step": 800
57
+ },
58
+ {
59
+ "epoch": 0.39,
60
+ "learning_rate": 2.6564299424184264e-05,
61
+ "loss": 0.7334,
62
+ "step": 900
63
+ },
64
+ {
65
+ "epoch": 0.44,
66
+ "learning_rate": 2.4644913627639156e-05,
67
+ "loss": 0.6934,
68
+ "step": 1000
69
+ },
70
+ {
71
+ "epoch": 0.48,
72
+ "learning_rate": 2.2725527831094052e-05,
73
+ "loss": 0.6665,
74
+ "step": 1100
75
+ },
76
+ {
77
+ "epoch": 0.53,
78
+ "learning_rate": 2.0806142034548945e-05,
79
+ "loss": 0.6466,
80
+ "step": 1200
81
+ },
82
+ {
83
+ "epoch": 0.57,
84
+ "learning_rate": 1.888675623800384e-05,
85
+ "loss": 0.6095,
86
+ "step": 1300
87
+ },
88
+ {
89
+ "epoch": 0.61,
90
+ "learning_rate": 1.6967370441458736e-05,
91
+ "loss": 0.5723,
92
+ "step": 1400
93
+ },
94
+ {
95
+ "epoch": 0.66,
96
+ "learning_rate": 1.5047984644913629e-05,
97
+ "loss": 0.5313,
98
+ "step": 1500
99
+ }
100
+ ],
101
+ "max_steps": 2284,
102
+ "num_train_epochs": 2,
103
+ "total_flos": 1.180688434569216e+16,
104
+ "trial_name": null,
105
+ "trial_params": null
106
+ }
checkpoint-1500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d103bb9dc68bc805d39193913fa2ce0e74adedf06cc7010a1886731a77b04945
3
+ size 4472
checkpoint-2000/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.4.0
checkpoint-2000/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 64,
11
+ "lora_dropout": 0.1,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 16,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "gate_proj",
18
+ "o_proj",
19
+ "down_proj",
20
+ "k_proj",
21
+ "q_proj",
22
+ "up_proj",
23
+ "v_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
checkpoint-2000/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ccc83436ea1a6808dba3169b8df28e2ee5dee0f1e2d9d9e417797452f6959c0
3
+ size 50573530
checkpoint-2000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6baf7dc63d8bc8c2c0bdcee3a640fc4ec7631da9be72127776f83c9f1ec209ad
3
+ size 25635172
checkpoint-2000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebd12198d5f6092c1fb1768eacc06dd69b1944ac7971bd7f3004ab2a6a9928e0
3
+ size 14244
checkpoint-2000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6694039294b87b655d3c24d5f366c71a02846ba5055d940a8a47014e08ff8a86
3
+ size 1064
checkpoint-2000/trainer_state.json ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.8757525998905309,
5
+ "global_step": 2000,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "learning_rate": 2e-05,
13
+ "loss": 1.5875,
14
+ "step": 100
15
+ },
16
+ {
17
+ "epoch": 0.09,
18
+ "learning_rate": 4e-05,
19
+ "loss": 0.9455,
20
+ "step": 200
21
+ },
22
+ {
23
+ "epoch": 0.13,
24
+ "learning_rate": 3.8080614203454896e-05,
25
+ "loss": 0.9132,
26
+ "step": 300
27
+ },
28
+ {
29
+ "epoch": 0.18,
30
+ "learning_rate": 3.6161228406909795e-05,
31
+ "loss": 0.8896,
32
+ "step": 400
33
+ },
34
+ {
35
+ "epoch": 0.22,
36
+ "learning_rate": 3.424184261036469e-05,
37
+ "loss": 0.8479,
38
+ "step": 500
39
+ },
40
+ {
41
+ "epoch": 0.26,
42
+ "learning_rate": 3.232245681381958e-05,
43
+ "loss": 0.8214,
44
+ "step": 600
45
+ },
46
+ {
47
+ "epoch": 0.31,
48
+ "learning_rate": 3.0403071017274476e-05,
49
+ "loss": 0.7844,
50
+ "step": 700
51
+ },
52
+ {
53
+ "epoch": 0.35,
54
+ "learning_rate": 2.8483685220729368e-05,
55
+ "loss": 0.7575,
56
+ "step": 800
57
+ },
58
+ {
59
+ "epoch": 0.39,
60
+ "learning_rate": 2.6564299424184264e-05,
61
+ "loss": 0.7334,
62
+ "step": 900
63
+ },
64
+ {
65
+ "epoch": 0.44,
66
+ "learning_rate": 2.4644913627639156e-05,
67
+ "loss": 0.6934,
68
+ "step": 1000
69
+ },
70
+ {
71
+ "epoch": 0.48,
72
+ "learning_rate": 2.2725527831094052e-05,
73
+ "loss": 0.6665,
74
+ "step": 1100
75
+ },
76
+ {
77
+ "epoch": 0.53,
78
+ "learning_rate": 2.0806142034548945e-05,
79
+ "loss": 0.6466,
80
+ "step": 1200
81
+ },
82
+ {
83
+ "epoch": 0.57,
84
+ "learning_rate": 1.888675623800384e-05,
85
+ "loss": 0.6095,
86
+ "step": 1300
87
+ },
88
+ {
89
+ "epoch": 0.61,
90
+ "learning_rate": 1.6967370441458736e-05,
91
+ "loss": 0.5723,
92
+ "step": 1400
93
+ },
94
+ {
95
+ "epoch": 0.66,
96
+ "learning_rate": 1.5047984644913629e-05,
97
+ "loss": 0.5313,
98
+ "step": 1500
99
+ },
100
+ {
101
+ "epoch": 0.7,
102
+ "learning_rate": 1.3128598848368523e-05,
103
+ "loss": 0.514,
104
+ "step": 1600
105
+ },
106
+ {
107
+ "epoch": 0.74,
108
+ "learning_rate": 1.1209213051823417e-05,
109
+ "loss": 0.4911,
110
+ "step": 1700
111
+ },
112
+ {
113
+ "epoch": 0.79,
114
+ "learning_rate": 9.289827255278311e-06,
115
+ "loss": 0.4456,
116
+ "step": 1800
117
+ },
118
+ {
119
+ "epoch": 0.83,
120
+ "learning_rate": 7.370441458733206e-06,
121
+ "loss": 0.423,
122
+ "step": 1900
123
+ },
124
+ {
125
+ "epoch": 0.88,
126
+ "learning_rate": 5.4510556621881e-06,
127
+ "loss": 0.3877,
128
+ "step": 2000
129
+ }
130
+ ],
131
+ "max_steps": 2284,
132
+ "num_train_epochs": 2,
133
+ "total_flos": 1.572952566915072e+16,
134
+ "trial_name": null,
135
+ "trial_params": null
136
+ }
checkpoint-2000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d103bb9dc68bc805d39193913fa2ce0e74adedf06cc7010a1886731a77b04945
3
+ size 4472
checkpoint-500/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.4.0
checkpoint-500/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 64,
11
+ "lora_dropout": 0.1,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 16,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "gate_proj",
18
+ "o_proj",
19
+ "down_proj",
20
+ "k_proj",
21
+ "q_proj",
22
+ "up_proj",
23
+ "v_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
checkpoint-500/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7275cedf197cd842ca2abe0c3de41f39424d762442f077d9d68746c5d4cf891f
3
+ size 50573530
checkpoint-500/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:044a4cc3a4b31e7c5638013cf29eadda904294d0e9556c01ed264cd57161b63e
3
+ size 25635172
checkpoint-500/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f7e5971ccda88471d0e0ee14f528a1f787491dec416c57c2242871d2b757e5e
3
+ size 14244
checkpoint-500/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c687ffd376818df6e7a49363064165446e25056ec982bb4ed3a722f5122b4c4
3
+ size 1064
checkpoint-500/trainer_state.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.21893814997263272,
5
+ "global_step": 500,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "learning_rate": 2e-05,
13
+ "loss": 1.5875,
14
+ "step": 100
15
+ },
16
+ {
17
+ "epoch": 0.09,
18
+ "learning_rate": 4e-05,
19
+ "loss": 0.9455,
20
+ "step": 200
21
+ },
22
+ {
23
+ "epoch": 0.13,
24
+ "learning_rate": 3.8080614203454896e-05,
25
+ "loss": 0.9132,
26
+ "step": 300
27
+ },
28
+ {
29
+ "epoch": 0.18,
30
+ "learning_rate": 3.6161228406909795e-05,
31
+ "loss": 0.8896,
32
+ "step": 400
33
+ },
34
+ {
35
+ "epoch": 0.22,
36
+ "learning_rate": 3.424184261036469e-05,
37
+ "loss": 0.8479,
38
+ "step": 500
39
+ }
40
+ ],
41
+ "max_steps": 2284,
42
+ "num_train_epochs": 2,
43
+ "total_flos": 3928515764797440.0,
44
+ "trial_name": null,
45
+ "trial_params": null
46
+ }
checkpoint-500/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d103bb9dc68bc805d39193913fa2ce0e74adedf06cc7010a1886731a77b04945
3
+ size 4472
runs/Apr23_07-25-06_d6ff39b69086/events.out.tfevents.1713857106.d6ff39b69086.384.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:128795998dd9d52f0c942e3721c0b284474fe6d66638b13600f922ef014e64f4
3
+ size 8308
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ }
27
+ },
28
+ "bos_token": "<s>",
29
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
30
+ "clean_up_tokenization_spaces": false,
31
+ "eos_token": "</s>",
32
+ "legacy": false,
33
+ "model_max_length": 2048,
34
+ "pad_token": "</s>",
35
+ "padding_side": "right",
36
+ "sp_model_kwargs": {},
37
+ "tokenizer_class": "LlamaTokenizer",
38
+ "unk_token": "<unk>",
39
+ "use_default_system_prompt": false
40
+ }
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 1.7946134696558592e+16,
4
+ "train_loss": 0.6671193393433366,
5
+ "train_runtime": 6110.3906,
6
+ "train_samples_per_second": 5.981,
7
+ "train_steps_per_second": 0.374
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0001094690749863,
5
+ "global_step": 2284,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "learning_rate": 2e-05,
13
+ "loss": 1.5875,
14
+ "step": 100
15
+ },
16
+ {
17
+ "epoch": 0.09,
18
+ "learning_rate": 4e-05,
19
+ "loss": 0.9455,
20
+ "step": 200
21
+ },
22
+ {
23
+ "epoch": 0.13,
24
+ "learning_rate": 3.8080614203454896e-05,
25
+ "loss": 0.9132,
26
+ "step": 300
27
+ },
28
+ {
29
+ "epoch": 0.18,
30
+ "learning_rate": 3.6161228406909795e-05,
31
+ "loss": 0.8896,
32
+ "step": 400
33
+ },
34
+ {
35
+ "epoch": 0.22,
36
+ "learning_rate": 3.424184261036469e-05,
37
+ "loss": 0.8479,
38
+ "step": 500
39
+ },
40
+ {
41
+ "epoch": 0.26,
42
+ "learning_rate": 3.232245681381958e-05,
43
+ "loss": 0.8214,
44
+ "step": 600
45
+ },
46
+ {
47
+ "epoch": 0.31,
48
+ "learning_rate": 3.0403071017274476e-05,
49
+ "loss": 0.7844,
50
+ "step": 700
51
+ },
52
+ {
53
+ "epoch": 0.35,
54
+ "learning_rate": 2.8483685220729368e-05,
55
+ "loss": 0.7575,
56
+ "step": 800
57
+ },
58
+ {
59
+ "epoch": 0.39,
60
+ "learning_rate": 2.6564299424184264e-05,
61
+ "loss": 0.7334,
62
+ "step": 900
63
+ },
64
+ {
65
+ "epoch": 0.44,
66
+ "learning_rate": 2.4644913627639156e-05,
67
+ "loss": 0.6934,
68
+ "step": 1000
69
+ },
70
+ {
71
+ "epoch": 0.48,
72
+ "learning_rate": 2.2725527831094052e-05,
73
+ "loss": 0.6665,
74
+ "step": 1100
75
+ },
76
+ {
77
+ "epoch": 0.53,
78
+ "learning_rate": 2.0806142034548945e-05,
79
+ "loss": 0.6466,
80
+ "step": 1200
81
+ },
82
+ {
83
+ "epoch": 0.57,
84
+ "learning_rate": 1.888675623800384e-05,
85
+ "loss": 0.6095,
86
+ "step": 1300
87
+ },
88
+ {
89
+ "epoch": 0.61,
90
+ "learning_rate": 1.6967370441458736e-05,
91
+ "loss": 0.5723,
92
+ "step": 1400
93
+ },
94
+ {
95
+ "epoch": 0.66,
96
+ "learning_rate": 1.5047984644913629e-05,
97
+ "loss": 0.5313,
98
+ "step": 1500
99
+ },
100
+ {
101
+ "epoch": 0.7,
102
+ "learning_rate": 1.3128598848368523e-05,
103
+ "loss": 0.514,
104
+ "step": 1600
105
+ },
106
+ {
107
+ "epoch": 0.74,
108
+ "learning_rate": 1.1209213051823417e-05,
109
+ "loss": 0.4911,
110
+ "step": 1700
111
+ },
112
+ {
113
+ "epoch": 0.79,
114
+ "learning_rate": 9.289827255278311e-06,
115
+ "loss": 0.4456,
116
+ "step": 1800
117
+ },
118
+ {
119
+ "epoch": 0.83,
120
+ "learning_rate": 7.370441458733206e-06,
121
+ "loss": 0.423,
122
+ "step": 1900
123
+ },
124
+ {
125
+ "epoch": 0.88,
126
+ "learning_rate": 5.4510556621881e-06,
127
+ "loss": 0.3877,
128
+ "step": 2000
129
+ },
130
+ {
131
+ "epoch": 0.92,
132
+ "learning_rate": 3.5508637236084453e-06,
133
+ "loss": 0.3594,
134
+ "step": 2100
135
+ },
136
+ {
137
+ "epoch": 0.96,
138
+ "learning_rate": 1.63147792706334e-06,
139
+ "loss": 0.346,
140
+ "step": 2200
141
+ },
142
+ {
143
+ "epoch": 1.0,
144
+ "step": 2284,
145
+ "total_flos": 1.7946134696558592e+16,
146
+ "train_loss": 0.6671193393433366,
147
+ "train_runtime": 6110.3906,
148
+ "train_samples_per_second": 5.981,
149
+ "train_steps_per_second": 0.374
150
+ }
151
+ ],
152
+ "max_steps": 2284,
153
+ "num_train_epochs": 2,
154
+ "total_flos": 1.7946134696558592e+16,
155
+ "trial_name": null,
156
+ "trial_params": null
157
+ }