Hugging Face's logo

ZeroUniqueness
/
qlora-llama-2-13b-code

PEFT
Model card Files
xet
 Community
ZeroUniqueness commited on
Commit
12fa827
·
1 Parent(s): 9f166b1

Training in progress, step 400

Browse files
Files changed (27) hide show
  1. adapter_config.json +3 -3
  2. adapter_model.bin +1 -1
  3. checkpoint-300/adapter_model/README.md +20 -0
  4. checkpoint-300/adapter_model/adapter_config.json +26 -0
  5. checkpoint-300/adapter_model/adapter_model.bin +3 -0
  6. checkpoint-400/README.md +20 -0
  7. checkpoint-400/adapter_config.json +26 -0
  8. checkpoint-400/adapter_model.bin +3 -0
  9. checkpoint-400/optimizer.pt +3 -0
  10. checkpoint-400/rng_state_0.pth +3 -0
  11. checkpoint-400/rng_state_1.pth +3 -0
  12. checkpoint-400/rng_state_10.pth +3 -0
  13. checkpoint-400/rng_state_11.pth +3 -0
  14. checkpoint-400/rng_state_12.pth +3 -0
  15. checkpoint-400/rng_state_13.pth +3 -0
  16. checkpoint-400/rng_state_2.pth +3 -0
  17. checkpoint-400/rng_state_3.pth +3 -0
  18. checkpoint-400/rng_state_4.pth +3 -0
  19. checkpoint-400/rng_state_5.pth +3 -0
  20. checkpoint-400/rng_state_6.pth +3 -0
  21. checkpoint-400/rng_state_7.pth +3 -0
  22. checkpoint-400/rng_state_8.pth +3 -0
  23. checkpoint-400/rng_state_9.pth +3 -0
  24. checkpoint-400/scheduler.pt +3 -0
  25. checkpoint-400/trainer_state.json +88 -0
  26. checkpoint-400/training_args.bin +3 -0
  27. training_args.bin +1 -1
adapter_config.json CHANGED
@@ -14,13 +14,13 @@
14
  "r": 32,
15
  "revision": null,
16
  "target_modules": [
 
17
  "v_proj",
18
  "q_proj",
19
- "up_proj",
20
  "gate_proj",
21
  "o_proj",
22
- "k_proj",
23
- "down_proj"
24
  ],
25
  "task_type": "CAUSAL_LM"
26
  }
 
14
  "r": 32,
15
  "revision": null,
16
  "target_modules": [
17
+ "up_proj",
18
  "v_proj",
19
  "q_proj",
 
20
  "gate_proj",
21
  "o_proj",
22
+ "down_proj",
23
+ "k_proj"
24
  ],
25
  "task_type": "CAUSAL_LM"
26
  }
adapter_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6361cf788eb2eecf574488f94018046acb7d636e2b8c01e3508587db412f3cd2
3
  size 500897101
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e512492edd536e51c60bcbb8c9957e1d0e6f7931b1e74e7ec9ee0234700192c3
3
  size 500897101
checkpoint-300/adapter_model/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.5.0.dev0
checkpoint-300/adapter_model/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.05,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 32,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "v_proj",
18
+ "q_proj",
19
+ "up_proj",
20
+ "gate_proj",
21
+ "o_proj",
22
+ "k_proj",
23
+ "down_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
checkpoint-300/adapter_model/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6361cf788eb2eecf574488f94018046acb7d636e2b8c01e3508587db412f3cd2
3
+ size 500897101
checkpoint-400/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: bfloat16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.5.0.dev0
checkpoint-400/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
4
+ "bias": "none",
5
+ "fan_in_fan_out": null,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.05,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 32,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "up_proj",
18
+ "v_proj",
19
+ "q_proj",
20
+ "gate_proj",
21
+ "o_proj",
22
+ "down_proj",
23
+ "k_proj"
24
+ ],
25
+ "task_type": "CAUSAL_LM"
26
+ }
checkpoint-400/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e512492edd536e51c60bcbb8c9957e1d0e6f7931b1e74e7ec9ee0234700192c3
3
+ size 500897101
checkpoint-400/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbefa594d25ff7379baec9edc330359e9a73df5f3e343556fef3aeb00d319fb1
3
+ size 1001752701
checkpoint-400/rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f1eb674bb02e67d4589199ae9931473cb62018c242855cf09b9dd43f8484638
3
+ size 27772
checkpoint-400/rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0be2a68c6c20b21b8f680f47c7e5475082da81e04d2e83110360930c6be157d3
3
+ size 27772
checkpoint-400/rng_state_10.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b747a7482a958133db09c40664dbe39ee65470b22016451da8bd1f43c6eaec6
3
+ size 27789
checkpoint-400/rng_state_11.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10728b0cb8c17194fccb52800bf489683b6dd0894e19ec54679adae843382d0e
3
+ size 27789
checkpoint-400/rng_state_12.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e8c9b7ea97d4cd26ef693fe751e56148293d5257d27265a99cceb1f8bcd9f4c
3
+ size 27789
checkpoint-400/rng_state_13.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7093eac9f530543bb5433f6c71fd2e3d7662efefe6c744b586ee55e9b1483e3
3
+ size 27789
checkpoint-400/rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e00aa011461a34945b68d5fdf023371253b2de3cc91b24c58270b9a606c0af1
3
+ size 27772
checkpoint-400/rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f372ab6037d82e169d3ba8549bd74dfbdc8c8a7b3304d895cf594dc43900ac83
3
+ size 27772
checkpoint-400/rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfab7205a91d0392273aaf1c5d792acfa7ae48b2854b2c24dd5c16a9bf74f78e
3
+ size 27772
checkpoint-400/rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f46b6acfc982a7a20ee730a0170f5d166d726ff8f50bc2072a37fdeb4c65005f
3
+ size 27772
checkpoint-400/rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:846fae8a563fbe2d2e02aea1ba22ebc36cd98e1f757f0b91180f7f90be3c741c
3
+ size 27772
checkpoint-400/rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f92957e3fc01793f6473567bfe29f88ffabd86be522f71501d72300cf4130fb2
3
+ size 27772
checkpoint-400/rng_state_8.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4eb8a9c648ccdb199ce4e6eed47c4b6e665966762b666aef22b8e0cb0e78a81
3
+ size 27772
checkpoint-400/rng_state_9.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:803b4e5f36e19c3c75458cce335aa8826ef1a896380d4dcdecde4d791c7ce857
3
+ size 27772
checkpoint-400/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b200bf94e090388e19457e61e5dc10f4bbbee568d22b1db4e6b2f28c0739166b
3
+ size 627
checkpoint-400/trainer_state.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.15509887553315238,
5
+ "global_step": 400,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.02,
12
+ "learning_rate": 0.0001999867761371633,
13
+ "loss": 1.0435,
14
+ "step": 50
15
+ },
16
+ {
17
+ "epoch": 0.04,
18
+ "learning_rate": 0.00019993306018843102,
19
+ "loss": 0.8918,
20
+ "step": 100
21
+ },
22
+ {
23
+ "epoch": 0.06,
24
+ "learning_rate": 0.00019983804784290833,
25
+ "loss": 0.8874,
26
+ "step": 150
27
+ },
28
+ {
29
+ "epoch": 0.08,
30
+ "learning_rate": 0.00019970177836355307,
31
+ "loss": 0.8839,
32
+ "step": 200
33
+ },
34
+ {
35
+ "epoch": 0.09,
36
+ "learning_rate": 0.00019961818913082012,
37
+ "loss": 0.8801,
38
+ "step": 225
39
+ },
40
+ {
41
+ "epoch": 0.1,
42
+ "learning_rate": 0.00019952430806244534,
43
+ "loss": 0.8753,
44
+ "step": 250
45
+ },
46
+ {
47
+ "epoch": 0.11,
48
+ "learning_rate": 0.00019942014485754635,
49
+ "loss": 0.8754,
50
+ "step": 275
51
+ },
52
+ {
53
+ "epoch": 0.12,
54
+ "learning_rate": 0.00019930571027751713,
55
+ "loss": 0.8751,
56
+ "step": 300
57
+ },
58
+ {
59
+ "epoch": 0.13,
60
+ "learning_rate": 0.0001991810161449164,
61
+ "loss": 0.8819,
62
+ "step": 325
63
+ },
64
+ {
65
+ "epoch": 0.14,
66
+ "learning_rate": 0.00019904607534224612,
67
+ "loss": 0.8744,
68
+ "step": 350
69
+ },
70
+ {
71
+ "epoch": 0.15,
72
+ "learning_rate": 0.00019890090181062063,
73
+ "loss": 0.8735,
74
+ "step": 375
75
+ },
76
+ {
77
+ "epoch": 0.16,
78
+ "learning_rate": 0.00019874551054832625,
79
+ "loss": 0.8703,
80
+ "step": 400
81
+ }
82
+ ],
83
+ "max_steps": 7737,
84
+ "num_train_epochs": 3,
85
+ "total_flos": 1.7210263808560333e+18,
86
+ "trial_name": null,
87
+ "trial_params": null
88
+ }
checkpoint-400/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d24c68a2a0fe89f30403f3e593da35ac1d29335618b90ef963a22423c586398b
3
+ size 4027
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fe6d7bbf6b8ed54bb30d4fb9291907ac865f24b396951fe9912d2874dc58e000
3
  size 4027
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d24c68a2a0fe89f30403f3e593da35ac1d29335618b90ef963a22423c586398b
3
  size 4027