Skywalker1910 commited on
Commit
51aabe3
·
verified ·
1 Parent(s): 6616e5a

Upload folder using huggingface_hub

Browse files
configs/bpe_shakespeare_v007.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v007 — Tokenizer comparison: BPE on Shakespeare with a larger model
2
+ # Goal: compare BPE vs character tokenization on the same corpus
3
+ experiment:
4
+ objective: bpe_shakespeare_language_model
5
+ status: completed
6
+ baseline: bb8-char-medium-v006
7
+ change: BPE-3000 tokenizer, d_model 128->256, layers 4->6, context 256->512
8
+
9
+ model:
10
+ d_model: 256
11
+ num_layers: 6
12
+ num_heads: 8
13
+ d_ff: 1024
14
+ max_seq_len: 512
15
+ dropout: 0.1
16
+ activation: gelu
17
+ tie_weights: true
18
+
19
+ tokenizer:
20
+ type: bpe
21
+ vocab_size: 3000
22
+
23
+ training:
24
+ batch_size: 16
25
+ num_epochs: 20
26
+ lr: 3.0e-4
27
+ weight_decay: 0.1
28
+ max_grad_norm: 1.0
29
+ warmup_steps: 200
30
+ mixed_precision: true
31
+ log_interval: 10
32
+ eval_interval: 200
33
+ save_interval: 1000
34
+
35
+ data:
36
+ seq_len: 512
37
+ stride: 64
38
+ val_fraction: 0.1
39
+
40
+ generation:
41
+ max_new_tokens: 300
42
+ temperature: 0.8
43
+ top_k: 50
44
+ top_p: 0.9
45
+ repetition_penalty: 1.1
configs/bpe_shakespeare_v007a.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v007a — Smaller BPE vocab (1000) to isolate vocab size effect
2
+ # Goal: compare BPE-1000 vs BPE-3000 on same architecture
3
+ experiment:
4
+ objective: bpe_vocab_size_ablation
5
+ status: completed
6
+ baseline: bb8-bpe-shakespeare-v007
7
+ change: vocab_size 3000->1000
8
+
9
+ model:
10
+ d_model: 256
11
+ num_layers: 6
12
+ num_heads: 8
13
+ d_ff: 1024
14
+ max_seq_len: 512
15
+ dropout: 0.1
16
+ activation: gelu
17
+ tie_weights: true
18
+
19
+ tokenizer:
20
+ type: bpe
21
+ vocab_size: 1000
22
+
23
+ training:
24
+ batch_size: 16
25
+ num_epochs: 20
26
+ lr: 3.0e-4
27
+ weight_decay: 0.1
28
+ max_grad_norm: 1.0
29
+ warmup_steps: 200
30
+ mixed_precision: true
31
+ log_interval: 10
32
+ eval_interval: 200
33
+ save_interval: 1000
34
+
35
+ data:
36
+ seq_len: 512
37
+ stride: 256
38
+ val_fraction: 0.1
39
+
40
+ generation:
41
+ max_new_tokens: 300
42
+ temperature: 0.8
43
+ top_k: 50
44
+ top_p: 0.9
45
+ repetition_penalty: 1.1
configs/char_medium_v006.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v006 — Scaling study: same Shakespeare data, 7x more parameters
2
+ # Goal: measure quality gain from wider embeddings, more layers, longer context
3
+ experiment:
4
+ objective: character_language_model_scaling
5
+ status: completed
6
+ baseline: bb8-char-small-v002
7
+ change: d_model 64->128, layers 2->4, context 128->256
8
+
9
+ model:
10
+ d_model: 128
11
+ num_layers: 4
12
+ num_heads: 4
13
+ d_ff: 512
14
+ max_seq_len: 256
15
+ dropout: 0.1
16
+ activation: gelu
17
+ tie_weights: true
18
+
19
+ tokenizer:
20
+ type: char
21
+ vocab_size: null
22
+
23
+ training:
24
+ batch_size: 32
25
+ num_epochs: 15
26
+ lr: 3.0e-4
27
+ weight_decay: 0.1
28
+ max_grad_norm: 1.0
29
+ warmup_steps: 100
30
+ mixed_precision: true
31
+ log_interval: 20
32
+ eval_interval: 200
33
+ save_interval: 1000
34
+
35
+ data:
36
+ seq_len: 256
37
+ stride: 128
38
+ val_fraction: 0.1
39
+
40
+ generation:
41
+ max_new_tokens: 200
42
+ temperature: 0.8
43
+ top_k: 50
44
+ top_p: 0.9
45
+ repetition_penalty: 1.1
configs/char_medium_v006b.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v006b — Deeper variant: 6 layers instead of 4, same width
2
+ # Goal: test if depth helps more than width for character models
3
+ experiment:
4
+ objective: character_language_model_depth_ablation
5
+ status: completed
6
+ baseline: bb8-char-medium-v006
7
+ change: num_layers 4->6, keeping d_model=128
8
+
9
+ model:
10
+ d_model: 128
11
+ num_layers: 6
12
+ num_heads: 4
13
+ d_ff: 512
14
+ max_seq_len: 256
15
+ dropout: 0.1
16
+ activation: gelu
17
+ tie_weights: true
18
+
19
+ tokenizer:
20
+ type: char
21
+ vocab_size: null
22
+
23
+ training:
24
+ batch_size: 32
25
+ num_epochs: 15
26
+ lr: 3.0e-4
27
+ weight_decay: 0.1
28
+ max_grad_norm: 1.0
29
+ warmup_steps: 100
30
+ mixed_precision: true
31
+ log_interval: 20
32
+ eval_interval: 200
33
+ save_interval: 1000
34
+
35
+ data:
36
+ seq_len: 256
37
+ stride: 128
38
+ val_fraction: 0.1
39
+
40
+ generation:
41
+ max_new_tokens: 200
42
+ temperature: 0.8
43
+ top_k: 50
44
+ top_p: 0.9
45
+ repetition_penalty: 1.1
configs/chat_models.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "grounded-base": {
3
+ "label": "Grounded Qwen — unchanged baseline + evidence",
4
+ "model_dir": "deployment/qwen-grounded-baseline",
5
+ "origin": "external_pretrained"
6
+ },
7
+ "grounded-lora": {
8
+ "label": "BB8 v005 pilot — our LoRA + evidence (experimental)",
9
+ "model_dir": "checkpoints/bb8-grounded-v005-pilot",
10
+ "origin": "local_fine_tune_not_promoted"
11
+ },
12
+ "qwen-instruct": {
13
+ "label": "Qwen Instruct — external baseline (recommended)",
14
+ "model_dir": "deployment/qwen-instruct-baseline",
15
+ "origin": "external_pretrained"
16
+ }
17
+ }
configs/default.yaml ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ d_model: 128
3
+ num_layers: 4
4
+ num_heads: 4
5
+ d_ff: 512
6
+ max_seq_len: 256
7
+ dropout: 0.1
8
+ activation: gelu
9
+ tie_weights: true
10
+
11
+ tokenizer:
12
+ type: char # char | word | bpe
13
+ vocab_size: 5000 # used only for word/bpe; ignored for char
14
+
15
+ training:
16
+ batch_size: 32
17
+ num_epochs: 10
18
+ lr: 3.0e-4
19
+ weight_decay: 0.1
20
+ max_grad_norm: 1.0
21
+ warmup_steps: 100
22
+ mixed_precision: true
23
+ log_interval: 10
24
+ eval_interval: 100
25
+ save_interval: 500
26
+
27
+ data:
28
+ seq_len: 256
29
+ stride: 1
30
+ val_fraction: 0.1
31
+
32
+ generation:
33
+ max_new_tokens: 200
34
+ temperature: 0.8
35
+ top_k: 50
36
+ top_p: 0.9
37
+ repetition_penalty: 1.1
configs/instruction_v003.yaml ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Exploratory instruction-language-model run for the RTX 4070 (8 GB).
2
+ experiment:
3
+ objective: instruction_language_model
4
+ status: exploratory
5
+ baseline: bb8-char-small-v002
6
+ change: Dolly 15K chat data, BPE tokenization, larger Transformer
7
+
8
+ model:
9
+ d_model: 256
10
+ num_layers: 6
11
+ num_heads: 8
12
+ d_ff: 1024
13
+ max_seq_len: 256
14
+ dropout: 0.1
15
+ activation: gelu
16
+ tie_weights: true
17
+
18
+ tokenizer:
19
+ type: bpe
20
+ vocab_size: 1024
21
+ training_chars: 2000000
22
+
23
+ training:
24
+ batch_size: 24
25
+ num_epochs: 8
26
+ lr: 5.0e-4
27
+ weight_decay: 0.1
28
+ max_grad_norm: 1.0
29
+ warmup_steps: 100
30
+ mixed_precision: true
31
+ log_interval: 20
32
+ eval_interval: 0
33
+ save_interval: 1000
34
+
35
+ data:
36
+ seq_len: 256
37
+ stride: 128
38
+ val_fraction: 0.1
39
+ record_separator: "\n\n"
40
+ manifest: data/manifests/dolly_15k.json
41
+
42
+ generation:
43
+ max_new_tokens: 120
44
+ temperature: 0.7
45
+ top_k: 40
46
+ top_p: 0.9
47
+ repetition_penalty: 1.15
configs/medium_model.yaml ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Medium model — better quality, more compute required
2
+ model:
3
+ d_model: 256
4
+ num_layers: 6
5
+ num_heads: 8
6
+ d_ff: 1024
7
+ max_seq_len: 512
8
+ dropout: 0.1
9
+ activation: gelu
10
+ tie_weights: true
11
+
12
+ tokenizer:
13
+ type: bpe
14
+ vocab_size: 5000
15
+
16
+ training:
17
+ batch_size: 16
18
+ num_epochs: 20
19
+ lr: 3.0e-4
20
+ weight_decay: 0.1
21
+ max_grad_norm: 1.0
22
+ warmup_steps: 200
23
+ mixed_precision: true
24
+ log_interval: 10
25
+ eval_interval: 200
26
+ save_interval: 1000
27
+
28
+ data:
29
+ seq_len: 512
30
+ stride: 1
31
+ val_fraction: 0.1
32
+
33
+ generation:
34
+ max_new_tokens: 300
35
+ temperature: 0.8
36
+ top_k: 50
37
+ top_p: 0.9
38
+ repetition_penalty: 1.1
configs/qwen_grounded_v005.yaml ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ experiment:
2
+ objective: evidence_grounded_answering
3
+ status: experimental_not_promoted
4
+ baseline: Qwen2.5-0.5B-Instruct
5
+ change: native-chat assistant-only LoRA on synthetic evidence/citation/abstention examples
6
+ base_model:
7
+ id: Qwen/Qwen2.5-0.5B-Instruct
8
+ revision: 7ae557604adf67be50417f59c2c2f167def9a775
9
+ license: Apache-2.0
10
+ cache_dir: .cache/huggingface
11
+ data:
12
+ path: data/grounded_v1/train.jsonl
13
+ validation_path: data/grounded_v1/validation.jsonl
14
+ manifest: data/grounded_v1/manifest.json
15
+ preprocessing: chat_full_v1
16
+ seed: 42
17
+ max_length: 768
18
+ lora:
19
+ rank: 8
20
+ alpha: 16
21
+ dropout: 0.05
22
+ target_modules: [q_proj, v_proj]
23
+ training:
24
+ epochs: 3
25
+ batch_size: 2
26
+ eval_batch_size: 2
27
+ gradient_accumulation_steps: 4
28
+ learning_rate: 5.0e-5
29
+ weight_decay: 0.01
30
+ warmup_ratio: 0.05
31
+ max_grad_norm: 1.0
32
+ precision: bf16
33
+ gradient_checkpointing: true
34
+ log_interval: 10
35
+ generation:
36
+ max_new_tokens: 96
37
+ temperature: 0.6
38
+ top_p: 0.9
39
+ repetition_penalty: 1.0
configs/qwen_instruct_v008.yaml ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v008 — Instruction-tuned base + LoRA, longer context
2
+ # Goal: compare starting from Qwen-Instruct vs raw Qwen base
3
+ experiment:
4
+ objective: supervised_instruction_fine_tuning
5
+ status: completed
6
+ baseline: bb8-qwen-lora-v004-dev
7
+ change: instruction-tuned base model, 512-token context, 2 epochs, lower LR
8
+
9
+ base_model:
10
+ id: Qwen/Qwen2.5-0.5B-Instruct
11
+ revision: 7ae557604adf67be50417f59c2c2f167def9a775
12
+ license: Apache-2.0
13
+ cache_dir: .cache/huggingface
14
+
15
+ data:
16
+ preprocessing: full_example_v2
17
+ path: data/databricks-dolly-15k.jsonl
18
+ manifest: data/manifests/dolly_15k.json
19
+ validation_fraction: 0.05
20
+ seed: 42
21
+ max_length: 512
22
+
23
+ lora:
24
+ rank: 16
25
+ alpha: 32
26
+ dropout: 0.05
27
+ target_modules:
28
+ - q_proj
29
+ - k_proj
30
+ - v_proj
31
+ - o_proj
32
+ - gate_proj
33
+ - up_proj
34
+ - down_proj
35
+
36
+ training:
37
+ epochs: 2
38
+ batch_size: 4
39
+ eval_batch_size: 4
40
+ gradient_accumulation_steps: 4
41
+ learning_rate: 1.0e-4
42
+ weight_decay: 0.01
43
+ warmup_ratio: 0.05
44
+ max_grad_norm: 1.0
45
+ precision: bf16
46
+ gradient_checkpointing: true
47
+ log_interval: 25
48
+
49
+ generation:
50
+ max_new_tokens: 120
51
+ temperature: 0.6
52
+ top_p: 0.85
53
+ repetition_penalty: 1.1
configs/qwen_instruct_v008a.yaml ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # v008a — Quick test: smaller LoRA rank, 1 epoch
2
+ # Goal: check if rank=8 is enough before committing to rank=16 and 2 epochs
3
+ experiment:
4
+ objective: supervised_instruction_fine_tuning
5
+ status: completed
6
+ baseline: bb8-qwen-lora-v004-dev
7
+ change: instruction-tuned base, rank 8, 1 epoch, 512 context
8
+
9
+ base_model:
10
+ id: Qwen/Qwen2.5-0.5B-Instruct
11
+ revision: 7ae557604adf67be50417f59c2c2f167def9a775
12
+ license: Apache-2.0
13
+ cache_dir: .cache/huggingface
14
+
15
+ data:
16
+ preprocessing: full_example_v2
17
+ path: data/databricks-dolly-15k.jsonl
18
+ manifest: data/manifests/dolly_15k.json
19
+ validation_fraction: 0.05
20
+ seed: 42
21
+ max_length: 512
22
+
23
+ lora:
24
+ rank: 8
25
+ alpha: 16
26
+ dropout: 0.05
27
+ target_modules:
28
+ - q_proj
29
+ - k_proj
30
+ - v_proj
31
+ - o_proj
32
+ - gate_proj
33
+ - up_proj
34
+ - down_proj
35
+
36
+ training:
37
+ epochs: 1
38
+ batch_size: 4
39
+ eval_batch_size: 4
40
+ gradient_accumulation_steps: 4
41
+ learning_rate: 2.0e-4
42
+ weight_decay: 0.01
43
+ warmup_ratio: 0.03
44
+ max_grad_norm: 1.0
45
+ precision: bf16
46
+ gradient_checkpointing: true
47
+ log_interval: 25
48
+
49
+ generation:
50
+ max_new_tokens: 120
51
+ temperature: 0.6
52
+ top_p: 0.85
53
+ repetition_penalty: 1.1
configs/qwen_lora_v004.yaml ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ experiment:
2
+ objective: supervised_instruction_fine_tuning
3
+ status: exploratory
4
+ baseline: bb8-bpe-instruct-v003-dev
5
+ change: pretrained Qwen2.5 base model with assistant-only LoRA fine-tuning
6
+
7
+ base_model:
8
+ id: Qwen/Qwen2.5-0.5B
9
+ revision: 060db6499f32faf8b98477b0a26969ef7d8b9987
10
+ license: Apache-2.0
11
+ cache_dir: .cache/huggingface
12
+
13
+ data:
14
+ preprocessing: legacy_v1
15
+ path: data/databricks-dolly-15k.jsonl
16
+ manifest: data/manifests/dolly_15k.json
17
+ validation_fraction: 0.05
18
+ seed: 42
19
+ max_length: 256
20
+
21
+ lora:
22
+ rank: 16
23
+ alpha: 32
24
+ dropout: 0.05
25
+ target_modules:
26
+ - q_proj
27
+ - k_proj
28
+ - v_proj
29
+ - o_proj
30
+ - gate_proj
31
+ - up_proj
32
+ - down_proj
33
+
34
+ training:
35
+ epochs: 1
36
+ batch_size: 4
37
+ eval_batch_size: 4
38
+ gradient_accumulation_steps: 4
39
+ learning_rate: 2.0e-4
40
+ weight_decay: 0.01
41
+ warmup_ratio: 0.03
42
+ max_grad_norm: 1.0
43
+ precision: bf16
44
+ gradient_checkpointing: true
45
+ log_interval: 25
46
+
47
+ generation:
48
+ max_new_tokens: 80
49
+ temperature: 0.6
50
+ top_p: 0.85
51
+ repetition_penalty: 1.1
configs/small_model.yaml ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Small model — fast iteration, quick experiments
2
+ model:
3
+ d_model: 64
4
+ num_layers: 2
5
+ num_heads: 2
6
+ d_ff: 256
7
+ max_seq_len: 128
8
+ dropout: 0.1
9
+ activation: gelu
10
+ tie_weights: true
11
+
12
+ tokenizer:
13
+ type: char
14
+ vocab_size: null
15
+
16
+ training:
17
+ batch_size: 64
18
+ num_epochs: 5
19
+ lr: 1.0e-3
20
+ weight_decay: 0.1
21
+ max_grad_norm: 1.0
22
+ warmup_steps: 50
23
+ mixed_precision: false # float32 is faster for this very small model
24
+ log_interval: 100
25
+ eval_interval: 0 # validation still runs at the end of each epoch
26
+ save_interval: 5000
27
+
28
+ data:
29
+ seq_len: 128
30
+ stride: 1
31
+ val_fraction: 0.1
32
+
33
+ generation:
34
+ max_new_tokens: 100
35
+ temperature: 0.8
36
+ top_k: 40
37
+ top_p: 0.9
38
+ repetition_penalty: 1.1