ospanbatyr commited on
Commit
f92d08b
·
verified ·
1 Parent(s): 5647020

Add 13 model file(s)

Browse files
t1_pure_text_s_d10.2_eb1024_lr1e-3_seed2/branches/branch_5Blb/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a842a73533c4e8a15808c88c3f52d05b96017478a6bd0bad0dc9bbd693e504a
3
+ size 1944810018
t1_pure_text_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-10Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9677a33d8d05f2b2636144540d56ef5967272f38128b53898cf1edfb7fccf1d7
3
+ size 1944809762
t1_pure_text_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:951b5889511faac1c4d051736c2d682923a45656106459acf46f54e32172c413
3
+ size 1944809257
t1_pure_text_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db53b0b1fb3f5c2d1b9c0c576a5063def6d793f0ef80fac75133d6693233d0d9
3
+ size 1944809762
t1_pure_text_s_d10.2_eb512_lr1e-3_seed1/checkpoint-10Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33732409d52f9a6f16a925641272d00388a8f6c5e99751078a236fdbe9f67c51
3
+ size 1944809762
t1_pure_text_s_d10.2_eb512_lr1e-3_seed1/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:358469427a97472eb55c54339e0ae6259bc2cc787f73e8718535cf20885bbf09
3
+ size 1944809257
t1_pure_text_s_d10.2_eb512_lr1e-3_seed1/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4ea086368bfe56c968bab43397aabec2c8f7bc8c395486f84fb10ffa7fa1894
3
+ size 1944809762
t1_pure_text_s_d10.2_eb512_lr1e-3_seed1/config.json ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_text_s_d10.2_eb512_lr1e-3_seed1",
3
+ "epoch": 0,
4
+ "n_params": 165213696,
5
+ "args": {
6
+ "run_name": "t1_pure_text_s_d10.2_eb512_lr1e-3_seed1",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 10.2,
11
+ "accum_iter": 16,
12
+ "effective_batch_size": 512,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.001,
39
+ "min_blr": 5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "warmup_steps": 250,
43
+ "cooldown_steps": -1,
44
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml",
45
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml",
46
+ "cross_eval_val": "",
47
+ "s3_endpoint": "",
48
+ "s3_data_endpoint": "",
49
+ "s3_multipart_chunksize_mb": 512,
50
+ "s3_multipart_threshold_mb": 512,
51
+ "s3_max_io_queue": 100,
52
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
53
+ "eval_freq": 1,
54
+ "eval_freq_tokens": 2,
55
+ "dist_eval": true,
56
+ "eval": false,
57
+ "val_sampling_mode": "without_replacement",
58
+ "output_dir": "output/cc/t1_pure_text_s_d10.2_eb512_lr1e-3_seed1",
59
+ "device": "cuda",
60
+ "seed": 1,
61
+ "resume": "",
62
+ "auto_resume": true,
63
+ "start_epoch": 0,
64
+ "num_workers": 10,
65
+ "pin_mem": true,
66
+ "find_unused_params": true,
67
+ "rlimit": 4096,
68
+ "print_all": false,
69
+ "show_user_warnings": false,
70
+ "s3_save_dir": "",
71
+ "dist_url": "env://",
72
+ "log_wandb": true,
73
+ "wandb_project": "loom",
74
+ "wandb_entity": null,
75
+ "wandb_run_name": "t1_pure_text_s_d10.2_eb512_lr1e-3_seed1",
76
+ "wandb_group": "cc-solo",
77
+ "dim": null,
78
+ "depth": null,
79
+ "num_heads": null,
80
+ "mlp_ratio": null,
81
+ "num_experts": 1,
82
+ "granularity": 1,
83
+ "routing": null,
84
+ "moe_aux_coef": 0.01,
85
+ "z_loss_coef": null,
86
+ "moe_compile": false,
87
+ "log_grad_norms_every": 0,
88
+ "save_at_tokens": null,
89
+ "unigram_entropies": null,
90
+ "cooldown_branch": false,
91
+ "cooldown_tokens": 0,
92
+ "config_path": "cfgs/default/4m/experiments/confound/solo_s/text/t1_pure_text_s_d10.2_eb512_lr1e-3_seed1.yaml",
93
+ "rank": 0,
94
+ "world_size": 1,
95
+ "gpu": 0,
96
+ "distributed": true,
97
+ "dist_backend": "nccl",
98
+ "num_tasks": 1,
99
+ "sampling_mode": "without_replacement",
100
+ "max_micro_steps": 311280,
101
+ "lossbearing_target": 10200000000.0,
102
+ "lossbearing_mode": true,
103
+ "lr": 0.001,
104
+ "min_lr": 5e-05,
105
+ "global_micro_step": 311280,
106
+ "cum_lossbearing": 10200765474.0,
107
+ "cum_mod_tokens": {
108
+ "text": 10200765474
109
+ }
110
+ },
111
+ "source_configs": {
112
+ "cfgs/default/4m/experiments/confound/solo_s/text/t1_pure_text_s_d10.2_eb512_lr1e-3_seed1.yaml": "# S solo variant t1_pure_text_s_d10.2_eb512_lr1e-3_seed1 (confound program D6-1).\n# Clone of t1/unimodal/text/run.yaml: D 10.2B, effective batch 512, absolute LR 0.001 (lr_batch_scaling none), seed 1.\n# Self-cooled WSD endpoint; compared with the other D6-1 solo variants of the same modality\n# and D (seed spread, LR x batch grid, warmup / z-loss controls).\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_text_s_d10.2_eb512_lr1e-3_seed1\ntotal_tokens: -1\nlossbearing_tokens: 10.2\ndata_config: cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 2\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-solo\neffective_batch_size: 512\nlr_batch_scaling: none\nblr: 0.001\nseed: 1\naccum_iter: 16\nmin_blr: 5.0e-05\n",
113
+ "cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml": "# T1 unimodal training data \u2014 text\n# Sources: dclm\n\nmodalities:\n- text\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: dclm\n shards: data/shards/dclm_reshuffled/train/shard_*.tar\n weight: 426009612\n modality_tokens:\n text: 1300\n format: npz\n",
114
+ "cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml": "# T1 unimodal validation data \u2014 text\n# Sources: dclm_val\n\nmodalities:\n- text\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: dclm_val\n shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n text: 1300\n format: npz\n"
115
+ }
116
+ }
t1_pure_text_s_d10.2_eb512_lr1e-3_seed2/checkpoint-10Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ffeeb3a04df6cb851ef1234852066ba4910c7e99616ff166abeaaeb8e17b4a1
3
+ size 1944809762
t1_pure_text_s_d10.2_eb512_lr1e-3_seed2/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e6fe8ddc16796dd082e43b9fd4f38335c836187f59452a9035e5559b45dee7f
3
+ size 1944809257
t1_pure_text_s_d10.2_eb512_lr1e-3_seed2/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1148e5c7caebdd2291117ef5ec61d28777bad8274fff7dce4a95a49fa701c773
3
+ size 1944809762
t1_pure_text_s_d10.2_eb512_lr1e-3_seed2/config.json ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_text_s_d10.2_eb512_lr1e-3_seed2",
3
+ "epoch": 0,
4
+ "n_params": 165213696,
5
+ "args": {
6
+ "run_name": "t1_pure_text_s_d10.2_eb512_lr1e-3_seed2",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 10.2,
11
+ "accum_iter": 16,
12
+ "effective_batch_size": 512,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.001,
39
+ "min_blr": 5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "warmup_steps": 250,
43
+ "cooldown_steps": -1,
44
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml",
45
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml",
46
+ "cross_eval_val": "",
47
+ "s3_endpoint": "",
48
+ "s3_data_endpoint": "",
49
+ "s3_multipart_chunksize_mb": 512,
50
+ "s3_multipart_threshold_mb": 512,
51
+ "s3_max_io_queue": 100,
52
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
53
+ "eval_freq": 1,
54
+ "eval_freq_tokens": 2,
55
+ "dist_eval": true,
56
+ "eval": false,
57
+ "val_sampling_mode": "without_replacement",
58
+ "output_dir": "output/cc/t1_pure_text_s_d10.2_eb512_lr1e-3_seed2",
59
+ "device": "cuda",
60
+ "seed": 2,
61
+ "resume": "",
62
+ "auto_resume": true,
63
+ "start_epoch": 0,
64
+ "num_workers": 10,
65
+ "pin_mem": true,
66
+ "find_unused_params": true,
67
+ "rlimit": 4096,
68
+ "print_all": false,
69
+ "show_user_warnings": false,
70
+ "s3_save_dir": "",
71
+ "dist_url": "env://",
72
+ "log_wandb": true,
73
+ "wandb_project": "loom",
74
+ "wandb_entity": null,
75
+ "wandb_run_name": "t1_pure_text_s_d10.2_eb512_lr1e-3_seed2",
76
+ "wandb_group": "cc-solo",
77
+ "dim": null,
78
+ "depth": null,
79
+ "num_heads": null,
80
+ "mlp_ratio": null,
81
+ "num_experts": 1,
82
+ "granularity": 1,
83
+ "routing": null,
84
+ "moe_aux_coef": 0.01,
85
+ "z_loss_coef": null,
86
+ "moe_compile": false,
87
+ "log_grad_norms_every": 0,
88
+ "save_at_tokens": null,
89
+ "unigram_entropies": null,
90
+ "cooldown_branch": false,
91
+ "cooldown_tokens": 0,
92
+ "config_path": "cfgs/default/4m/experiments/confound/solo_s/text/t1_pure_text_s_d10.2_eb512_lr1e-3_seed2.yaml",
93
+ "rank": 0,
94
+ "world_size": 1,
95
+ "gpu": 0,
96
+ "distributed": true,
97
+ "dist_backend": "nccl",
98
+ "num_tasks": 1,
99
+ "sampling_mode": "without_replacement",
100
+ "max_micro_steps": 311280,
101
+ "lossbearing_target": 10200000000.0,
102
+ "lossbearing_mode": true,
103
+ "lr": 0.001,
104
+ "min_lr": 5e-05,
105
+ "global_micro_step": 311280,
106
+ "cum_lossbearing": 10200077159.0,
107
+ "cum_mod_tokens": {
108
+ "text": 10200077159
109
+ }
110
+ },
111
+ "source_configs": {
112
+ "cfgs/default/4m/experiments/confound/solo_s/text/t1_pure_text_s_d10.2_eb512_lr1e-3_seed2.yaml": "# S solo variant t1_pure_text_s_d10.2_eb512_lr1e-3_seed2 (confound program D6-1).\n# Clone of t1/unimodal/text/run.yaml: D 10.2B, effective batch 512, absolute LR 0.001 (lr_batch_scaling none), seed 2.\n# Self-cooled WSD endpoint; compared with the other D6-1 solo variants of the same modality\n# and D (seed spread, LR x batch grid, warmup / z-loss controls).\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_text_s_d10.2_eb512_lr1e-3_seed2\ntotal_tokens: -1\nlossbearing_tokens: 10.2\ndata_config: cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 2\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-solo\neffective_batch_size: 512\nlr_batch_scaling: none\nblr: 0.001\nseed: 2\naccum_iter: 16\nmin_blr: 5.0e-05\n",
113
+ "cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml": "# T1 unimodal training data \u2014 text\n# Sources: dclm\n\nmodalities:\n- text\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: dclm\n shards: data/shards/dclm_reshuffled/train/shard_*.tar\n weight: 426009612\n modality_tokens:\n text: 1300\n format: npz\n",
114
+ "cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml": "# T1 unimodal validation data \u2014 text\n# Sources: dclm_val\n\nmodalities:\n- text\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: dclm_val\n shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n text: 1300\n format: npz\n"
115
+ }
116
+ }
t1_pure_text_s_d40_eb1024_lr1e-3_seed0/config.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_text_s_d40_eb1024_lr1e-3_seed0",
3
+ "epoch": 0,
4
+ "n_params": 165213696,
5
+ "args": {
6
+ "run_name": "t1_pure_text_s_d40_eb1024_lr1e-3_seed0",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 40,
11
+ "accum_iter": 32,
12
+ "effective_batch_size": 1024,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.001,
39
+ "min_blr": 5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "lr_shape": "wsd",
43
+ "warmup_steps": 250,
44
+ "cooldown_steps": -1,
45
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml",
46
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml",
47
+ "cross_eval_val": "",
48
+ "s3_endpoint": "",
49
+ "s3_data_endpoint": "",
50
+ "s3_multipart_chunksize_mb": 512,
51
+ "s3_multipart_threshold_mb": 512,
52
+ "s3_max_io_queue": 100,
53
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
54
+ "eval_freq": 1,
55
+ "eval_freq_tokens": 2,
56
+ "dist_eval": true,
57
+ "eval": false,
58
+ "val_sampling_mode": "without_replacement",
59
+ "output_dir": "output/cc/t1_pure_text_s_d40_eb1024_lr1e-3_seed0",
60
+ "device": "cuda",
61
+ "seed": 0,
62
+ "resume": "",
63
+ "auto_resume": true,
64
+ "start_epoch": 0,
65
+ "num_workers": 10,
66
+ "pin_mem": true,
67
+ "find_unused_params": true,
68
+ "rlimit": 4096,
69
+ "print_all": false,
70
+ "show_user_warnings": false,
71
+ "s3_save_dir": "",
72
+ "dist_url": "env://",
73
+ "log_wandb": true,
74
+ "wandb_project": "loom",
75
+ "wandb_entity": null,
76
+ "wandb_run_name": "t1_pure_text_s_d40_eb1024_lr1e-3_seed0",
77
+ "wandb_group": "cc-datascale",
78
+ "dim": null,
79
+ "depth": null,
80
+ "num_heads": null,
81
+ "mlp_ratio": null,
82
+ "num_experts": 1,
83
+ "granularity": 1,
84
+ "routing": null,
85
+ "moe_aux_coef": 0.01,
86
+ "z_loss_coef": null,
87
+ "moe_compile": false,
88
+ "log_grad_norms_every": 0,
89
+ "save_at_tokens": "5,10,20",
90
+ "unigram_entropies": null,
91
+ "cooldown_branch": false,
92
+ "cooldown_tokens": 0,
93
+ "config_path": "cfgs/default/4m/experiments/confound/datascale/text/t1_pure_text_s_d40_eb1024_lr1e-3_seed0.yaml",
94
+ "rank": 0,
95
+ "world_size": 1,
96
+ "gpu": 0,
97
+ "distributed": true,
98
+ "dist_backend": "nccl",
99
+ "num_tasks": 1,
100
+ "sampling_mode": "without_replacement",
101
+ "max_micro_steps": 1220704,
102
+ "lossbearing_target": 40000000000.0,
103
+ "lossbearing_mode": true,
104
+ "lr": 0.001,
105
+ "min_lr": 5e-05,
106
+ "global_micro_step": 1220704,
107
+ "cum_lossbearing": 40001253589.0,
108
+ "cum_mod_tokens": {
109
+ "text": 40001253589
110
+ }
111
+ },
112
+ "source_configs": {
113
+ "cfgs/default/4m/experiments/confound/datascale/text/t1_pure_text_s_d40_eb1024_lr1e-3_seed0.yaml": "# DATA-SCALE run t1_pure_text_s_d40_eb1024_lr1e-3_seed0 (confound program stage 2 \u2014 D6-4 \u00a72.3, option C).\n# Clone of t1/unimodal/text/run.yaml extended to 40B loss-bearing tokens at\n# effective batch 1024, LR held at 1e-3, single pass. Checkpoints at [5, 10, 20]B are cooled\n# into branches (allow-listed); the eb1024 - eb512 difference at each D is the step gain as\n# a function of data, at two model sizes (S and XS) over the same D.\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_text_s_d40_eb1024_lr1e-3_seed0\ntotal_tokens: -1\nlossbearing_tokens: 40\ndata_config: cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 2\nsave_ckpt_freq_tokens: 5\nsave_at_tokens: 5,10,20\nwandb_group: cc-datascale\neffective_batch_size: 1024\nlr_batch_scaling: none\nblr: 0.001\nseed: 0\naccum_iter: 32\nmin_blr: 5.0e-05\n",
114
+ "cfgs/default/4m/experiments/t1/unimodal/text/data_train.yaml": "# T1 unimodal training data \u2014 text\n# Sources: dclm\n\nmodalities:\n- text\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: dclm\n shards: data/shards/dclm_reshuffled/train/shard_*.tar\n weight: 426009612\n modality_tokens:\n text: 1300\n format: npz\n",
115
+ "cfgs/default/4m/experiments/t1/unimodal/text/data_val.yaml": "# T1 unimodal validation data \u2014 text\n# Sources: dclm_val\n\nmodalities:\n- text\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: dclm_val\n shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n text: 1300\n format: npz\n"
116
+ }
117
+ }