ospanbatyr commited on
Commit
85536f7
·
verified ·
1 Parent(s): 21c9acd

Add 8 model file(s)

Browse files
t1_pure_image_m_d10.2_eb1024_seed0/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75bb50f8d9078dde538f05ff3a53b707ceb5c6ae8c68aa42c16761510e902dcf
3
+ size 3575499504
t1_pure_image_m_d10.2_eb512_lr6e-4_seed0/checkpoint-10Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b443a203c1e3d2bdc62717ca3cdefb6bac94f9eba53d70863354daa7d99d9723
3
+ size 3575499504
t1_pure_image_m_d10.2_eb512_lr6e-4_seed0/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99e720203748331360a2424f96a617fcfdc822ac26a5d1e6141085a7cbc66710
3
+ size 3575498981
t1_pure_image_m_d10.2_eb512_lr6e-4_seed0/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37b6c5fab504a0b1fa9bbb7b97a41b706459e157a7da94e2aad6b8f9c749df22
3
+ size 3575499504
t1_pure_image_m_d10.2_eb512_lr6e-4_seed0/config.json ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_image_m_d10.2_eb512_lr6e-4_seed0",
3
+ "epoch": 0,
4
+ "n_params": 303188320,
5
+ "args": {
6
+ "run_name": "t1_pure_image_m_d10.2_eb512_lr6e-4_seed0",
7
+ "batch_size": 16,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 10.2,
11
+ "accum_iter": 32,
12
+ "effective_batch_size": 512,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_medium_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.0006,
39
+ "min_blr": 3.5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "lr_shape": "wsd",
43
+ "warmup_steps": 250,
44
+ "cooldown_steps": -1,
45
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/image/data_train_pure.yaml",
46
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/image/data_val_pure.yaml",
47
+ "cross_eval_val": "",
48
+ "s3_endpoint": "",
49
+ "s3_data_endpoint": "",
50
+ "s3_multipart_chunksize_mb": 512,
51
+ "s3_multipart_threshold_mb": 512,
52
+ "s3_max_io_queue": 100,
53
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
54
+ "eval_freq": 1,
55
+ "eval_freq_tokens": 3,
56
+ "dist_eval": true,
57
+ "eval": false,
58
+ "val_sampling_mode": "without_replacement",
59
+ "output_dir": "output/cc/t1_pure_image_m_d10.2_eb512_lr6e-4_seed0",
60
+ "device": "cuda",
61
+ "seed": 0,
62
+ "resume": "",
63
+ "auto_resume": true,
64
+ "start_epoch": 0,
65
+ "num_workers": 10,
66
+ "pin_mem": true,
67
+ "find_unused_params": true,
68
+ "rlimit": 4096,
69
+ "print_all": false,
70
+ "show_user_warnings": false,
71
+ "s3_save_dir": "",
72
+ "dist_url": "env://",
73
+ "log_wandb": true,
74
+ "wandb_project": "loom",
75
+ "wandb_entity": null,
76
+ "wandb_run_name": "t1_pure_image_m_d10.2_eb512_lr6e-4_seed0",
77
+ "wandb_group": "cc-mscale",
78
+ "dim": null,
79
+ "depth": null,
80
+ "num_heads": null,
81
+ "mlp_ratio": null,
82
+ "num_experts": 1,
83
+ "granularity": 1,
84
+ "routing": null,
85
+ "moe_aux_coef": 0.01,
86
+ "z_loss_coef": null,
87
+ "moe_compile": false,
88
+ "log_grad_norms_every": 0,
89
+ "save_at_tokens": null,
90
+ "unigram_entropies": null,
91
+ "cooldown_branch": false,
92
+ "cooldown_tokens": 0,
93
+ "config_path": "cfgs/default/4m/experiments/confound/mscale/image/t1_pure_image_m_d10.2_eb512_lr6e-4_seed0.yaml",
94
+ "rank": 0,
95
+ "world_size": 1,
96
+ "gpu": 0,
97
+ "distributed": true,
98
+ "dist_backend": "nccl",
99
+ "num_tasks": 1,
100
+ "sampling_mode": "without_replacement",
101
+ "max_micro_steps": 622559,
102
+ "lossbearing_target": 10200000000.0,
103
+ "lossbearing_mode": true,
104
+ "lr": 0.0006,
105
+ "min_lr": 3.5e-05,
106
+ "global_micro_step": 622559,
107
+ "cum_lossbearing": 10200547634.0,
108
+ "cum_mod_tokens": {
109
+ "tok_rgb@384": 10165656394,
110
+ "text": 34891240
111
+ }
112
+ },
113
+ "source_configs": {
114
+ "cfgs/default/4m/experiments/confound/mscale/image/t1_pure_image_m_d10.2_eb512_lr6e-4_seed0.yaml": "# M-scale half-batch t1_pure_image_m_d10.2_eb512_lr6e-4_seed0 (confound program stage 2 \u2014 D6-4 \u00a72.4).\n# Clone of t1/unimodal/image/run_m_pure.yaml at D 10.2B, effective batch 512, M's own LR 0.0006\n# HELD (lr_batch_scaling none) \u2014 the convention of every stage-0/1 comparison. Compared with the\n# fresh eb1024 baseline t1_pure_image_m_d10.2_eb1024_seed0 (LR 0.0006 at the reference batch).\n_base_: ../../../../models/dec/4m-m-dec.yaml\nrun_name: t1_pure_image_m_d10.2_eb512_lr6e-4_seed0\ntotal_tokens: -1\nlossbearing_tokens: 10.2\ndata_config: cfgs/default/4m/experiments/t1/unimodal/image/data_train_pure.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/image/data_val_pure.yaml\nwarmup_steps: 250\neval_freq_tokens: 3\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-mscale\neffective_batch_size: 512\nlr_batch_scaling: none\nblr: 0.0006\nseed: 0\naccum_iter: 32\nmin_blr: 3.5e-05\n",
115
+ "cfgs/default/4m/experiments/t1/unimodal/image/data_train_pure.yaml": "# T1 pure unimodal training data \u2014 rgb\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_rgb@384\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: coyo\n shards: data/shards/coyo_shuffled/train/shard_{000000..022189}.tar\n weight: 22189292\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: cc12m\n shards: data/shards/cc12m_shuffled/train/shard_{000000..012527}.tar\n weight: 12527184\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: obelics\n shards: data/shards/obelics_shuffled/train/shard_{000000..009245}.tar\n weight: 9245143\n modality_tokens:\n tok_rgb@384: 751\n format: npz\n- name: commoncatalog\n shards: data/shards/commoncatalog_reshuffled/train/shard_*.tar\n weight: 71320000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\nsampling_mode: without_replacement\n",
116
+ "cfgs/default/4m/experiments/t1/unimodal/image/data_val_pure.yaml": "# T1 pure unimodal validation data \u2014 rgb\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_rgb@384\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: coyo_val\n shards: data/shards/coyo_shuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: cc12m_val\n shards: data/shards/cc12m_shuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: obelics_val\n shards: data/shards/obelics_shuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 751\n format: npz\n- name: commoncatalog_val\n shards: data/shards/commoncatalog_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n"
117
+ }
118
+ }
t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0/checkpoint-10Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca06ebca6a2b2b09060cebcfb291bb4c0b311d3d9faad0ecb80cf39eeef03361
3
+ size 2099994096
t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a4afb2b9262db6d1c708fd019b3402a2b81b394176025277f79454a47d7abed
3
+ size 2099993573
t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0/config.json ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0",
3
+ "epoch": 0,
4
+ "n_params": 178790176,
5
+ "args": {
6
+ "run_name": "t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 10.2,
11
+ "accum_iter": 32,
12
+ "effective_batch_size": 1024,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.003,
39
+ "min_blr": 0.00015,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "lr_shape": "wsd",
43
+ "warmup_steps": 250,
44
+ "cooldown_steps": -1,
45
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/image/data_train_pure.yaml",
46
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/image/data_val_pure.yaml",
47
+ "cross_eval_val": "",
48
+ "s3_endpoint": "",
49
+ "s3_data_endpoint": "",
50
+ "s3_multipart_chunksize_mb": 512,
51
+ "s3_multipart_threshold_mb": 512,
52
+ "s3_max_io_queue": 100,
53
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
54
+ "eval_freq": 1,
55
+ "eval_freq_tokens": 2,
56
+ "dist_eval": true,
57
+ "eval": false,
58
+ "val_sampling_mode": "without_replacement",
59
+ "output_dir": "output/cc/t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0",
60
+ "device": "cuda",
61
+ "seed": 0,
62
+ "resume": "",
63
+ "auto_resume": true,
64
+ "start_epoch": 0,
65
+ "num_workers": 10,
66
+ "pin_mem": true,
67
+ "find_unused_params": true,
68
+ "rlimit": 4096,
69
+ "print_all": false,
70
+ "show_user_warnings": false,
71
+ "s3_save_dir": "",
72
+ "dist_url": "env://",
73
+ "log_wandb": true,
74
+ "wandb_project": "loom",
75
+ "wandb_entity": null,
76
+ "wandb_run_name": "t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0",
77
+ "wandb_group": "cc-solo",
78
+ "dim": null,
79
+ "depth": null,
80
+ "num_heads": null,
81
+ "mlp_ratio": null,
82
+ "num_experts": 1,
83
+ "granularity": 1,
84
+ "routing": null,
85
+ "moe_aux_coef": 0.01,
86
+ "z_loss_coef": null,
87
+ "moe_compile": false,
88
+ "log_grad_norms_every": 0,
89
+ "save_at_tokens": null,
90
+ "unigram_entropies": null,
91
+ "cooldown_branch": false,
92
+ "cooldown_tokens": 0,
93
+ "config_path": "cfgs/default/4m/experiments/confound/solo_s/image/t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0.yaml",
94
+ "rank": 0,
95
+ "world_size": 1,
96
+ "gpu": 0,
97
+ "distributed": true,
98
+ "dist_backend": "nccl",
99
+ "num_tasks": 1,
100
+ "sampling_mode": "without_replacement",
101
+ "max_micro_steps": 311280,
102
+ "lossbearing_target": 10200000000.0,
103
+ "lossbearing_mode": true,
104
+ "lr": 0.003,
105
+ "min_lr": 0.00015,
106
+ "global_micro_step": 311280,
107
+ "cum_lossbearing": 10201453243.0,
108
+ "cum_mod_tokens": {
109
+ "tok_rgb@384": 10166558822,
110
+ "text": 34894421
111
+ }
112
+ },
113
+ "source_configs": {
114
+ "cfgs/default/4m/experiments/confound/solo_s/image/t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0.yaml": "# S solo variant t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0 (confound program D6-1).\n# Clone of t1/unimodal/image/run_pure.yaml: D 10.2B, effective batch 1024, absolute LR 0.003 (lr_batch_scaling none), seed 0.\n# Self-cooled WSD endpoint; compared with the other D6-1 solo variants of the same modality\n# and D (seed spread, LR x batch grid, warmup / z-loss controls).\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_image_s_d10.2_eb1024_lr3e-3_seed0\ntotal_tokens: -1\nlossbearing_tokens: 10.2\ndata_config: cfgs/default/4m/experiments/t1/unimodal/image/data_train_pure.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/image/data_val_pure.yaml\nwarmup_steps: 250\neval_freq_tokens: 2\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-solo\neffective_batch_size: 1024\nlr_batch_scaling: none\nblr: 0.003\nseed: 0\naccum_iter: 32\nmin_blr: 0.00015\n",
115
+ "cfgs/default/4m/experiments/t1/unimodal/image/data_train_pure.yaml": "# T1 pure unimodal training data \u2014 rgb\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_rgb@384\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: coyo\n shards: data/shards/coyo_shuffled/train/shard_{000000..022189}.tar\n weight: 22189292\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: cc12m\n shards: data/shards/cc12m_shuffled/train/shard_{000000..012527}.tar\n weight: 12527184\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: obelics\n shards: data/shards/obelics_shuffled/train/shard_{000000..009245}.tar\n weight: 9245143\n modality_tokens:\n tok_rgb@384: 751\n format: npz\n- name: commoncatalog\n shards: data/shards/commoncatalog_reshuffled/train/shard_*.tar\n weight: 71320000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\nsampling_mode: without_replacement\n",
116
+ "cfgs/default/4m/experiments/t1/unimodal/image/data_val_pure.yaml": "# T1 pure unimodal validation data \u2014 rgb\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_rgb@384\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: coyo_val\n shards: data/shards/coyo_shuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: cc12m_val\n shards: data/shards/cc12m_shuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n- name: obelics_val\n shards: data/shards/obelics_shuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 751\n format: npz\n- name: commoncatalog_val\n shards: data/shards/commoncatalog_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_rgb@384: 576\n format: npz\n"
117
+ }
118
+ }