Hugging Face's logo

ospanbatyr
/
kx

Model card Files
xet
 Community
ospanbatyr commited on
Commit
6a5a2ae
·
verified ·
1 Parent(s): 775527c

Add 9 model file(s)

Browse files
Files changed (9) hide show
  1. t1_pure_video_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-5Blb.pth +3 -0
  2. t1_pure_video_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-final.pth +3 -0
  3. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/branches/branch_5Blb/checkpoint-final.pth +3 -0
  4. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-10Blb.pth +3 -0
  5. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-3.5Blb.pth +3 -0
  6. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-5Blb.pth +3 -0
  7. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-final.pth +3 -0
  8. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/config.json +117 -0
  9. t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1/config.json +117 -0
t1_pure_video_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b9e28617d78f2505b3ebf4403a350401b3e2b3bc40215085e38a8cee78ed833
3
+ size 2541167205
t1_pure_video_s_d10.2_eb1024_lr1e-3_seed2/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2ee5a6e29ecbb7d40c64849477280f3cd6abf153a94c24f7f3f25771f5e84eb
3
+ size 2541167728
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/branches/branch_5Blb/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f31e9b2679c5ec71dbfc164ceb135a4fdfa68f78867d90d2dfebad5a0144a42c
3
+ size 2541168048
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-10Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd17430de18757f602cdb13d37bcde5e3c6dd8df19616a4ff312d464ee8cb603
3
+ size 2541167856
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-3.5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab8258bca96663899e6c4af6e93ecd4144434be81480a9d04740ef7b32213a72
3
+ size 2541168251
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca9304664ea70030ec2076c458379c95320a70718811fb9e0a4a348a6de586cb
3
+ size 2541167333
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6ae1ca7eb6f4f2132338289c9da793e84607738fb88e16e6ea8a518c7553ac7
3
+ size 2541167856
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/config.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0",
3
+ "epoch": 0,
4
+ "n_params": 215985414,
5
+ "args": {
6
+ "run_name": "t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 10.2,
11
+ "accum_iter": 32,
12
+ "effective_batch_size": 1024,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.001,
39
+ "min_blr": 5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "warmup_steps": 1000,
43
+ "cooldown_steps": -1,
44
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml",
45
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml",
46
+ "cross_eval_val": "",
47
+ "s3_endpoint": "",
48
+ "s3_data_endpoint": "",
49
+ "s3_multipart_chunksize_mb": 512,
50
+ "s3_multipart_threshold_mb": 512,
51
+ "s3_max_io_queue": 100,
52
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
53
+ "eval_freq": 1,
54
+ "eval_freq_tokens": 2,
55
+ "dist_eval": true,
56
+ "eval": false,
57
+ "val_sampling_mode": "without_replacement",
58
+ "output_dir": "output/cc/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0",
59
+ "device": "cuda",
60
+ "seed": 0,
61
+ "resume": "output/cc/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0/checkpoint-3.5Blb.pth",
62
+ "auto_resume": true,
63
+ "start_epoch": 0,
64
+ "num_workers": 10,
65
+ "pin_mem": true,
66
+ "find_unused_params": true,
67
+ "rlimit": 4096,
68
+ "print_all": false,
69
+ "show_user_warnings": false,
70
+ "s3_save_dir": "",
71
+ "dist_url": "env://",
72
+ "log_wandb": true,
73
+ "wandb_project": "loom",
74
+ "wandb_entity": null,
75
+ "wandb_run_name": "t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0",
76
+ "wandb_group": "cc-solo",
77
+ "dim": null,
78
+ "depth": null,
79
+ "num_heads": null,
80
+ "mlp_ratio": null,
81
+ "num_experts": 1,
82
+ "granularity": 1,
83
+ "routing": null,
84
+ "moe_aux_coef": 0.01,
85
+ "z_loss_coef": null,
86
+ "moe_compile": false,
87
+ "log_grad_norms_every": 0,
88
+ "save_at_tokens": null,
89
+ "unigram_entropies": null,
90
+ "cooldown_branch": false,
91
+ "cooldown_tokens": 0,
92
+ "config_path": "cfgs/default/4m/experiments/confound/solo_s/video/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0.yaml",
93
+ "rank": 0,
94
+ "world_size": 1,
95
+ "gpu": 0,
96
+ "distributed": true,
97
+ "dist_backend": "nccl",
98
+ "num_tasks": 1,
99
+ "sampling_mode": "without_replacement",
100
+ "max_micro_steps": 311280,
101
+ "lossbearing_target": 10200000000.0,
102
+ "lossbearing_mode": true,
103
+ "lr": 0.001,
104
+ "min_lr": 5e-05,
105
+ "global_micro_step": 311280,
106
+ "cum_lossbearing": 10200756386.0,
107
+ "cum_mod_tokens": {
108
+ "text": 28283500,
109
+ "tok_video@128": 10172472886
110
+ }
111
+ },
112
+ "source_configs": {
113
+ "cfgs/default/4m/experiments/confound/solo_s/video/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0.yaml": "# S solo variant t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0 (confound program D6-1).\n# Clone of t1/unimodal/video/run_pure.yaml: D 10.2B, effective batch 1024, absolute LR 0.001 (lr_batch_scaling none), seed 0, warmup 1000 optimiser steps (the stability screen's warmup).\n# Self-cooled WSD endpoint; compared with the other D6-1 solo variants of the same modality\n# and D (seed spread, LR x batch grid, warmup / z-loss controls).\n# CONTINUES the screen run t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0 from its checkpoint-3.5Blb (peak-LR phase; same\n# seed, data order, batch and warmup). Before the first submit: python scripts/seed_continuation.py\n# --src-config cfgs/default/4m/experiments/confound/screen/video/t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0.yaml --dst-config cfgs/default/4m/experiments/confound/solo_s/video/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0.yaml\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed0\ntotal_tokens: -1\nlossbearing_tokens: 10.2\ndata_config: cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml\nwarmup_steps: 1000\neval_freq_tokens: 2\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-solo\neffective_batch_size: 1024\nlr_batch_scaling: none\nblr: 0.001\nseed: 0\naccum_iter: 32\nmin_blr: 5.0e-05\n",
114
+ "cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml": "# T1 pure unimodal training data \u2014 video\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_video@128\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: openvid\n shards: data/shards/openvid_shuffled/train/shard_{000000..001009}.tar\n weight: 1010000\n modality_tokens:\n tok_video@128: 1600\n format: npz\n- name: webvid\n shards: data/shards/webvid_reshuffled/train/shard_*.tar\n weight: 9305227\n modality_tokens:\n tok_video@128: 694\n format: npz\nsampling_mode: without_replacement\n",
115
+ "cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml": "# T1 pure unimodal validation data \u2014 video\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_video@128\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: openvid_val\n shards: data/shards/openvid_shuffled/val/shard_{000000..000014}.tar\n weight: 14500\n modality_tokens:\n tok_video@128: 1600\n format: npz\n- name: webvid_val\n shards: data/shards/webvid_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_video@128: 694\n format: npz\n"
116
+ }
117
+ }
t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1/config.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1",
3
+ "epoch": 0,
4
+ "n_params": 215985414,
5
+ "args": {
6
+ "run_name": "t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 10.2,
11
+ "accum_iter": 32,
12
+ "effective_batch_size": 1024,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 5,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.001,
39
+ "min_blr": 5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "warmup_steps": 1000,
43
+ "cooldown_steps": -1,
44
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml",
45
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml",
46
+ "cross_eval_val": "",
47
+ "s3_endpoint": "",
48
+ "s3_data_endpoint": "",
49
+ "s3_multipart_chunksize_mb": 512,
50
+ "s3_multipart_threshold_mb": 512,
51
+ "s3_max_io_queue": 100,
52
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
53
+ "eval_freq": 1,
54
+ "eval_freq_tokens": 2,
55
+ "dist_eval": true,
56
+ "eval": false,
57
+ "val_sampling_mode": "without_replacement",
58
+ "output_dir": "output/cc/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1",
59
+ "device": "cuda",
60
+ "seed": 1,
61
+ "resume": "output/cc/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1/checkpoint-3.5Blb.pth",
62
+ "auto_resume": true,
63
+ "start_epoch": 0,
64
+ "num_workers": 10,
65
+ "pin_mem": true,
66
+ "find_unused_params": true,
67
+ "rlimit": 4096,
68
+ "print_all": false,
69
+ "show_user_warnings": false,
70
+ "s3_save_dir": "",
71
+ "dist_url": "env://",
72
+ "log_wandb": true,
73
+ "wandb_project": "loom",
74
+ "wandb_entity": null,
75
+ "wandb_run_name": "t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1",
76
+ "wandb_group": "cc-solo",
77
+ "dim": null,
78
+ "depth": null,
79
+ "num_heads": null,
80
+ "mlp_ratio": null,
81
+ "num_experts": 1,
82
+ "granularity": 1,
83
+ "routing": null,
84
+ "moe_aux_coef": 0.01,
85
+ "z_loss_coef": null,
86
+ "moe_compile": false,
87
+ "log_grad_norms_every": 0,
88
+ "save_at_tokens": null,
89
+ "unigram_entropies": null,
90
+ "cooldown_branch": false,
91
+ "cooldown_tokens": 0,
92
+ "config_path": "cfgs/default/4m/experiments/confound/solo_s/video/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1.yaml",
93
+ "rank": 0,
94
+ "world_size": 1,
95
+ "gpu": 0,
96
+ "distributed": true,
97
+ "dist_backend": "nccl",
98
+ "num_tasks": 1,
99
+ "sampling_mode": "without_replacement",
100
+ "max_micro_steps": 311280,
101
+ "lossbearing_target": 10200000000.0,
102
+ "lossbearing_mode": true,
103
+ "lr": 0.001,
104
+ "min_lr": 5e-05,
105
+ "global_micro_step": 311280,
106
+ "cum_lossbearing": 10200930889.0,
107
+ "cum_mod_tokens": {
108
+ "text": 28284806,
109
+ "tok_video@128": 10172646083
110
+ }
111
+ },
112
+ "source_configs": {
113
+ "cfgs/default/4m/experiments/confound/solo_s/video/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1.yaml": "# S solo variant t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1 (confound program D6-1).\n# Clone of t1/unimodal/video/run_pure.yaml: D 10.2B, effective batch 1024, absolute LR 0.001 (lr_batch_scaling none), seed 1, warmup 1000 optimiser steps (the stability screen's warmup).\n# Self-cooled WSD endpoint; compared with the other D6-1 solo variants of the same modality\n# and D (seed spread, LR x batch grid, warmup / z-loss controls).\n# CONTINUES the screen run t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed1 from its checkpoint-3.5Blb (peak-LR phase; same\n# seed, data order, batch and warmup). Before the first submit: python scripts/seed_continuation.py\n# --src-config cfgs/default/4m/experiments/confound/screen/video/t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed1.yaml --dst-config cfgs/default/4m/experiments/confound/solo_s/video/t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1.yaml\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_video_s_d10.2_eb1024_lr1e-3_wu1000_seed1\ntotal_tokens: -1\nlossbearing_tokens: 10.2\ndata_config: cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml\nwarmup_steps: 1000\neval_freq_tokens: 2\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-solo\neffective_batch_size: 1024\nlr_batch_scaling: none\nblr: 0.001\nseed: 1\naccum_iter: 32\nmin_blr: 5.0e-05\n",
114
+ "cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml": "# T1 pure unimodal training data \u2014 video\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_video@128\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: openvid\n shards: data/shards/openvid_shuffled/train/shard_{000000..001009}.tar\n weight: 1010000\n modality_tokens:\n tok_video@128: 1600\n format: npz\n- name: webvid\n shards: data/shards/webvid_reshuffled/train/shard_*.tar\n weight: 9305227\n modality_tokens:\n tok_video@128: 694\n format: npz\nsampling_mode: without_replacement\n",
115
+ "cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml": "# T1 pure unimodal validation data \u2014 video\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_video@128\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: openvid_val\n shards: data/shards/openvid_shuffled/val/shard_{000000..000014}.tar\n weight: 14500\n modality_tokens:\n tok_video@128: 1600\n format: npz\n- name: webvid_val\n shards: data/shards/webvid_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_video@128: 694\n format: npz\n"
116
+ }
117
+ }