ospanbatyr commited on
Commit
b0967e8
·
verified ·
1 Parent(s): b00219c

Add 8 model file(s)

Browse files
t1_pure_video_s_d4_eb1024_lr1e-3_wr_seed1/checkpoint-2Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83b977b430cb32ef882957aabe07499e260d2e2434702b704521dcffef886042
3
+ size 2541167205
t1_pure_video_s_d4_eb1024_lr1e-3_wr_seed1/checkpoint-3.5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0e92c1f5243109965f44527d83d3c9a203316e806e038b79ac24984fddd27d9
3
+ size 2541168251
t1_pure_video_s_d4_eb1024_lr1e-3_wr_seed1/checkpoint-4Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c73604f69f80048c9f724e755fd9a61ff8eccdd5a229a98a2a50ccb461005cc5
3
+ size 2541167205
t1_pure_video_s_d4_eb1024_lr1e-3_wr_seed1/checkpoint-final.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd81c9fb1f27c1cd0704b322cc9ece2301c60c1f09a189a0b60662f1afdd2a09
3
+ size 2541167728
t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0/checkpoint-2Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:432713470fa2fe189f6cf8d71de9ea97e98db5af63f1aa95bdf82177b542a207
3
+ size 2541167205
t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0/checkpoint-3.5Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab8258bca96663899e6c4af6e93ecd4144434be81480a9d04740ef7b32213a72
3
+ size 2541168251
t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0/checkpoint-4Blb.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab824d5547b538bd9715e42f34f50c1507330146dc0bc29b01f643815b230de2
3
+ size 2541167205
t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0/config.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "run_name": "t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0",
3
+ "epoch": 0,
4
+ "n_params": 215985414,
5
+ "args": {
6
+ "run_name": "t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0",
7
+ "batch_size": 32,
8
+ "epochs": 1,
9
+ "total_tokens": -1,
10
+ "lossbearing_tokens": 4,
11
+ "accum_iter": 32,
12
+ "effective_batch_size": 1024,
13
+ "lr_batch_scaling": "none",
14
+ "save_ckpt_freq": 1,
15
+ "save_ckpt_freq_tokens": 2,
16
+ "debug_batch_stats": 0,
17
+ "model": "fm_small_12d_swiglu_nobias",
18
+ "dtype": "bfloat16",
19
+ "loss_type": "token",
20
+ "context_length": 2048,
21
+ "pos_emb_ablation": "vanilla",
22
+ "max_blocks": 32,
23
+ "intra_pos_emb": "default",
24
+ "video_pos_3d": false,
25
+ "finetune": "",
26
+ "opt": "adamw",
27
+ "opt_eps": 1e-08,
28
+ "opt_betas": [
29
+ 0.9,
30
+ 0.95
31
+ ],
32
+ "compute_grad_norm": true,
33
+ "clip_grad": 1.0,
34
+ "skip_grad": null,
35
+ "momentum": 0.9,
36
+ "weight_decay": 0.1,
37
+ "weight_decay_end": 0.1,
38
+ "blr": 0.001,
39
+ "min_blr": 5e-05,
40
+ "scheduler": "wsd",
41
+ "decay_fraction": 0.1,
42
+ "warmup_steps": 1000,
43
+ "cooldown_steps": -1,
44
+ "data_config": "cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml",
45
+ "data_config_val": "cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml",
46
+ "cross_eval_val": "",
47
+ "s3_endpoint": "",
48
+ "s3_data_endpoint": "",
49
+ "s3_multipart_chunksize_mb": 512,
50
+ "s3_multipart_threshold_mb": 512,
51
+ "s3_max_io_queue": 100,
52
+ "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
53
+ "eval_freq": 1,
54
+ "eval_freq_tokens": 0.5,
55
+ "dist_eval": true,
56
+ "eval": false,
57
+ "val_sampling_mode": "without_replacement",
58
+ "output_dir": "output/cc/t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0",
59
+ "device": "cuda",
60
+ "seed": 0,
61
+ "resume": "",
62
+ "auto_resume": true,
63
+ "start_epoch": 0,
64
+ "num_workers": 10,
65
+ "pin_mem": true,
66
+ "find_unused_params": true,
67
+ "rlimit": 4096,
68
+ "print_all": false,
69
+ "show_user_warnings": false,
70
+ "s3_save_dir": "",
71
+ "dist_url": "env://",
72
+ "log_wandb": true,
73
+ "wandb_project": "loom",
74
+ "wandb_entity": null,
75
+ "wandb_run_name": "t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0",
76
+ "wandb_group": "cc-screen",
77
+ "dim": null,
78
+ "depth": null,
79
+ "num_heads": null,
80
+ "mlp_ratio": null,
81
+ "num_experts": 1,
82
+ "granularity": 1,
83
+ "routing": null,
84
+ "moe_aux_coef": 0.01,
85
+ "z_loss_coef": null,
86
+ "moe_compile": false,
87
+ "log_grad_norms_every": 0,
88
+ "save_at_tokens": "3.5",
89
+ "unigram_entropies": null,
90
+ "cooldown_branch": false,
91
+ "cooldown_tokens": 0,
92
+ "config_path": "cfgs/default/4m/experiments/confound/screen/video/t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0.yaml",
93
+ "rank": 0,
94
+ "world_size": 1,
95
+ "gpu": 0,
96
+ "distributed": true,
97
+ "dist_backend": "nccl",
98
+ "num_tasks": 1,
99
+ "sampling_mode": "without_replacement",
100
+ "max_micro_steps": 122071,
101
+ "lossbearing_target": 4000000000.0,
102
+ "lossbearing_mode": true,
103
+ "lr": 0.001,
104
+ "min_lr": 5e-05,
105
+ "global_micro_step": 122071,
106
+ "cum_lossbearing": 4000342090.0,
107
+ "cum_mod_tokens": {
108
+ "text": 11092188,
109
+ "tok_video@128": 3989249902
110
+ }
111
+ },
112
+ "source_configs": {
113
+ "cfgs/default/4m/experiments/confound/screen/video/t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0.yaml": "# STABILITY SCREEN t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0 (confound program stage 1 \u2014 D6-2 \u00a72 finding 3).\n# Clone of t1/unimodal/video/run_pure.yaml to 4B at batch 1024, LR 1e-3, seed 0: warmup 1000 steps, no z-loss, without-replacement shards.\n# Read: the hot evals every 0.5B against the stage-0 10.2B seeds (warmup 250, no z-loss,\n# without replacement; same schedule until 9.2B). The variant that collapses the seed\n# spread gates every later text run. checkpoint-3.5Blb precedes the decay: copy it\n# into a new output dir and --auto_resume with a 10.2B config to continue the run.\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t1_pure_video_s_d4_eb1024_lr1e-3_wu1000_seed0\ntotal_tokens: -1\nlossbearing_tokens: 4\ndata_config: cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml\ndata_config_val: cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml\nwarmup_steps: 1000\neval_freq_tokens: 0.5\nsave_ckpt_freq_tokens: 2\nsave_at_tokens: '3.5'\nwandb_group: cc-screen\neffective_batch_size: 1024\nlr_batch_scaling: none\nblr: 0.001\nseed: 0\naccum_iter: 32\nmin_blr: 5.0e-05\n",
114
+ "cfgs/default/4m/experiments/t1/unimodal/video/data_train_pure.yaml": "# T1 pure unimodal training data \u2014 video\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_video@128\nepoch_size: 50000000\nsampler_type: temperature\ntemperature_tau: 1.0\nsources:\n- name: openvid\n shards: data/shards/openvid_shuffled/train/shard_{000000..001009}.tar\n weight: 1010000\n modality_tokens:\n tok_video@128: 1600\n format: npz\n- name: webvid\n shards: data/shards/webvid_reshuffled/train/shard_*.tar\n weight: 9305227\n modality_tokens:\n tok_video@128: 694\n format: npz\nsampling_mode: without_replacement\n",
115
+ "cfgs/default/4m/experiments/t1/unimodal/video/data_val_pure.yaml": "# T1 pure unimodal validation data \u2014 video\n# Text stripped from all sources (pure modality-only condition)\n\nmodalities:\n- tok_video@128\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: openvid_val\n shards: data/shards/openvid_shuffled/val/shard_{000000..000014}.tar\n weight: 14500\n modality_tokens:\n tok_video@128: 1600\n format: npz\n- name: webvid_val\n shards: data/shards/webvid_reshuffled/val/shard_{000000..000049}.tar\n weight: 50000\n modality_tokens:\n tok_video@128: 694\n format: npz\n"
116
+ }
117
+ }