Add 10 model file(s)

t1_pure_video_s_d10.2_eb1024_lr1e-3_clip2_seed0/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2370b372ee4e1abbaef86f5f551cf805f45c445e3da1fc8160e149dfe82cd87d
+size 2541167792

t2_text_timeseries_s_eb2048/checkpoint-10Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2673814c66803bfd95a30185d1b37f462d3aa5250e78f44b32267eea1091e2fe
+size 1948049200

t2_text_timeseries_s_eb2048/checkpoint-5Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf795c23ea6c06438f37f258015d41ae275b97c7f23da2b0bf13f1761755e1a2
+size 1948048677

t2_text_timeseries_s_eb2048/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a45333ef2cb87668d3b58489d2e45495810690e2f68d01edc5e05fed209aa0f1
+size 1948049200

t2_text_timeseries_s_eb2048/config.json

@@ -0,0 +1,118 @@
+{
+  "run_name": "t2_text_timeseries_s_eb2048",
+  "epoch": 0,
+  "n_params": 165527360,
+  "args": {
+    "run_name": "t2_text_timeseries_s_eb2048",
+    "batch_size": 32,
+    "epochs": 1,
+    "total_tokens": -1,
+    "lossbearing_tokens": 13.04,
+    "accum_iter": 64,
+    "effective_batch_size": 2048,
+    "lr_batch_scaling": "none",
+    "save_ckpt_freq": 1,
+    "save_ckpt_freq_tokens": 5,
+    "debug_batch_stats": 0,
+    "model": "fm_small_12d_swiglu_nobias",
+    "dtype": "bfloat16",
+    "loss_type": "token",
+    "context_length": 2048,
+    "pos_emb_ablation": "vanilla",
+    "max_blocks": 32,
+    "intra_pos_emb": "default",
+    "video_pos_3d": false,
+    "finetune": "",
+    "opt": "adamw",
+    "opt_eps": 1e-08,
+    "opt_betas": [
+      0.9,
+      0.95
+    ],
+    "compute_grad_norm": true,
+    "clip_grad": 1.0,
+    "skip_grad": null,
+    "momentum": 0.9,
+    "weight_decay": 0.1,
+    "weight_decay_end": 0.1,
+    "blr": 0.001,
+    "min_blr": 5e-05,
+    "scheduler": "wsd",
+    "decay_fraction": 0.1,
+    "lr_shape": "wsd",
+    "warmup_steps": 250,
+    "cooldown_steps": -1,
+    "data_config": "cfgs/default/4m/experiments/t2/text_timeseries/data_train.yaml",
+    "data_config_val": "cfgs/default/4m/experiments/t2/text_timeseries/data_val.yaml",
+    "cross_eval_val": "",
+    "s3_endpoint": "",
+    "s3_data_endpoint": "",
+    "s3_multipart_chunksize_mb": 512,
+    "s3_multipart_threshold_mb": 512,
+    "s3_max_io_queue": 100,
+    "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
+    "eval_freq": 1,
+    "eval_freq_tokens": 3,
+    "dist_eval": true,
+    "eval": false,
+    "val_sampling_mode": "without_replacement",
+    "output_dir": "output/cc/t2_text_timeseries_s_eb2048",
+    "device": "cuda",
+    "seed": 0,
+    "resume": "",
+    "auto_resume": true,
+    "start_epoch": 0,
+    "num_workers": 10,
+    "pin_mem": true,
+    "find_unused_params": true,
+    "rlimit": 4096,
+    "print_all": false,
+    "show_user_warnings": false,
+    "s3_save_dir": "",
+    "dist_url": "env://",
+    "log_wandb": true,
+    "wandb_project": "loom",
+    "wandb_entity": null,
+    "wandb_run_name": "t2_text_timeseries_s_eb2048",
+    "wandb_group": "cc-pairs",
+    "dim": null,
+    "depth": null,
+    "num_heads": null,
+    "mlp_ratio": null,
+    "num_experts": 1,
+    "granularity": 1,
+    "routing": null,
+    "moe_aux_coef": 0.01,
+    "z_loss_coef": null,
+    "moe_compile": false,
+    "log_grad_norms_every": 0,
+    "save_at_tokens": null,
+    "unigram_entropies": null,
+    "cooldown_branch": false,
+    "cooldown_tokens": 0,
+    "config_path": "cfgs/default/4m/experiments/confound/pairs/text_timeseries/run_eb2048.yaml",
+    "rank": 0,
+    "world_size": 1,
+    "gpu": 0,
+    "distributed": true,
+    "dist_backend": "nccl",
+    "num_tasks": 1,
+    "sampling_mode": "without_replacement",
+    "max_micro_steps": 397950,
+    "lossbearing_target": 13040000000.0,
+    "lossbearing_mode": true,
+    "lr": 0.001,
+    "min_lr": 5e-05,
+    "global_micro_step": 397950,
+    "cum_lossbearing": 13043952265.0,
+    "cum_mod_tokens": {
+      "text": 6579918465,
+      "tok_timeseries": 6464033800
+    }
+  },
+  "source_configs": {
+    "cfgs/default/4m/experiments/confound/pairs/text_timeseries/run_eb2048.yaml": "# STEP-MATCHED PAIR text_timeseries (confound program D6-1, scripts/gen_confound_configs.py).\n# Clone of experiments/t2/text_timeseries/run.yaml with the effective batch 2048 (not 1024)\n# and LR held at 0.001 (lr_batch_scaling none): each modality gets 1024 of its own\n# sequences per step, the same optimiser steps and the same LR as its SOLO run. Same\n# per-modality tokens as the original pair. Compare each recipient with its solo loss at\n# the same tokens (cooled solo surface / D6-1 solo baselines): the difference is the\n# partner's effect C with no schedule difference.\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t2_text_timeseries_s_eb2048\ntotal_tokens: -1\nlossbearing_tokens: 13.04\ndata_config: cfgs/default/4m/experiments/t2/text_timeseries/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t2/text_timeseries/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 3\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-pairs\neffective_batch_size: 2048\nlr_batch_scaling: none\nblr: 0.001\nseed: 0\naccum_iter: 64\nmin_blr: 5.0e-05\n",
+    "cfgs/default/4m/experiments/t2/text_timeseries/data_train.yaml": "# AUTO-GENERATED by scripts/gen_matrix_configs.py \u2014 DO NOT EDIT BY HAND.\n# Stage 2 bimodal transfer-matrix cell: text x timeseries (S-scale).\n# Rationale + token accounting: decisions/D2-1_bimodal_matrix_design.md\n\nmodalities:\n- text\n- tok_timeseries\nepoch_size: 50000000\nsampler_type: forced_equal\ntarget_modalities:\n- text\n- tok_timeseries\nreverse_block_probability: 0.5\nsampling_mode: without_replacement\nsources:\n- name: dclm\n  shards: data/shards/dclm_reshuffled/train/shard_*.tar\n  weight: 426009612\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: utsd\n  shards: data/shards/utsd_shuffled/train/shard_{000000..002934}.tar\n  weight: 2934667\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: forecasting\n  shards: data/shards/forecasting_shuffled/train/shard_{000000..000098}.tar\n  weight: 98423\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: timeseries\n  shards: data/shards/timeseries_shuffled/train/shard_{000000..000109}.tar\n  weight: 109190\n  modality_tokens:\n    tok_timeseries: 14\n  format: npz\n- name: timeseries_pile\n  shards: data/shards/timeseries_pile_shuffled/train/shard_{000000..005159}.tar\n  weight: 5159748\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: time300b\n  shards: data/shards/time300b_shuffled/train/shard_{000000..017340}.tar\n  weight: 17340008\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n",
+    "cfgs/default/4m/experiments/t2/text_timeseries/data_val.yaml": "# AUTO-GENERATED by scripts/gen_matrix_configs.py \u2014 DO NOT EDIT BY HAND.\n# Stage 2 bimodal transfer-matrix cell: text x timeseries (S-scale).\n# Rationale + token accounting: decisions/D2-1_bimodal_matrix_design.md\n\nmodalities:\n- text\n- tok_timeseries\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: dclm_val\n  shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: utsd_val\n  shards: data/shards/utsd_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: forecasting_val\n  shards: data/shards/forecasting_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: timeseries_val\n  shards: data/shards/timeseries_shuffled/val/shard_{000000..000002}.tar\n  weight: 2200\n  modality_tokens:\n    tok_timeseries: 14\n  format: npz\n- name: timeseries_pile_val\n  shards: data/shards/timeseries_pile_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: time300b_val\n  shards: data/shards/time300b_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n"
+  }
+}

t2_text_video_s_eb2048/checkpoint-10Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dc3085662943dc933c870e6c75ca6120b10a9ddec298815216636b79321fa26
+size 2541167664

t2_text_video_s_eb2048/checkpoint-15Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46978622bc80b9f84a662a9b7d2f3bf3c7ca61abb316ad472bb891071f7aff60
+size 2541167664

t2_text_video_s_eb2048/checkpoint-20Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ba031fe650bf1b9f2b290813463676b999d271bafc0685835024207fc393a00
+size 2541167664

t2_text_video_s_eb2048/checkpoint-5Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5681b381fb83ac65737234b2af0848c0caaf6b220102dff5213a9f42e693600a
+size 2541167141

t2_text_video_s_eb2048/config.json

@@ -0,0 +1,118 @@
+{
+  "run_name": "t2_text_video_s_eb2048",
+  "epoch": 0,
+  "n_params": 215985414,
+  "args": {
+    "run_name": "t2_text_video_s_eb2048",
+    "batch_size": 32,
+    "epochs": 1,
+    "total_tokens": -1,
+    "lossbearing_tokens": 20.0,
+    "accum_iter": 64,
+    "effective_batch_size": 2048,
+    "lr_batch_scaling": "none",
+    "save_ckpt_freq": 1,
+    "save_ckpt_freq_tokens": 5,
+    "debug_batch_stats": 0,
+    "model": "fm_small_12d_swiglu_nobias",
+    "dtype": "bfloat16",
+    "loss_type": "token",
+    "context_length": 2048,
+    "pos_emb_ablation": "vanilla",
+    "max_blocks": 32,
+    "intra_pos_emb": "default",
+    "video_pos_3d": false,
+    "finetune": "",
+    "opt": "adamw",
+    "opt_eps": 1e-08,
+    "opt_betas": [
+      0.9,
+      0.95
+    ],
+    "compute_grad_norm": true,
+    "clip_grad": 1.0,
+    "skip_grad": null,
+    "momentum": 0.9,
+    "weight_decay": 0.1,
+    "weight_decay_end": 0.1,
+    "blr": 0.001,
+    "min_blr": 5e-05,
+    "scheduler": "wsd",
+    "decay_fraction": 0.1,
+    "lr_shape": "wsd",
+    "warmup_steps": 250,
+    "cooldown_steps": -1,
+    "data_config": "cfgs/default/4m/experiments/t2/text_video/data_train.yaml",
+    "data_config_val": "cfgs/default/4m/experiments/t2/text_video/data_val.yaml",
+    "cross_eval_val": "",
+    "s3_endpoint": "",
+    "s3_data_endpoint": "",
+    "s3_multipart_chunksize_mb": 512,
+    "s3_multipart_threshold_mb": 512,
+    "s3_max_io_queue": 100,
+    "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
+    "eval_freq": 1,
+    "eval_freq_tokens": 4,
+    "dist_eval": true,
+    "eval": false,
+    "val_sampling_mode": "without_replacement",
+    "output_dir": "output/cc/t2_text_video_s_eb2048",
+    "device": "cuda",
+    "seed": 0,
+    "resume": "",
+    "auto_resume": true,
+    "start_epoch": 0,
+    "num_workers": 10,
+    "pin_mem": true,
+    "find_unused_params": true,
+    "rlimit": 4096,
+    "print_all": false,
+    "show_user_warnings": false,
+    "s3_save_dir": "",
+    "dist_url": "env://",
+    "log_wandb": true,
+    "wandb_project": "loom",
+    "wandb_entity": null,
+    "wandb_run_name": "t2_text_video_s_eb2048",
+    "wandb_group": "cc-pairs",
+    "dim": null,
+    "depth": null,
+    "num_heads": null,
+    "mlp_ratio": null,
+    "num_experts": 1,
+    "granularity": 1,
+    "routing": null,
+    "moe_aux_coef": 0.01,
+    "z_loss_coef": null,
+    "moe_compile": false,
+    "log_grad_norms_every": 0,
+    "save_at_tokens": null,
+    "unigram_entropies": null,
+    "cooldown_branch": false,
+    "cooldown_tokens": 0,
+    "config_path": "cfgs/default/4m/experiments/confound/pairs/text_video/run_eb2048.yaml",
+    "rank": 0,
+    "world_size": 1,
+    "gpu": 0,
+    "distributed": true,
+    "dist_backend": "nccl",
+    "num_tasks": 1,
+    "sampling_mode": "without_replacement",
+    "max_micro_steps": 610352,
+    "lossbearing_target": 20000000000.0,
+    "lossbearing_mode": true,
+    "lr": 0.001,
+    "min_lr": 5e-05,
+    "global_micro_step": 610352,
+    "cum_lossbearing": 20002047610.0,
+    "cum_mod_tokens": {
+      "text": 10018547360,
+      "tok_video@128": 9983500250
+    }
+  },
+  "source_configs": {
+    "cfgs/default/4m/experiments/confound/pairs/text_video/run_eb2048.yaml": "# STEP-MATCHED PAIR text_video (confound program D6-1, scripts/gen_confound_configs.py).\n# Clone of experiments/t2/text_video/run.yaml with the effective batch 2048 (not 1024)\n# and LR held at 0.001 (lr_batch_scaling none): each modality gets 1024 of its own\n# sequences per step, the same optimiser steps and the same LR as its SOLO run. Same\n# per-modality tokens as the original pair. Compare each recipient with its solo loss at\n# the same tokens (cooled solo surface / D6-1 solo baselines): the difference is the\n# partner's effect C with no schedule difference.\n_base_: ../../../../models/dec/4m-s-dec.yaml\nrun_name: t2_text_video_s_eb2048\ntotal_tokens: -1\nlossbearing_tokens: 20.0\ndata_config: cfgs/default/4m/experiments/t2/text_video/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t2/text_video/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 4\nsave_ckpt_freq_tokens: 5\nwandb_group: cc-pairs\neffective_batch_size: 2048\nlr_batch_scaling: none\nblr: 0.001\nseed: 0\naccum_iter: 64\nmin_blr: 5.0e-05\n",
+    "cfgs/default/4m/experiments/t2/text_video/data_train.yaml": "# AUTO-GENERATED by scripts/gen_matrix_configs.py \u2014 DO NOT EDIT BY HAND.\n# Stage 2 bimodal transfer-matrix cell: text x video (S-scale).\n# Rationale + token accounting: decisions/D2-1_bimodal_matrix_design.md\n\nmodalities:\n- text\n- tok_video@128\nepoch_size: 50000000\nsampler_type: forced_equal\ntarget_modalities:\n- text\n- tok_video@128\nreverse_block_probability: 0.5\nsampling_mode: without_replacement\nsources:\n- name: dclm\n  shards: data/shards/dclm_reshuffled/train/shard_*.tar\n  weight: 426009612\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: openvid\n  shards: data/shards/openvid_shuffled/train/shard_{000000..001009}.tar\n  weight: 1010000\n  modality_tokens:\n    tok_video@128: 950\n  format: npz\n- name: webvid\n  shards: data/shards/webvid_reshuffled/train/shard_*.tar\n  weight: 9305227\n  modality_tokens:\n    tok_video@128: 693\n  format: npz\n",
+    "cfgs/default/4m/experiments/t2/text_video/data_val.yaml": "# AUTO-GENERATED by scripts/gen_matrix_configs.py \u2014 DO NOT EDIT BY HAND.\n# Stage 2 bimodal transfer-matrix cell: text x video (S-scale).\n# Rationale + token accounting: decisions/D2-1_bimodal_matrix_design.md\n\nmodalities:\n- text\n- tok_video@128\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: dclm_val\n  shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: openvid_val\n  shards: data/shards/openvid_shuffled/val/shard_{000000..000014}.tar\n  weight: 14500\n  modality_tokens:\n    tok_video@128: 950\n  format: npz\n- name: webvid_val\n  shards: data/shards/webvid_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_video@128: 693\n  format: npz\n"
+  }
+}