Add 15 model file(s)

t4_core4_xxs3/branches/branch_15Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01363370ca82e2de3bf217c8263bacb908dad1d145a2f152cdb3abd89eeb4cd9
+size 1127392780

t4_core4_xxs3/branches/branch_16Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7108d12450bdb900c3ad82b645269b27f836a48ab87b423f4885de0f72e3234f
+size 1127392780

t4_core4_xxs3/branches/branch_4Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59ed1690ddd2fa8b68253f94357e5d72ea1fbae133b320c82fd43baae5417eac
+size 1127392716

t4_core4_xxs3/branches/branch_5Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75bd7180e71f8789265ad7360dc5508ade021a850274bcbc78cf46c029ba0eb1
+size 1127392716

t4_core4_xxs3/branches/branch_8Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c731990a9ac20da7ac5a6f74cdc36894d5b266337d76f395169edd33dc0c670
+size 1127392716

t4_core4_xxs3/checkpoint-10Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a834bdcb3fc894173cafeb96b18d15cddeba341146c180f7a78f427fd8be4e
+size 1127392524

t4_core4_xxs3/checkpoint-12Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13337836d6189cfc6ecb571d1fa112892ed4fb8e0ab87a1cfc7cff6f2bcb1bf2
+size 1127392524

t4_core4_xxs3/checkpoint-15Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d015c74809d09ec4d87f74f365e05d5fc2de0aae5e6c5c71c9a20905e310d6
+size 1127392524

t4_core4_xxs3/checkpoint-16Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59b79c337d21ac3ad19037e134b042d40b51b7183972319e87517e7cd4dce5b6
+size 1127392524

t4_core4_xxs3/checkpoint-20Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30e4f8f1b6f0107bf9ceffb207658e2f6ab965689c0ddb1c3cb6b656cc116563
+size 1127392524

t4_core4_xxs3/checkpoint-4Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e801da6d6e9ffe968bbf80fff2be7cdefbda3b830067285ad1dd81af72754501
+size 1127391965

t4_core4_xxs3/checkpoint-5Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26c6c09ac57fa5fc37c6157084bb75b9f7be1d80a1b2b26e6f8c0789132f72b5
+size 1127391965

t4_core4_xxs3/checkpoint-8Blb.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ea85e66562cb2e07ed0edca5c827a11e718954c0d5d8bd49dc1a212a4de4021
+size 1127391965

t4_core4_xxs3/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4b31e7cddc8366da4746cac6334814328b3e6454c0cf17a28d333e81282bb03
+size 1127392524

t8_all8_m/config.json

@@ -0,0 +1,117 @@
+{
+  "run_name": "t8_all8_m",
+  "epoch": 0,
+  "n_params": 409368782,
+  "args": {
+    "run_name": "t8_all8_m",
+    "batch_size": 16,
+    "epochs": 1,
+    "total_tokens": -1,
+    "lossbearing_tokens": 40.0,
+    "accum_iter": 64,
+    "effective_batch_size": null,
+    "lr_batch_scaling": "sqrt",
+    "save_ckpt_freq": 1,
+    "save_ckpt_freq_tokens": 5,
+    "debug_batch_stats": 0,
+    "model": "fm_medium_12d_swiglu_nobias",
+    "dtype": "bfloat16",
+    "loss_type": "token",
+    "context_length": 2048,
+    "pos_emb_ablation": "vanilla",
+    "max_blocks": 32,
+    "intra_pos_emb": "default",
+    "video_pos_3d": false,
+    "finetune": "",
+    "opt": "adamw",
+    "opt_eps": 1e-08,
+    "opt_betas": [
+      0.9,
+      0.95
+    ],
+    "compute_grad_norm": true,
+    "clip_grad": 1.0,
+    "skip_grad": null,
+    "momentum": 0.9,
+    "weight_decay": 0.1,
+    "weight_decay_end": 0.1,
+    "blr": 0.0006,
+    "min_blr": 3.5e-05,
+    "scheduler": "wsd",
+    "decay_fraction": 0.1,
+    "warmup_steps": 250,
+    "cooldown_steps": -1,
+    "data_config": "cfgs/default/4m/experiments/t8/all8/data_train.yaml",
+    "data_config_val": "cfgs/default/4m/experiments/t8/all8/data_val.yaml",
+    "cross_eval_val": "",
+    "s3_endpoint": "",
+    "s3_data_endpoint": "",
+    "s3_multipart_chunksize_mb": 512,
+    "s3_multipart_threshold_mb": 512,
+    "s3_max_io_queue": 100,
+    "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
+    "eval_freq": 1,
+    "eval_freq_tokens": 4,
+    "dist_eval": true,
+    "eval": false,
+    "val_sampling_mode": "without_replacement",
+    "output_dir": "output/curse/t8_all8_m",
+    "device": "cuda",
+    "seed": 0,
+    "resume": "output/curse/t8_all8_m/checkpoint-35Blb.pth",
+    "auto_resume": true,
+    "start_epoch": 0,
+    "num_workers": 10,
+    "pin_mem": true,
+    "find_unused_params": true,
+    "rlimit": 4096,
+    "print_all": false,
+    "show_user_warnings": false,
+    "s3_save_dir": "",
+    "dist_url": "env://",
+    "log_wandb": true,
+    "wandb_project": "loom",
+    "wandb_entity": null,
+    "wandb_run_name": "t8_all8_m",
+    "wandb_group": "curse-m",
+    "dim": null,
+    "depth": null,
+    "num_heads": null,
+    "mlp_ratio": null,
+    "log_grad_norms_every": 0,
+    "save_at_tokens": "8,16,24,32,40",
+    "unigram_entropies": null,
+    "cooldown_branch": false,
+    "cooldown_tokens": 0,
+    "config_path": "cfgs/default/4m/experiments/t8/all8/run_m.yaml",
+    "rank": 0,
+    "world_size": 1,
+    "gpu": 0,
+    "distributed": true,
+    "dist_backend": "nccl",
+    "num_tasks": 1,
+    "sampling_mode": "without_replacement",
+    "max_micro_steps": 2441407,
+    "lossbearing_target": 40000000000.0,
+    "lossbearing_mode": true,
+    "lr": 0.0006,
+    "min_lr": 3.5e-05,
+    "global_micro_step": 2441407,
+    "cum_lossbearing": 40000573084.0,
+    "cum_mod_tokens": {
+      "tok_rgb@384": 4964108139,
+      "text": 5536400149,
+      "tok_audio": 4846051570,
+      "tok_video@128": 4866595046,
+      "tok_timeseries": 4939376569,
+      "tok_3d": 4863017852,
+      "tok_molecule": 5106510722,
+      "tok_satellite@128": 4878513037
+    }
+  },
+  "source_configs": {
+    "cfgs/default/4m/experiments/t8/all8/run_m.yaml": "# AUTO-GENERATED by scripts/gen_curse_configs.py \u2014 DO NOT EDIT BY HAND.\n# Curse-of-multimodality K=8 cell (all8: 3d x audio x image x molecule x satellite x text x timeseries x video) \u2014 M (d=1088).\n# Design + token accounting: decisions/D5-1_forward_program.md \u00a73.\n\n_base_: ../../../models/dec/4m-m-dec.yaml\nrun_name: t8_all8_m\ntotal_tokens: -1\nlossbearing_tokens: 40.0\ndata_config: cfgs/default/4m/experiments/t8/all8/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t8/all8/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 4\nsave_at_tokens: 8,16,24,32,40\nsave_ckpt_freq_tokens: 5\nwandb_group: curse-m\n",
+    "cfgs/default/4m/experiments/t8/all8/data_train.yaml": "# AUTO-GENERATED by scripts/gen_curse_configs.py \u2014 DO NOT EDIT BY HAND.\n# Curse-of-multimodality K=8 cell (all8: 3d x audio x image x molecule x satellite x text x timeseries x video) \u2014 scale-independent data.\n# Design + token accounting: decisions/D5-1_forward_program.md \u00a73.\n\nmodalities: &id001\n- tok_3d\n- tok_audio\n- tok_rgb@384\n- tok_molecule\n- tok_satellite@128\n- text\n- tok_timeseries\n- tok_video@128\nepoch_size: 50000000\nsampler_type: forced_equal\ntarget_modalities: *id001\nreverse_block_probability: 0.5\nsampling_mode: without_replacement\nsources:\n- name: 3d\n  shards: data/shards/3d_shuffled/train/shard_{000000..000685}.tar\n  weight: 685736\n  modality_tokens:\n    tok_3d: 1875\n  format: npz\n- name: instrumental_audio\n  shards: data/shards/instrumental_audio_shuffled/train/shard_{000000..005675}.tar\n  weight: 5675407\n  modality_tokens:\n    tok_audio: 298\n  format: npz\n- name: yodas_en\n  shards: data/shards/yodas_en_reshuffled/train/shard_*.tar\n  weight: 46281311\n  modality_tokens:\n    tok_audio: 363\n  format: npz\n- name: common_voice\n  shards: data/shards/common_voice_shuffled/train/shard_{000000..002192}.tar\n  weight: 2192311\n  modality_tokens:\n    tok_audio: 216\n  format: npz\n- name: libritts_r_clean\n  shards: data/shards/libritts_r_clean_shuffled/train/shard_{000000..000114}.tar\n  weight: 114426\n  modality_tokens:\n    tok_audio: 236\n  format: npz\n- name: libritts_r_other\n  shards: data/shards/libritts_r_other_shuffled/train/shard_{000000..000203}.tar\n  weight: 203035\n  modality_tokens:\n    tok_audio: 217\n  format: npz\n- name: audiocaps\n  shards: data/shards/audiocaps_shuffled/train/shard_{000000..000043}.tar\n  weight: 43178\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: wavcaps\n  shards: data/shards/wavcaps_shuffled/train/shard_{000000..000094}.tar\n  weight: 94328\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: clotho\n  shards: data/shards/clotho_shuffled/train/shard_{000000..000003}.tar\n  weight: 3839\n  modality_tokens:\n    tok_audio: 902\n  format: npz\n- name: coyo\n  shards: data/shards/coyo_shuffled/train/shard_{000000..022189}.tar\n  weight: 22189292\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: cc12m\n  shards: data/shards/cc12m_shuffled/train/shard_{000000..012527}.tar\n  weight: 12527184\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: obelics\n  shards: data/shards/obelics_shuffled/train/shard_{000000..009245}.tar\n  weight: 9245143\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: commoncatalog\n  shards: data/shards/commoncatalog_reshuffled/train/shard_*.tar\n  weight: 71320000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: molecule_zinc20m\n  shards: data/shards/molecule_zinc20m_shuffled/train/shard_{000000..021643}.tar\n  weight: 21643268\n  modality_tokens:\n    tok_molecule: 24\n  format: npz\n- name: molecule_pubchemqc\n  shards: data/shards/molecule_pubchemqc_reshuffled/train/shard_*.tar\n  weight: 213751114\n  modality_tokens:\n    tok_molecule: 24\n  format: npz\n- name: molecule_chebi20\n  shards: data/shards/molecule_chebi20_shuffled/train/shard_{000000..000021}.tar\n  weight: 21378\n  modality_tokens:\n    tok_molecule: 26\n  format: npz\n- name: molecule_molcap\n  shards: data/shards/molecule_molcap_shuffled/train/shard_{000000..000070}.tar\n  weight: 70805\n  modality_tokens:\n    tok_molecule: 29\n  format: npz\n- name: satellite_v2\n  shards: data/shards/satellite_v2_reshuffled/train/shard_*.tar\n  weight: 22408820\n  modality_tokens:\n    tok_satellite@128: 256\n  format: npz\n- name: satellite_ben\n  shards: data/shards/satellite_ben_shuffled/train/shard_{000000..000470}.tar\n  weight: 470438\n  modality_tokens:\n    tok_satellite@128: 256\n  format: npz\n- name: dclm\n  shards: data/shards/dclm_reshuffled/train/shard_*.tar\n  weight: 426009612\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: utsd\n  shards: data/shards/utsd_shuffled/train/shard_{000000..002934}.tar\n  weight: 2934667\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: forecasting\n  shards: data/shards/forecasting_shuffled/train/shard_{000000..000098}.tar\n  weight: 98423\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: timeseries\n  shards: data/shards/timeseries_shuffled/train/shard_{000000..000109}.tar\n  weight: 109190\n  modality_tokens:\n    tok_timeseries: 14\n  format: npz\n- name: timeseries_pile\n  shards: data/shards/timeseries_pile_shuffled/train/shard_{000000..005159}.tar\n  weight: 5159748\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: time300b\n  shards: data/shards/time300b_shuffled/train/shard_{000000..017340}.tar\n  weight: 17340008\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: openvid\n  shards: data/shards/openvid_shuffled/train/shard_{000000..001009}.tar\n  weight: 1010000\n  modality_tokens:\n    tok_video@128: 950\n  format: npz\n- name: webvid\n  shards: data/shards/webvid_reshuffled/train/shard_*.tar\n  weight: 9305227\n  modality_tokens:\n    tok_video@128: 693\n  format: npz\n",
+    "cfgs/default/4m/experiments/t8/all8/data_val.yaml": "# AUTO-GENERATED by scripts/gen_curse_configs.py \u2014 DO NOT EDIT BY HAND.\n# Curse-of-multimodality K=8 cell (all8: 3d x audio x image x molecule x satellite x text x timeseries x video) \u2014 scale-independent data.\n# Design + token accounting: decisions/D5-1_forward_program.md \u00a73.\n\nmodalities:\n- tok_3d\n- tok_audio\n- tok_rgb@384\n- tok_molecule\n- tok_satellite@128\n- text\n- tok_timeseries\n- tok_video@128\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: 3d_val\n  shards: data/shards/3d_shuffled/val/shard_{000000..000013}.tar\n  weight: 14000\n  modality_tokens:\n    tok_3d: 1875\n  format: npz\n- name: instrumental_audio_val\n  shards: data/shards/instrumental_audio_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_audio: 298\n  format: npz\n- name: yodas_en_val\n  shards: data/shards/yodas_en_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_audio: 363\n  format: npz\n- name: audiocaps_val\n  shards: data/shards/audiocaps_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: clotho_val\n  shards: data/shards/clotho_shuffled/val/shard_000000.tar\n  weight: 500\n  modality_tokens:\n    tok_audio: 902\n  format: npz\n- name: wavcaps_val\n  shards: data/shards/wavcaps_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: common_voice_val\n  shards: data/shards/common_voice_shuffled/val/shard_{000000..000009}.tar\n  weight: 10000\n  modality_tokens:\n    tok_audio: 216\n  format: npz\n- name: libritts_r_clean_val\n  shards: data/shards/libritts_r_clean_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 236\n  format: npz\n- name: libritts_r_other_val\n  shards: data/shards/libritts_r_other_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 217\n  format: npz\n- name: coyo_val\n  shards: data/shards/coyo_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: cc12m_val\n  shards: data/shards/cc12m_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: obelics_val\n  shards: data/shards/obelics_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: commoncatalog_val\n  shards: data/shards/commoncatalog_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: molecule_zinc20m_val\n  shards: data/shards/molecule_zinc20m_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_molecule: 24\n  format: npz\n- name: molecule_pubchemqc_val\n  shards: data/shards/molecule_pubchemqc_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_molecule: 24\n  format: npz\n- name: molecule_chebi20_val\n  shards: data/shards/molecule_chebi20_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_molecule: 26\n  format: npz\n- name: molecule_molcap_val\n  shards: data/shards/molecule_molcap_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_molecule: 29\n  format: npz\n- name: satellite_v2_val\n  shards: data/shards/satellite_v2_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_satellite@128: 256\n  format: npz\n- name: satellite_ben_val\n  shards: data/shards/satellite_ben_shuffled/val/shard_{000000..000009}.tar\n  weight: 9600\n  modality_tokens:\n    tok_satellite@128: 256\n  format: npz\n- name: dclm_val\n  shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: utsd_val\n  shards: data/shards/utsd_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: forecasting_val\n  shards: data/shards/forecasting_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: timeseries_val\n  shards: data/shards/timeseries_shuffled/val/shard_{000000..000002}.tar\n  weight: 2200\n  modality_tokens:\n    tok_timeseries: 14\n  format: npz\n- name: timeseries_pile_val\n  shards: data/shards/timeseries_pile_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: time300b_val\n  shards: data/shards/time300b_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_timeseries: 64\n  format: npz\n- name: openvid_val\n  shards: data/shards/openvid_shuffled/val/shard_{000000..000014}.tar\n  weight: 14500\n  modality_tokens:\n    tok_video@128: 950\n  format: npz\n- name: webvid_val\n  shards: data/shards/webvid_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_video@128: 693\n  format: npz\n"
+  }
+}