Add 5 model file(s)

t4_core4_m/branches/branch_10Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ec6843d1dac0caedc896356014eb6352ef9b3cc00d1a85ea950cad626dd6960
+size 4495255828

t4_core4_m/branches/branch_12Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0db51a7f6a7c796ff26249c1fb933fc13dff94f78e1abb08e775e4de97405ba
+size 4495255828

t4_core4_m/branches/branch_15Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97b06a309411e1d75caed3a769626ddf7fd73df296325a3526169a3dddce8494
+size 4495255828

t4_core4_m/branches/branch_16Blb/checkpoint-final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee7c83bebcfb5c8548cda1b5dc51433b2b8301ccb8b77c194cb914dd8b3c7925
+size 4495255828

t4_core4_m/config.json

@@ -0,0 +1,113 @@
+{
+  "run_name": "t4_core4_m",
+  "epoch": 0,
+  "n_params": 383810150,
+  "args": {
+    "run_name": "t4_core4_m",
+    "batch_size": 16,
+    "epochs": 1,
+    "total_tokens": -1,
+    "lossbearing_tokens": 20.0,
+    "accum_iter": 64,
+    "effective_batch_size": null,
+    "lr_batch_scaling": "sqrt",
+    "save_ckpt_freq": 1,
+    "save_ckpt_freq_tokens": 5,
+    "debug_batch_stats": 0,
+    "model": "fm_medium_12d_swiglu_nobias",
+    "dtype": "bfloat16",
+    "loss_type": "token",
+    "context_length": 2048,
+    "pos_emb_ablation": "vanilla",
+    "max_blocks": 32,
+    "intra_pos_emb": "default",
+    "video_pos_3d": false,
+    "finetune": "",
+    "opt": "adamw",
+    "opt_eps": 1e-08,
+    "opt_betas": [
+      0.9,
+      0.95
+    ],
+    "compute_grad_norm": true,
+    "clip_grad": 1.0,
+    "skip_grad": null,
+    "momentum": 0.9,
+    "weight_decay": 0.1,
+    "weight_decay_end": 0.1,
+    "blr": 0.0006,
+    "min_blr": 3.5e-05,
+    "scheduler": "wsd",
+    "decay_fraction": 0.1,
+    "warmup_steps": 250,
+    "cooldown_steps": -1,
+    "data_config": "cfgs/default/4m/experiments/t4/core4/data_train.yaml",
+    "data_config_val": "cfgs/default/4m/experiments/t4/core4/data_val.yaml",
+    "cross_eval_val": "",
+    "s3_endpoint": "",
+    "s3_data_endpoint": "",
+    "s3_multipart_chunksize_mb": 512,
+    "s3_multipart_threshold_mb": 512,
+    "s3_max_io_queue": 100,
+    "text_tokenizer_path": "fourm/utils/tokenizer/trained/smollm_tokenizer.json",
+    "eval_freq": 1,
+    "eval_freq_tokens": 2,
+    "dist_eval": true,
+    "eval": false,
+    "val_sampling_mode": "without_replacement",
+    "output_dir": "output/curse/t4_core4_m",
+    "device": "cuda",
+    "seed": 0,
+    "resume": "",
+    "auto_resume": true,
+    "start_epoch": 0,
+    "num_workers": 10,
+    "pin_mem": true,
+    "find_unused_params": true,
+    "rlimit": 4096,
+    "print_all": false,
+    "show_user_warnings": false,
+    "s3_save_dir": "",
+    "dist_url": "env://",
+    "log_wandb": true,
+    "wandb_project": "loom",
+    "wandb_entity": null,
+    "wandb_run_name": "t4_core4_m",
+    "wandb_group": "curse-m",
+    "dim": null,
+    "depth": null,
+    "num_heads": null,
+    "mlp_ratio": null,
+    "log_grad_norms_every": 0,
+    "save_at_tokens": "4,8,12,16,20",
+    "unigram_entropies": null,
+    "cooldown_branch": false,
+    "cooldown_tokens": 0,
+    "config_path": "cfgs/default/4m/experiments/t4/core4/run_m.yaml",
+    "rank": 0,
+    "world_size": 1,
+    "gpu": 0,
+    "distributed": true,
+    "dist_backend": "nccl",
+    "num_tasks": 1,
+    "sampling_mode": "without_replacement",
+    "max_micro_steps": 1220704,
+    "lossbearing_target": 20000000000.0,
+    "lossbearing_mode": true,
+    "lr": 0.0006,
+    "min_lr": 3.5e-05,
+    "global_micro_step": 1220704,
+    "cum_lossbearing": 20000788463.0,
+    "cum_mod_tokens": {
+      "tok_rgb@384": 5064382084,
+      "text": 5029108551,
+      "tok_audio": 4944168568,
+      "tok_video@128": 4963129260
+    }
+  },
+  "source_configs": {
+    "cfgs/default/4m/experiments/t4/core4/run_m.yaml": "# AUTO-GENERATED by scripts/gen_curse_configs.py \u2014 DO NOT EDIT BY HAND.\n# Curse-of-multimodality K=4 cell (core4: audio x image x text x video) \u2014 M (d=1088).\n# Design + token accounting: decisions/D5-1_forward_program.md \u00a73.\n\n_base_: ../../../models/dec/4m-m-dec.yaml\nrun_name: t4_core4_m\ntotal_tokens: -1\nlossbearing_tokens: 20.0\ndata_config: cfgs/default/4m/experiments/t4/core4/data_train.yaml\ndata_config_val: cfgs/default/4m/experiments/t4/core4/data_val.yaml\nwarmup_steps: 250\neval_freq_tokens: 2\nsave_at_tokens: 4,8,12,16,20\nsave_ckpt_freq_tokens: 5\nwandb_group: curse-m\n",
+    "cfgs/default/4m/experiments/t4/core4/data_train.yaml": "# AUTO-GENERATED by scripts/gen_curse_configs.py \u2014 DO NOT EDIT BY HAND.\n# Curse-of-multimodality K=4 cell (core4: audio x image x text x video) \u2014 scale-independent data.\n# Design + token accounting: decisions/D5-1_forward_program.md \u00a73.\n\nmodalities: &id001\n- tok_audio\n- tok_rgb@384\n- text\n- tok_video@128\nepoch_size: 50000000\nsampler_type: forced_equal\ntarget_modalities: *id001\nreverse_block_probability: 0.5\nsampling_mode: without_replacement\nsources:\n- name: instrumental_audio\n  shards: data/shards/instrumental_audio_shuffled/train/shard_{000000..005675}.tar\n  weight: 5675407\n  modality_tokens:\n    tok_audio: 298\n  format: npz\n- name: yodas_en\n  shards: data/shards/yodas_en_reshuffled/train/shard_*.tar\n  weight: 46281311\n  modality_tokens:\n    tok_audio: 363\n  format: npz\n- name: common_voice\n  shards: data/shards/common_voice_shuffled/train/shard_{000000..002192}.tar\n  weight: 2192311\n  modality_tokens:\n    tok_audio: 216\n  format: npz\n- name: libritts_r_clean\n  shards: data/shards/libritts_r_clean_shuffled/train/shard_{000000..000114}.tar\n  weight: 114426\n  modality_tokens:\n    tok_audio: 236\n  format: npz\n- name: libritts_r_other\n  shards: data/shards/libritts_r_other_shuffled/train/shard_{000000..000203}.tar\n  weight: 203035\n  modality_tokens:\n    tok_audio: 217\n  format: npz\n- name: audiocaps\n  shards: data/shards/audiocaps_shuffled/train/shard_{000000..000043}.tar\n  weight: 43178\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: wavcaps\n  shards: data/shards/wavcaps_shuffled/train/shard_{000000..000094}.tar\n  weight: 94328\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: clotho\n  shards: data/shards/clotho_shuffled/train/shard_{000000..000003}.tar\n  weight: 3839\n  modality_tokens:\n    tok_audio: 902\n  format: npz\n- name: coyo\n  shards: data/shards/coyo_shuffled/train/shard_{000000..022189}.tar\n  weight: 22189292\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: cc12m\n  shards: data/shards/cc12m_shuffled/train/shard_{000000..012527}.tar\n  weight: 12527184\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: obelics\n  shards: data/shards/obelics_shuffled/train/shard_{000000..009245}.tar\n  weight: 9245143\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: commoncatalog\n  shards: data/shards/commoncatalog_reshuffled/train/shard_*.tar\n  weight: 71320000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: dclm\n  shards: data/shards/dclm_reshuffled/train/shard_*.tar\n  weight: 426009612\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: openvid\n  shards: data/shards/openvid_shuffled/train/shard_{000000..001009}.tar\n  weight: 1010000\n  modality_tokens:\n    tok_video@128: 950\n  format: npz\n- name: webvid\n  shards: data/shards/webvid_reshuffled/train/shard_*.tar\n  weight: 9305227\n  modality_tokens:\n    tok_video@128: 693\n  format: npz\n",
+    "cfgs/default/4m/experiments/t4/core4/data_val.yaml": "# AUTO-GENERATED by scripts/gen_curse_configs.py \u2014 DO NOT EDIT BY HAND.\n# Curse-of-multimodality K=4 cell (core4: audio x image x text x video) \u2014 scale-independent data.\n# Design + token accounting: decisions/D5-1_forward_program.md \u00a73.\n\nmodalities:\n- tok_audio\n- tok_rgb@384\n- text\n- tok_video@128\nval_steps: 200\nsampler_type: uniform\nsources:\n- name: instrumental_audio_val\n  shards: data/shards/instrumental_audio_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_audio: 298\n  format: npz\n- name: yodas_en_val\n  shards: data/shards/yodas_en_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_audio: 363\n  format: npz\n- name: audiocaps_val\n  shards: data/shards/audiocaps_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: clotho_val\n  shards: data/shards/clotho_shuffled/val/shard_000000.tar\n  weight: 500\n  modality_tokens:\n    tok_audio: 902\n  format: npz\n- name: wavcaps_val\n  shards: data/shards/wavcaps_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 395\n  format: npz\n- name: common_voice_val\n  shards: data/shards/common_voice_shuffled/val/shard_{000000..000009}.tar\n  weight: 10000\n  modality_tokens:\n    tok_audio: 216\n  format: npz\n- name: libritts_r_clean_val\n  shards: data/shards/libritts_r_clean_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 236\n  format: npz\n- name: libritts_r_other_val\n  shards: data/shards/libritts_r_other_shuffled/val/shard_{000000..000001}.tar\n  weight: 2000\n  modality_tokens:\n    tok_audio: 217\n  format: npz\n- name: coyo_val\n  shards: data/shards/coyo_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: cc12m_val\n  shards: data/shards/cc12m_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: obelics_val\n  shards: data/shards/obelics_shuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: commoncatalog_val\n  shards: data/shards/commoncatalog_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_rgb@384: 576\n  format: npz\n- name: dclm_val\n  shards: data/shards/dclm_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    text: 1021\n  format: npz\n- name: openvid_val\n  shards: data/shards/openvid_shuffled/val/shard_{000000..000014}.tar\n  weight: 14500\n  modality_tokens:\n    tok_video@128: 950\n  format: npz\n- name: webvid_val\n  shards: data/shards/webvid_reshuffled/val/shard_{000000..000049}.tar\n  weight: 50000\n  modality_tokens:\n    tok_video@128: 693\n  format: npz\n"
+  }
+}