Upload checkpoint at step 0
Browse files- .gitattributes +10 -0
- 000000000000/metadata.json +130 -0
- 000000000000/state/_CHECKPOINT_METADATA +1 -0
- 000000000000/state/_METADATA +1 -0
- 000000000000/state/commit_success.txt +1 -0
- 000000000000/state/d/cd288e7534c44c3b9b3cfad34839ca5b +0 -0
- 000000000000/state/manifest.ocdbt +0 -0
- 000000000000/state/ocdbt.process_0/d/11be9c792dcf1c233f3c60fafe657655 +0 -0
- 000000000000/state/ocdbt.process_0/d/3d6f5235e1a51a69f07631b3fbfa8e7e +3 -0
- 000000000000/state/ocdbt.process_0/d/3f83256a0cac1031d0815f9c74ae89e0 +3 -0
- 000000000000/state/ocdbt.process_0/d/412ddeb82e11f2346c5c813ce2697531 +0 -0
- 000000000000/state/ocdbt.process_0/d/48ee302f617539a07b0e28dbdb7f072a +0 -0
- 000000000000/state/ocdbt.process_0/d/57f424e289139dc880b0a73b87c28d83 +3 -0
- 000000000000/state/ocdbt.process_0/d/5d6104d6240e9ee65a9fa8786f78c2e7 +3 -0
- 000000000000/state/ocdbt.process_0/d/776c7ca24b00fb03a55bb20bee67431c +3 -0
- 000000000000/state/ocdbt.process_0/d/8151f6479038b4aebcd91256091485ac +3 -0
- 000000000000/state/ocdbt.process_0/d/9bda002cf3b9b191d63880155383e4d8 +0 -0
- 000000000000/state/ocdbt.process_0/d/a246c75caf0e1994a8b2b370bdf828c9 +0 -0
- 000000000000/state/ocdbt.process_0/d/a2bff8c4e2eb081c0de2f8f8db409b48 +3 -0
- 000000000000/state/ocdbt.process_0/d/abadfef5df729f296cbc9b91b24859b8 +3 -0
- 000000000000/state/ocdbt.process_0/d/be13119facfce192c55a5e74ca7820d1 +3 -0
- 000000000000/state/ocdbt.process_0/d/dd49540f1a793fefd8ca0498be3deb22 +3 -0
- 000000000000/state/ocdbt.process_0/manifest.ocdbt +0 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
000000000000/state/ocdbt.process_0/d/3d6f5235e1a51a69f07631b3fbfa8e7e filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
000000000000/state/ocdbt.process_0/d/3f83256a0cac1031d0815f9c74ae89e0 filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
000000000000/state/ocdbt.process_0/d/57f424e289139dc880b0a73b87c28d83 filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
000000000000/state/ocdbt.process_0/d/5d6104d6240e9ee65a9fa8786f78c2e7 filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
000000000000/state/ocdbt.process_0/d/776c7ca24b00fb03a55bb20bee67431c filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
000000000000/state/ocdbt.process_0/d/8151f6479038b4aebcd91256091485ac filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
000000000000/state/ocdbt.process_0/d/a2bff8c4e2eb081c0de2f8f8db409b48 filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
000000000000/state/ocdbt.process_0/d/abadfef5df729f296cbc9b91b24859b8 filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
000000000000/state/ocdbt.process_0/d/be13119facfce192c55a5e74ca7820d1 filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
000000000000/state/ocdbt.process_0/d/dd49540f1a793fefd8ca0498be3deb22 filter=lfs diff=lfs merge=lfs -text
|
000000000000/metadata.json
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"git_commit": "ebcdc1332acafdbc395c873fc099d0e0b7e7542d",
|
| 3 |
+
"wandb_run_id": null,
|
| 4 |
+
"config": {
|
| 5 |
+
"seed": 42,
|
| 6 |
+
"notes": null,
|
| 7 |
+
"trainer": {
|
| 8 |
+
"run_dir": "local_data/runs/forte",
|
| 9 |
+
"checkpoint": {
|
| 10 |
+
"load_state": false,
|
| 11 |
+
"save_optimizer": false,
|
| 12 |
+
"interval": 50
|
| 13 |
+
},
|
| 14 |
+
"wandb": {
|
| 15 |
+
"project": "outer-loop-jax",
|
| 16 |
+
"enabled": true
|
| 17 |
+
},
|
| 18 |
+
"huggingface": {
|
| 19 |
+
"private": false,
|
| 20 |
+
"enabled": true
|
| 21 |
+
},
|
| 22 |
+
"global_batch_size": 128,
|
| 23 |
+
"max_steps": 1000000,
|
| 24 |
+
"num_logit_iterations": 8,
|
| 25 |
+
"aux_loss_weight": 0.1,
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"max_grad_value": null,
|
| 28 |
+
"debug": false,
|
| 29 |
+
"groups": {
|
| 30 |
+
"slow": {
|
| 31 |
+
"_target_": "optax.adamw",
|
| 32 |
+
"lr": 3e-05,
|
| 33 |
+
"b1": 0.9,
|
| 34 |
+
"b2": 0.95,
|
| 35 |
+
"eps": 1e-06,
|
| 36 |
+
"weight_decay": 0.0,
|
| 37 |
+
"warmup_steps": 100,
|
| 38 |
+
"stable_steps": 899900,
|
| 39 |
+
"decay_steps": 100000,
|
| 40 |
+
"min_lr_ratio": 0.0
|
| 41 |
+
},
|
| 42 |
+
"fast": {
|
| 43 |
+
"_target_": "optax.contrib.muon",
|
| 44 |
+
"lr": 0.0005,
|
| 45 |
+
"beta": 0.9,
|
| 46 |
+
"adam_b1": 0.9,
|
| 47 |
+
"adam_b2": 0.95,
|
| 48 |
+
"eps": 1e-06,
|
| 49 |
+
"weight_decay": 0.01,
|
| 50 |
+
"ns_steps": 5,
|
| 51 |
+
"consistent_rms": 0.2,
|
| 52 |
+
"nesterov": true,
|
| 53 |
+
"warmup_steps": 100,
|
| 54 |
+
"stable_steps": 899900,
|
| 55 |
+
"decay_steps": 100000,
|
| 56 |
+
"min_lr_ratio": 0.0
|
| 57 |
+
}
|
| 58 |
+
},
|
| 59 |
+
"_target_": "trainers.forte_trainer.ForteTrainer"
|
| 60 |
+
},
|
| 61 |
+
"data": {
|
| 62 |
+
"skip_batches": 0,
|
| 63 |
+
"dataset": {
|
| 64 |
+
"path": "aklein4/300k-horizons",
|
| 65 |
+
"split": "train",
|
| 66 |
+
"streaming": true
|
| 67 |
+
},
|
| 68 |
+
"collator": {
|
| 69 |
+
"_target_": "collators.horizon.HorizonCollator",
|
| 70 |
+
"tokenizer_url": "meta-llama/Llama-3.2-1B-Instruct",
|
| 71 |
+
"max_length": 1024,
|
| 72 |
+
"cluster_length": 64
|
| 73 |
+
}
|
| 74 |
+
},
|
| 75 |
+
"distributed": {
|
| 76 |
+
"initialize": "auto"
|
| 77 |
+
},
|
| 78 |
+
"mesh": {
|
| 79 |
+
"fsdp_axis_size": -1
|
| 80 |
+
},
|
| 81 |
+
"name": "forte",
|
| 82 |
+
"initialization": "aklein4/forte-init",
|
| 83 |
+
"initialization_step": 0,
|
| 84 |
+
"model": {
|
| 85 |
+
"sharding": {
|
| 86 |
+
"default": [],
|
| 87 |
+
"rules": [
|
| 88 |
+
{
|
| 89 |
+
"pattern": "layers/(self_attn/(q_proj|o_proj)|mlp/(gate_proj|up_proj|down_proj))",
|
| 90 |
+
"spec": [
|
| 91 |
+
null,
|
| 92 |
+
null,
|
| 93 |
+
"fsdp"
|
| 94 |
+
]
|
| 95 |
+
}
|
| 96 |
+
]
|
| 97 |
+
},
|
| 98 |
+
"_target_": "models.forte.ForteModel",
|
| 99 |
+
"vocab_size": 128256,
|
| 100 |
+
"pad_token_id": -100,
|
| 101 |
+
"hidden_size": 2048,
|
| 102 |
+
"intermediate_size": 8192,
|
| 103 |
+
"num_hidden_layers": 16,
|
| 104 |
+
"num_attention_heads": 32,
|
| 105 |
+
"num_key_value_heads": 8,
|
| 106 |
+
"head_dim": 64,
|
| 107 |
+
"max_position_embeddings": 131072,
|
| 108 |
+
"rope": {
|
| 109 |
+
"theta": 500000.0,
|
| 110 |
+
"factor": 32.0,
|
| 111 |
+
"high_freq_factor": 4.0,
|
| 112 |
+
"low_freq_factor": 1.0,
|
| 113 |
+
"original_context_len": 8192
|
| 114 |
+
},
|
| 115 |
+
"rms_norm_eps": 1e-05,
|
| 116 |
+
"compute_dtype": "bfloat16",
|
| 117 |
+
"gradient_checkpointing": true,
|
| 118 |
+
"attention_backend": "auto",
|
| 119 |
+
"fast_weight_size": 1024,
|
| 120 |
+
"base_lr": 0.001,
|
| 121 |
+
"pool_size": 1024,
|
| 122 |
+
"mixer_size": 5,
|
| 123 |
+
"grad_rms_eps": 1e-12
|
| 124 |
+
}
|
| 125 |
+
},
|
| 126 |
+
"step": 0,
|
| 127 |
+
"checkpoint_items": [
|
| 128 |
+
"params"
|
| 129 |
+
]
|
| 130 |
+
}
|
000000000000/state/_CHECKPOINT_METADATA
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"item_handlers": "orbax.checkpoint._src.handlers.standard_checkpoint_handler.StandardCheckpointHandler", "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1788549287923667589, "commit_timestamp_nsecs": 1788549293942963437, "custom_metadata": {}}
|
000000000000/state/_METADATA
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"tree_metadata": {"('params', 'embed_tokens', 'embedding')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "embed_tokens", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'input_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "input_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'down_fast', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "down_fast", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'down_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "down_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'a_scale_mixer', 'conv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "a_scale_mixer", "key_type": 2}, {"key": "conv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'a_scale_pass_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "a_scale_pass_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'a_scale_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "a_scale_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'a_scale_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "a_scale_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'g_scale_mixer', 'conv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "g_scale_mixer", "key_type": 2}, {"key": "conv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'g_scale_pass_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "g_scale_pass_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'g_scale_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "g_scale_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'g_scale_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "g_scale_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'log_lr')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "log_lr", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'g_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "g_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'g_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "g_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'r_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "r_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'v_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "v_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'v_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "v_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'w_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "w_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'passer', 'w_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "passer", "key_type": 2}, {"key": "w_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'token_gate_next_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "token_gate_next_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'token_gate_next_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "token_gate_next_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'token_gate_pass_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "token_gate_pass_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'token_gate_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "token_gate_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'fast_dynamic_lr', 'token_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_dynamic_lr", "key_type": 2}, {"key": "token_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'gate_fast', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gate_fast", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'gate_fast', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gate_fast", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'sig_fast', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "sig_fast", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'sig_fast', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "sig_fast", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'up_fast', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "up_fast", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'mlp', 'up_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "up_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'post_attention_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "post_attention_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'self_attn', 'k_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "k_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'self_attn', 'o_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "o_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'self_attn', 'q_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "q_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'layers', 'self_attn', 'v_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "v_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'lm_head', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "lm_head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null}
|
000000000000/state/commit_success.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Checkpoint commit was successful to /tmp/forte-init-qreadsh4/000000000000/state
|
000000000000/state/d/cd288e7534c44c3b9b3cfad34839ca5b
ADDED
|
Binary file (1.68 kB). View file
|
|
|
000000000000/state/manifest.ocdbt
ADDED
|
Binary file (117 Bytes). View file
|
|
|
000000000000/state/ocdbt.process_0/d/11be9c792dcf1c233f3c60fafe657655
ADDED
|
Binary file (201 Bytes). View file
|
|
|
000000000000/state/ocdbt.process_0/d/3d6f5235e1a51a69f07631b3fbfa8e7e
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:984e1abf2fb413220561a50b23085e8d317e8fdc0def654eeb110d67dfe51c81
|
| 3 |
+
size 62246912
|
000000000000/state/ocdbt.process_0/d/3f83256a0cac1031d0815f9c74ae89e0
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:83b66a673d6cc83f6268833c64b833f429daced905e9621451f89c9ab5c76a92
|
| 3 |
+
size 258658304
|
000000000000/state/ocdbt.process_0/d/412ddeb82e11f2346c5c813ce2697531
ADDED
|
Binary file (38.7 kB). View file
|
|
|
000000000000/state/ocdbt.process_0/d/48ee302f617539a07b0e28dbdb7f072a
ADDED
|
Binary file (765 Bytes). View file
|
|
|
000000000000/state/ocdbt.process_0/d/57f424e289139dc880b0a73b87c28d83
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:99d4ccc2b7b1d59f4ccc6b04da1e7c0a0ceed1ae0b7ba1fcb92c06fb57420506
|
| 3 |
+
size 513118208
|
000000000000/state/ocdbt.process_0/d/5d6104d6240e9ee65a9fa8786f78c2e7
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc45e4d1d2b7a69711530e5404eb46a99f9f6835770d2355bbf5adaee4986548
|
| 3 |
+
size 365447
|
000000000000/state/ocdbt.process_0/d/776c7ca24b00fb03a55bb20bee67431c
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c8601c6b53e78ffca3f352841a1fe49fad7532ff185c913e56e5defc89a18bbf
|
| 3 |
+
size 1028202496
|
000000000000/state/ocdbt.process_0/d/8151f6479038b4aebcd91256091485ac
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47c3a61bc62797f0020f6562e4ab3f2c67254c7df8cfbb71a03c46ea17902ae3
|
| 3 |
+
size 1184002048
|
000000000000/state/ocdbt.process_0/d/9bda002cf3b9b191d63880155383e4d8
ADDED
|
Binary file (61.8 kB). View file
|
|
|
000000000000/state/ocdbt.process_0/d/a246c75caf0e1994a8b2b370bdf828c9
ADDED
|
Binary file (61.7 kB). View file
|
|
|
000000000000/state/ocdbt.process_0/d/a2bff8c4e2eb081c0de2f8f8db409b48
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:62a8f18658a4138955ecd93837ca63aaf8c3a2d06ccc1938f92592e868c6759d
|
| 3 |
+
size 496529408
|
000000000000/state/ocdbt.process_0/d/abadfef5df729f296cbc9b91b24859b8
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47416d1c3d6bec8a504f431efb78af31b4daf9229c6306d4eb198042cfa60266
|
| 3 |
+
size 124485632
|
000000000000/state/ocdbt.process_0/d/be13119facfce192c55a5e74ca7820d1
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5e520974634f92ae7e19c2b87aacf706c647a11917bcd58a91e6317baea33980
|
| 3 |
+
size 501784576
|
000000000000/state/ocdbt.process_0/d/dd49540f1a793fefd8ca0498be3deb22
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:247dc3e42e7f592dbabb2712774d1b61d63c237a4d6cde0de3ee25786e321f6e
|
| 3 |
+
size 899929
|
000000000000/state/ocdbt.process_0/manifest.ocdbt
ADDED
|
Binary file (733 Bytes). View file
|
|
|