Upload checkpoint at step 250

.gitattributes

@@ -49,3 +49,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 000000000200/state/ocdbt.process_0/d/56e44b5be0a5e553830c4493a29ad89b filter=lfs diff=lfs merge=lfs -text
 000000000200/state/ocdbt.process_0/d/b91441669ee779ed2646f5c866c2f5a0 filter=lfs diff=lfs merge=lfs -text
 000000000200/state/ocdbt.process_0/d/fe2ad0f87dc91bf28ad4a97ec69765b2 filter=lfs diff=lfs merge=lfs -text
+000000000250/state/ocdbt.process_0/d/2340d2c7cfd4acd8c949694a267ee6cc filter=lfs diff=lfs merge=lfs -text
+000000000250/state/ocdbt.process_0/d/4e5ff02b5a3d33e7fa8882775a7a86f5 filter=lfs diff=lfs merge=lfs -text
+000000000250/state/ocdbt.process_0/d/75622e219b800289e24991332bb94d90 filter=lfs diff=lfs merge=lfs -text
+000000000250/state/ocdbt.process_0/d/e62f913d8357a81c0ad6010c2342c3d7 filter=lfs diff=lfs merge=lfs -text

000000000250/metadata.json

@@ -0,0 +1,140 @@
+{
+    "git_commit": "d279fca943d2a70a6da4b33e6d630462538b7f84",
+    "wandb_run_id": "464sj2t5",
+    "config": {
+        "seed": 42,
+        "notes": null,
+        "trainer": {
+            "run_dir": "local_data/runs/delta",
+            "checkpoint": {
+                "load_state": false,
+                "save_optimizer": false,
+                "interval": 50
+            },
+            "wandb": {
+                "project": "oloop-instruct",
+                "enabled": true
+            },
+            "huggingface": {
+                "private": false,
+                "enabled": true
+            },
+            "global_batch_size": 128,
+            "max_steps": 1000000,
+            "num_logit_iterations": 4,
+            "aux_loss_weight": 0.1,
+            "max_grad_norm": 1.0,
+            "debug": false,
+            "groups": {
+                "slow": {
+                    "_target_": "optax.adamw",
+                    "lr": 3e-05,
+                    "b1": 0.9,
+                    "b2": 0.95,
+                    "eps": 1e-06,
+                    "weight_decay": 0.0,
+                    "warmup_steps": 100,
+                    "stable_steps": 899900,
+                    "decay_steps": 100000,
+                    "min_lr_ratio": 0.0
+                },
+                "fast": {
+                    "_target_": "optax.contrib.muon",
+                    "lr": 0.0005,
+                    "beta": 0.9,
+                    "adam_b1": 0.9,
+                    "adam_b2": 0.95,
+                    "eps": 1e-06,
+                    "weight_decay": 0.01,
+                    "ns_steps": 5,
+                    "consistent_rms": 0.2,
+                    "nesterov": true,
+                    "warmup_steps": 100,
+                    "stable_steps": 899900,
+                    "decay_steps": 100000,
+                    "min_lr_ratio": 0.0
+                }
+            },
+            "_target_": "trainers.forte_trainer.ForteTrainer"
+        },
+        "data": {
+            "skip_batches": 0,
+            "dataset": {
+                "path": "aklein4/300k-horizons",
+                "split": "train",
+                "streaming": true
+            },
+            "collator": {
+                "_target_": "collators.horizon.HorizonCollator",
+                "tokenizer_url": "meta-llama/Llama-3.2-1B-Instruct",
+                "max_length": 1024,
+                "cluster_length": 64
+            }
+        },
+        "distributed": {
+            "initialize": "auto"
+        },
+        "mesh": {
+            "fsdp_axis_size": -1
+        },
+        "name": "delta",
+        "initialization": "aklein4/forte-init-delta",
+        "initialization_step": 0,
+        "model": {
+            "sharding": {
+                "default": [],
+                "rules": [
+                    {
+                        "pattern": "^embed_tokens/embedding$",
+                        "spec": [
+                            "fsdp",
+                            null
+                        ]
+                    },
+                    {
+                        "pattern": "layers/mlp/(gate_proj|up_proj|down_proj)",
+                        "spec": [
+                            null,
+                            null,
+                            "fsdp"
+                        ]
+                    }
+                ]
+            },
+            "_target_": "models.forte.ForteModel",
+            "vocab_size": 128256,
+            "pad_token_id": -100,
+            "hidden_size": 2048,
+            "intermediate_size": 8192,
+            "num_hidden_layers": 16,
+            "num_attention_heads": 32,
+            "num_key_value_heads": 8,
+            "head_dim": 64,
+            "max_position_embeddings": 131072,
+            "rope": {
+                "theta": 500000.0,
+                "factor": 32.0,
+                "high_freq_factor": 4.0,
+                "low_freq_factor": 1.0,
+                "original_context_len": 8192
+            },
+            "rms_norm_eps": 1e-05,
+            "compute_dtype": "bfloat16",
+            "gradient_checkpointing": true,
+            "attention_backend": "auto",
+            "num_fast_vectors": 512,
+            "num_fast_heads": 16,
+            "base_lr": 0.0001,
+            "grad_rms_eps": 1e-12,
+            "delta_alpha": 0.25,
+            "num_output_layers": 10,
+            "num_bidirectional_layers": 6,
+            "mixer_kernel_size": 5,
+            "pad_attention_bias_value": -1000.0
+        }
+    },
+    "step": 250,
+    "checkpoint_items": [
+        "params"
+    ]
+}

000000000250/state/_CHECKPOINT_METADATA

@@ -0,0 +1 @@
+{"item_handlers": "orbax.checkpoint._src.handlers.standard_checkpoint_handler.StandardCheckpointHandler", "metrics": {}, "performance_metrics": {}, "init_timestamp_nsecs": 1789847539154825899, "commit_timestamp_nsecs": 1789847606575041596, "custom_metadata": {}}

000000000250/state/_METADATA

@@ -0,0 +1 @@
+{"tree_metadata": {"('params', 'backbone_layers', 'input_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "input_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'down_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "down_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'delta_log_lr')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "delta_log_lr", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'forward_gate_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "forward_gate_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'forward_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "forward_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'k_gate')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "k_gate", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'k_up')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "k_up", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'log_lr')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "log_lr", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'o_proj')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "o_proj", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'q_gate_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "q_gate_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'q_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "q_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'q_up_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "q_up_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'update_rule', 'delta_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "update_rule", "key_type": 2}, {"key": "delta_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'update_rule', 'delta_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "update_rule", "key_type": 2}, {"key": "delta_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'fast_cartridge', 'update_rule', 'gradient_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "update_rule", "key_type": 2}, {"key": "gradient_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'mlp', 'up_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "up_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'post_attention_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "post_attention_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'self_attn', 'k_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "k_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'self_attn', 'o_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "o_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'self_attn', 'q_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "q_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'backbone_layers', 'self_attn', 'v_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "backbone_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "v_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'embedding_state_scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "embedding_state_scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'embedding_state_shift')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "embedding_state_shift", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'input_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "input_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'mlp', 'down_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "down_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'mlp', 'gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'mlp', 'up_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "up_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'post_attention_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "post_attention_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'k_mixer', 'conv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "k_mixer", "key_type": 2}, {"key": "conv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'k_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "k_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'o_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "o_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'q_mixer', 'conv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "q_mixer", "key_type": 2}, {"key": "conv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'q_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "q_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'v_mixer', 'conv', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "v_mixer", "key_type": 2}, {"key": "conv", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'layers', 'self_attn', 'v_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "v_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'bidirectional_head', 'norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "bidirectional_head", "key_type": 2}, {"key": "norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'embed_tokens', 'embedding')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "embed_tokens", "key_type": 2}, {"key": "embedding", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'lm_head', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "lm_head", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'norm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "norm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'input_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "input_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'down_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "down_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'delta_log_lr')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "delta_log_lr", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'forward_gate_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "forward_gate_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'forward_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "forward_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'k_gate')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "k_gate", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'k_up')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "k_up", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'log_lr')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "log_lr", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'o_proj')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "o_proj", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'q_gate_proj', 'bias')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "q_gate_proj", "key_type": 2}, {"key": "bias", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'q_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "q_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'q_up_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "q_up_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'update_rule', 'delta_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "update_rule", "key_type": 2}, {"key": "delta_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'update_rule', 'delta_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "update_rule", "key_type": 2}, {"key": "delta_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'fast_cartridge', 'update_rule', 'gradient_gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "fast_cartridge", "key_type": 2}, {"key": "update_rule", "key_type": 2}, {"key": "gradient_gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'gate_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "gate_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'mlp', 'up_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "mlp", "key_type": 2}, {"key": "up_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'post_attention_layernorm', 'scale')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "post_attention_layernorm", "key_type": 2}, {"key": "scale", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'self_attn', 'k_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "k_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'self_attn', 'o_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "o_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'self_attn', 'q_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "q_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}, "('params', 'output_layers', 'self_attn', 'v_proj', 'kernel')": {"key_metadata": [{"key": "params", "key_type": 2}, {"key": "output_layers", "key_type": 2}, {"key": "self_attn", "key_type": 2}, {"key": "v_proj", "key_type": 2}, {"key": "kernel", "key_type": 2}], "value_metadata": {"value_type": "np.ndarray", "skip_deserialize": false}}}, "use_ocdbt": true, "use_zarr3": false, "store_array_data_equal_to_fill_value": true, "custom_metadata": null}

000000000250/state/commit_success.txt

@@ -0,0 +1 @@
+Checkpoint commit was successful to /home/aklein4/outer-loop-jax/local_data/runs/delta/checkpoints/000000000250/state

000000000250/state/ocdbt.process_0/d/2340d2c7cfd4acd8c949694a267ee6cc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad8077bfab3553c9fd315630aaeb1d81ada2b17760e2ae07a4c942762b65f309
+size 129769472

000000000250/state/ocdbt.process_0/d/4e5ff02b5a3d33e7fa8882775a7a86f5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d02fb2f87d3b4199137164936e4b6ab1cf0b3000d434c90dedaf5660432eb58
+size 2486300672

000000000250/state/ocdbt.process_0/d/75622e219b800289e24991332bb94d90

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76fae544e256ee5b7b91e29544e504c90a66f8953023d80f4ba9a71583c90822
+size 2488643584

000000000250/state/ocdbt.process_0/d/e62f913d8357a81c0ad6010c2342c3d7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7abe31f4f10715d60e5f6e1041be1c002d0542159b7212d07a70e95c54be1fc
+size 2240598016