Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoints-DDPStrategy/step-00099999-cd140m-n5v2-dt0p3-mr27-mbd8-100k filter=lfs diff=lfs merge=lfs -text

checkpoints-DDPStrategy/step-00099999-cd140m-n5v2-dt0p3-mr27-mbd8-100k

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:517c3bd9d3a50dc3d263552f9ed23b6d2f32f02d7df6748d55ffdfd2a931059b
+size 578368517

run_config.json

@@ -0,0 +1,261 @@
+{
+    "run_name": "cd140m-n5v2-dt0p3-mr27-mbd8-100k",
+    "out_dir": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k",
+    "resume": false,
+    "resume_checkpoint_path": null,
+    "max_tokens": 0,
+    "max_steps": 100000,
+    "max_flops": null,
+    "token_to_param_ratio": null,
+    "exit_at_step": null,
+    "seed": 42,
+    "model_name": "parcae-small-140m",
+    "model_impl": "dynamic",
+    "block_size": 29,
+    "ignore_block_size_mismatch": true,
+    "model_checkpoint": "/datastor2/abao/checkpoints/parcae_runs/init/parcae-140m-fabric.pt",
+    "doc_block_attn": false,
+    "cache_attn": false,
+    "eod_token": null,
+    "recurrence_schedule": "linear",
+    "cirriculum_steps": 0,
+    "world_batch_size": 512,
+    "batch_size_ramp": 0,
+    "optimizer": "MuonAdamW",
+    "optim_config": {
+        "lr": 0.005,
+        "muon_lr": 0.005,
+        "weight_decay": 0.0,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-10,
+        "muon_momentum": 0.95,
+        "muon_wd": 0.0,
+        "muon_ns_steps": 5,
+        "muon_momentum_warmup_steps": 100
+    },
+    "grad_clip": 1.0,
+    "warmup_steps": 0.05,
+    "cooldown_steps": 0.2,
+    "lr_schedule": "trapezoid",
+    "min_lr": 0.0,
+    "no_weight_decay_for_bias_and_norm_params": true,
+    "label_smoothing": 0.0,
+    "z_regularization": 0.0,
+    "goldfish": {
+        "k_token_loss_dropout": 4,
+        "start_position": 0,
+        "context_width": 13,
+        "strategy": null
+    },
+    "fabric_strategy": "ddp",
+    "fabric_precision": "bf16-mixed",
+    "fabric_use_lightning_environment": false,
+    "fabric": {
+        "optimize_communication": false,
+        "all_reduce_dtype": null,
+        "row_tensor_parallel_size": 1,
+        "col_tensor_parallel_size": 1,
+        "depth_tensor_parallel_size": 1,
+        "optim_sharding": false,
+        "allow_optim_fusion": false,
+        "use_apex_adamw": false,
+        "fsdp_use_original_params": false
+    },
+    "micro_batch_size": 512,
+    "compile_model": true,
+    "compile_backend": "inductor",
+    "compile_mode": "default",
+    "compile_fullgraph": false,
+    "compile_dynamic": false,
+    "matmul_precision": "high",
+    "dataloader_num_workers": 0,
+    "n_chunks": 4,
+    "gradient_checkpointing": false,
+    "allow_nonfinite_loss": false,
+    "compiled_autograd": false,
+    "compile_optimizer": false,
+    "dynamo_ddp_config": "ddp_optimizer",
+    "loss_guardrail_active": false,
+    "skip_nonfinite_grads": true,
+    "fail_instead_of_recompile": false,
+    "logger_name": "wandb",
+    "logger_entity": "gilpinlab",
+    "logger_project": "parcae-countdown",
+    "logger_group": "cd140m-n5v2-long",
+    "data_telemetry": false,
+    "model_telemetry": true,
+    "extreme_telemetry": false,
+    "shape_watching_steps": 3,
+    "log_step_interval": 1,
+    "eval_iters": 3,
+    "eval_precision": "same",
+    "save_step_interval": 10000,
+    "eval_step_interval": 500,
+    "save_first_step": false,
+    "save_last_step": true,
+    "save_n_min_before_job_done": null,
+    "save_keep_last": 9,
+    "best_checkpoint_metric": null,
+    "best_checkpoint_mode": "max",
+    "measure_utilization": true,
+    "partial_depth_eval": [
+        8,
+        27,
+        32
+    ],
+    "generative_eval": {
+        "enabled": true,
+        "target": "countdown_store_eval_hook:evaluate",
+        "depths": [
+            8,
+            27,
+            32
+        ],
+        "num_batches": 1,
+        "seed": 1234,
+        "max_new_tokens": null,
+        "extra": {
+            "eval_train": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_train",
+            "eval_id": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_id",
+            "eval_ood": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/eval_ood"
+        }
+    },
+    "shuffle_filenames": true,
+    "shuffle_blocks": true,
+    "pad_to_block_size": true,
+    "add_bos": true,
+    "add_eos": true,
+    "pack_sequences": false,
+    "pack_buffer_size": 1000,
+    "data_signature": {
+        "keys": [
+            "text"
+        ],
+        "format_fn": "pass_text"
+    },
+    "collate_checks_enabled": true,
+    "all_block_size_tensors": false,
+    "use_chat_template": false,
+    "data_config": {
+        "train_data": [
+            {
+                "type": "hfds",
+                "prefix": "countdown-train",
+                "weight": 1,
+                "data_signature": {
+                    "keys": [
+                        "input",
+                        "target"
+                    ],
+                    "format_fn": "condition_input_supervise_target"
+                },
+                "name": null,
+                "data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/train",
+                "text_key": null,
+                "repetitions": null,
+                "wrap_epochs": true,
+                "max_epoch": null,
+                "scheduler": null,
+                "return_data_id": false
+            }
+        ],
+        "val_data": [
+            {
+                "type": "hfds",
+                "prefix": "countdown-val",
+                "weight": 1,
+                "data_signature": {
+                    "keys": [
+                        "input",
+                        "target"
+                    ],
+                    "format_fn": "condition_input_supervise_target"
+                },
+                "name": null,
+                "data_dir": "/datastor2/abao/data/countdown/store_mid_n5v2_sft/val",
+                "text_key": null,
+                "repetitions": null,
+                "wrap_epochs": false,
+                "max_epoch": null,
+                "scheduler": null,
+                "return_data_id": false
+            }
+        ]
+    },
+    "train_data_dir": "",
+    "val_data_dir": "",
+    "tokenizer_path": "/datastor2/abao/checkpoints/parcae_runs/cd140m-n5v2-dt0p3-mr27-mbd8-100k/tokenizer",
+    "model_config": {
+        "name": "parcae-small-140m",
+        "hf_config": {
+            "org": "SandyResearch",
+            "name": "parcae-small-140m"
+        },
+        "block_size": 2048,
+        "n_embd": 768,
+        "intermediate_size": 3072,
+        "num_attention_heads": 6,
+        "num_key_value_heads": 6,
+        "vocab_size": 32768,
+        "padding_multiple": 64,
+        "padded_vocab_size": 32768,
+        "rope_settings": {
+            "use_rope": true,
+            "rope_condense_ratio": 1,
+            "rope_base": 50000
+        },
+        "use_abacus": false,
+        "randomize_positions_from": null,
+        "block_class_name": "TransformerPreNormBlock",
+        "norm_class_name": "RMSNorm",
+        "attn_impl": "flash",
+        "norm_eps": 1e-05,
+        "mlp_class_name": "BaseMLP",
+        "nonlin_name": "ReLU2",
+        "bias": false,
+        "qk_bias": false,
+        "init_strategy": "scaled-zero",
+        "init_orthogonal": true,
+        "skip_initialization": false,
+        "mup_model_scaling_factor": 1,
+        "use_fused_head": "pytorch",
+        "debias_attention": false,
+        "center_attention": false,
+        "clip_qkv": null,
+        "qk_norm": true,
+        "logit_softcap": null,
+        "activation_checkpoint_impl": "per-iteration",
+        "simple_ops": false,
+        "strategy": "ddp",
+        "injection_type": "diagonal",
+        "dt_scale": 0.3,
+        "n_layers_in_recurrent_block": 2,
+        "n_layers_in_prelude": 2,
+        "n_layers_in_coda": 2,
+        "state_init": "like-init",
+        "recurrent_embedding_dimension": 768,
+        "recurrent_intermediation_embedding_dimension": 3072,
+        "recurrent_num_attention_heads": null,
+        "prelude_norm": true,
+        "sampling_scheme": "poisson-truncated-full",
+        "mean_recurrence": 27,
+        "mean_backprop_depth": 8,
+        "lockstep_n": false,
+        "lockstep_k": false,
+        "curriculum_target": "forward",
+        "recurrent_iteration_method": "per-sequence",
+        "tie_embeddings": true,
+        "model_class_name": "Parcae",
+        "_is_recurrent_block_config": false
+    },
+    "model_overwrite": {
+        "vocab_size": 32768,
+        "prelude_norm": true,
+        "dt_scale": 0.3,
+        "mean_recurrence": 27,
+        "mean_backprop_depth": 8
+    }
+}

special_token_ids.json

@@ -0,0 +1,16 @@
+{
+  "bos_id": 0,
+  "eos_id": 1,
+  "pad_id": 2,
+  "ignore_index": 2,
+  "add_bos": true,
+  "add_eos": true,
+  "vocab_size": 32768,
+  "tokenizer_source": "/datastor2/abao/lm_datasets/parcae_tokenizer_local",
+  "configs": [
+    "launch_configs/countdown-sft.yaml",
+    "launch_configs/countdown-sft-store-mid-n5v2.yaml",
+    "launch_configs/countdown-sft-140m-long100k.yaml"
+  ],
+  "model_overwrite": "{\"vocab_size\":32768,\"prelude_norm\":true,\"dt_scale\":0.3,\"mean_recurrence\":27,\"mean_backprop_depth\":8}"
+}

tokenizer/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "pad_token": "<|pad|>"
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,8 @@
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "model_max_length": 2048,
+  "clean_up_tokenization_spaces": false,
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "pad_token": "<|pad|>"
+}