Upload ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-03_21-51-50_CONFIG.json with huggingface_hub

ATHENA_V1_TINY_39.7M/checkpoints/RUN_001/RUN_001_DATETIME_2025-07-03_21-51-50_CONFIG.json

@@ -0,0 +1,112 @@
+{
+    "train_config": {
+        "vocab_size": 10000,
+        "context_length": 512,
+        "epochs": 5,
+        "checkpoint_steps": 1356,
+        "save_checkpoint_path": "main/checkpoints",
+        "save_hf": true,
+        "val_steps": 250,
+        "mixed_precision": false,
+        "max_grad_norm": 1.0,
+        "track_grad_norm": true,
+        "parallel_type": "fsdp",
+        "val_mixed_precision": false,
+        "val_mixed_precision_dtype": "f16",
+        "fsdp_wrap_policy": "transformer",
+        "wandb": true,
+        "log_level": "INFO",
+        "_compile": false,
+        "_compile_warmup_steps": null,
+        "hf_repo_config": {
+            "hf_repo_exists": true,
+            "hf_repo_id": "tiny-research/athena",
+            "hf_repo_type": "model",
+            "hf_root_path": "ATHENA_V1_TINY_39.7M/checkpoints"
+        },
+        "mixed_precision_dtype": "f16",
+        "log_root_path": "main/logs",
+        "load_checkpoint": false,
+        "load_checkpoint_path": null,
+        "extra_args": {}
+    },
+    "optimizer_config": {
+        "lr": 0.0005,
+        "betas": [
+            0.9,
+            0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 0.01,
+        "fused": true,
+        "extra_args": {}
+    },
+    "scheduler_config": {
+        "warmup_steps": 1000,
+        "constant_steps": 1000,
+        "decay_steps": 8852,
+        "max_lr": 0.0001,
+        "min_lr": 1e-05,
+        "extra_args": {}
+    },
+    "dataloader_config": {
+        "train_dataloader_config": {
+            "train_batch_size": 192,
+            "train_num_workers": 12,
+            "train_shuffle": true,
+            "train_pin_memory": true,
+            "train_data_root_path": "data/tensors/train"
+        },
+        "val_dataloader_config": {
+            "val_batch_size": 64,
+            "val_num_workers": 12,
+            "val_shuffle": false,
+            "val_pin_memory": true,
+            "val_data_root_path": "data/tensors/val"
+        },
+        "extra_args": {}
+    },
+    "model_config": {
+        "context_len": 512,
+        "d_model": 512,
+        "n_heads": 8,
+        "n_blocks": 12,
+        "vocab_size": 10000,
+        "pos_emb_dropout_p": 0.1,
+        "pos_emb_type": "rope",
+        "learned": false,
+        "ntk_rope_scaling": false,
+        "dyn_scaling": false,
+        "attn_type": "mhsa",
+        "n_groups": null,
+        "top_k_sparsev": null,
+        "p_threshold": null,
+        "p_threshold_steps_fraction": null,
+        "flash_attn": true,
+        "flash_attn_dtype": "float16",
+        "supress_warnings": true,
+        "verbose": null,
+        "model_name": "ATHENA_V1_TINY_39.7M",
+        "model_series_name": "ATHENA \n 39.7M",
+        "extra_args": {}
+    },
+    "criterion_config": {
+        "reduction": "none",
+        "ignore_index": 0,
+        "extra_args": {}
+    },
+    "wandb_config": {
+        "project": "ATHENA",
+        "name": "ATHENA_V1_TINY_39.7M",
+        "entity": "vxnuaj",
+        "tags": [
+            "ATHENA",
+            "39.7M",
+            "PRETRAIN",
+            "TINY",
+            "V1"
+        ],
+        "notes": "",
+        "extra_args": {}
+    }
+}