Add p2-lion

p2-lion/config.json

@@ -0,0 +1,19 @@
+{
+  "vocab_size": 16000,
+  "d_model": 512,
+  "n_ctx": 512,
+  "n_layers": 8,
+  "n_heads": 8,
+  "dropout": 0.0,
+  "rope_base": 10000.0,
+  "tie_embeddings": true,
+  "use_moe": false,
+  "d_ff": 2048,
+  "n_experts": 4,
+  "top_k": 1,
+  "n_shared": 0,
+  "d_expert": 512,
+  "aux_loss_coef": 0.01,
+  "z_loss_coef": 0.001,
+  "extra": {}
+}

p2-lion/meta.json

@@ -0,0 +1,29 @@
+{
+  "run_name": "p2-lion",
+  "optim": {
+    "name": "lion",
+    "lr": 0.0003,
+    "betas": [
+      0.9,
+      0.99
+    ],
+    "weight_decay": 0.3
+  },
+  "final_val_loss": 4.4956134414672855,
+  "final_test_bleu": 0.6004444778731338,
+  "train_time_s": 1057.1355731487274,
+  "params": {
+    "total": 33366528,
+    "active": 33366528,
+    "ffn_total": 16777216
+  },
+  "data": {
+    "train_tokens": 39151438,
+    "validation_tokens": 2169606,
+    "train_docs": 59696,
+    "validation_docs": 3312,
+    "test_docs": 3312,
+    "bleu_docs": 414
+  },
+  "frac": 1.0
+}

p2-lion/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72593f1b73d6945809bba19c65da186cc6f036a5e82586f8f0911d565559bc62
+size 133470864