Add part2/mars (moved from th4kur/anlp-a2-p2-mars)

part2/mars/README.md

@@ -0,0 +1,19 @@
+---
+license: mit
+language: [en]
+tags: [language-modeling, optimizers, anlp-assignment-2]
+datasets: [browndw/human-ai-parallel-corpus]
+---
+# mars — Part 2 (optimizer: `mars`)
+
+Dense decoder-only transformer (size M, 27,269,632 parameters) pre-trained from scratch on the HAP-E corpus for one
+pass over the training split with a from-scratch `mars` implementation (peak lr 0.001, seed 0).
+
+| metric | value |
+|---|---|
+| validation loss (nats/token) | 3.8364 |
+| test loss | 3.8306 |
+| test continuation BLEU | 3.34 |
+
+Load with the assignment repo: `from src.part1.hub import load_model; model, cfg = load_model("th4kur/anlp-a2", subfolder="part2/mars")`
+(tokenizer: `tokenizer.json`).

part2/mars/config.json

@@ -0,0 +1,69 @@
+{
+  "model": {
+    "vocab_size": 16384,
+    "D_model": 512,
+    "N_ctx": 256,
+    "n_layers": 6,
+    "n_heads": 8,
+    "d_ff": 2048,
+    "use_moe": false,
+    "n_experts": 0,
+    "n_shared": 0,
+    "top_k": 0,
+    "expert_ff": 2048,
+    "aux_loss_coef": 0.01,
+    "rope_base": 10000.0,
+    "init_std": 0.02
+  },
+  "train": {
+    "name": "mars",
+    "optimizer": "mars",
+    "lr": 0.001,
+    "adam_lr": null,
+    "hp": {
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "gamma": 0.025,
+      "eps": 1e-08,
+      "weight_decay": 0.1,
+      "gnorm": 1.0
+    },
+    "seed": 0,
+    "size": "M",
+    "model_overrides": {},
+    "data_dir": "data/part2",
+    "max_len": 257,
+    "budget_frac": 1.0,
+    "tokens_per_step": 2048,
+    "micro_tokens": 2048,
+    "warmup_frac": 0.03,
+    "min_lr_frac": 0.1,
+    "eval_every_frac": 0.1,
+    "eval_at_start": true,
+    "eval_test_loss": true,
+    "eval_bleu": true,
+    "eval_micro_tokens": 8192,
+    "bleu_batch": 128,
+    "final_eval": true,
+    "log_every": 10,
+    "out_dir": "outputs/part2",
+    "project": "anlp-a2-part2",
+    "group": "part2-final",
+    "wandb_mode": null,
+    "tags": [],
+    "hf_namespace": "th4kur",
+    "hf_private": true,
+    "free_local_weights": true,
+    "push_frac": 0.5,
+    "push_tries": 3,
+    "push_retry_wait_s": 30.0
+  },
+  "params": {
+    "total": 27269632,
+    "active": 27269632,
+    "non_embedding_total": 18881024,
+    "non_embedding_active": 18881024
+  }
+}

part2/mars/metrics.json

@@ -0,0 +1,30 @@
+{
+  "name": "mars",
+  "optimizer": "mars",
+  "lr": 0.001,
+  "adam_lr": null,
+  "seed": 0,
+  "size": "M",
+  "params/total": 27269632,
+  "params/active": 27269632,
+  "params/non_embedding_total": 18881024,
+  "params/non_embedding_active": 18881024,
+  "tokens/seen": 39223033,
+  "tokens/per_epoch": 39223033,
+  "steps": 10767,
+  "train/wall_clock_s": 1316.6094388090005,
+  "train/tokens_per_sec_avg": 29790.940155708588,
+  "optimizer/state_bytes_per_param": 12.0,
+  "dataset_frac": 1.0,
+  "chinchilla_ratio": 0.10386892416428262,
+  "val/loss": 3.8364470608148284,
+  "val/ppl": 46.360465565372024,
+  "val/loss_human": 4.469063456690576,
+  "val/loss_llm": 3.621612244163736,
+  "test/loss": 3.8306220518981555,
+  "test/loss_human": 4.480073894734004,
+  "test/loss_llm": 3.6153188738089908,
+  "test/bleu": 3.3352745763902414,
+  "test/bleu_human": 1.3491476994165936,
+  "test/bleu_llm": 3.912919636037709
+}

part2/mars/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50bf4bdc40cdc831bc4996bf6e1907ffda6ad056c0ccc807305caa7debbf115
+size 109082080

part2/mars/samples_test.json

@@ -0,0 +1,22 @@
+[
+ {
+  "prompt_tail": " invader from all sides, he had lifted up his voice and yelled. Then, as by a conjuring-tr",
+  "hyp": "uth, he had been a man of great importance, and had been a man of great importance, and had been a man of great importance, and had been a man of great importance. He had been a man of great importance, and had been a man of great importance, and had been a man of great importance",
+  "ref": "ick, Blessington had appeared with her comforting presence that quite robbed the dark of its terrors. It must still have been early in the night, for she had not yet gone to bed, and had on above her smooth grey hair her cap with its adorable blue ribands in it. At her"
+ },
+ {
+  "prompt_tail": " comes to manifesting not I share in my manifestation master class, but for now, I just want to share seven manifest",
+  "hyp": "ing the truth. I want to share the truth with you. I want to share the truth with you. I want to share the truth with you. I want to share the truth with you. I want to share the truth with you. I want to share the truth with you. I want to share the truth",
+  "ref": "ing languages that I got from Sarah Prout who is an author and meditation teacher and the manifesting maven. All this really is is creating realities through different modalities. We aren't locked into one language right some of us might call it. Whoo, you know, I call it knowledge other people might call"
+ },
+ {
+  "prompt_tail": " like casual observes of their constitutional rights. A right to choose should mean something more than just paying an extra 49",
+  "hyp": "% of the time. It's not about the size of the food, but about the size of the food. It's about the size of the food, the size of the food, the size of the food. It's about the size of the food, the size of the food, the size of the food",
+  "ref": " cents to up-size your goddamn generic fatty-meal. But as I said. After ten, maybe twenty years of wolfing down dozens of varieties of the flavor \"salt\", we start panicking because all of the sudden we're all fucking fat and we can't understand"
+ },
+ {
+  "prompt_tail": " the closet that her captors held her in ... Oh Minnie. Come watch with us. Yeah. It's history",
+  "hyp": ". I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry. I'm sorry",
+  "ref": " in the making. Charlotte doesn't take her eyes off the TV. No thanks. She's not mentally ill! Fuck this guy. Just because she ran away from her bourgeoisie family and started over. I know how you feel, Patty! Charlotte sips her drink, Minnie leans in. My"
+ }
+]