Upload folder using huggingface_hub

part2/mars-half/config.json

@@ -0,0 +1,54 @@
+{
+  "model": {
+    "d_model": 512,
+    "n_layers": 6,
+    "n_heads": 8,
+    "n_ctx": 256,
+    "d_ff": 1536,
+    "dropout": 0.0
+  },
+  "seed": 0,
+  "data": {
+    "name": "browndw/human-ai-parallel-corpus",
+    "fractions": {
+      "val": 0.05,
+      "test": 0.05
+    },
+    "seed": 0,
+    "limit": null
+  },
+  "tokenizer": {
+    "file": "unigram-16k.json",
+    "vocab_size": 16000
+  },
+  "train": {
+    "batch_size": 32,
+    "grad_clip": 0.0,
+    "warmup_fraction": 0.05,
+    "min_lr_ratio": 0.1,
+    "eval_every_fraction": 0.1,
+    "log_every": 20
+  },
+  "evaluate": {
+    "batch_size": 64,
+    "prompt_tokens": 128,
+    "reference_tokens": 64
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "anlp-assignment-2"
+  },
+  "name": "mars-half",
+  "optimizer": {
+    "name": "mars",
+    "lr": 0.002,
+    "betas": [
+      0.9,
+      0.95
+    ],
+    "gamma": 0.025,
+    "eps": 1e-10,
+    "weight_decay": 0.1,
+    "grad_norm": 1.0
+  }
+}

part2/mars-half/metrics.jsonl

@@ -0,0 +1,67 @@
+{"step": 0, "tokens": 0, "kind": "eval", "eval_seconds": 5.159359693527222, "val_loss": 9.76902662435539, "val_perplexity": 17483.740685742323, "dataset_fraction": 0.0, "test_bleu": 0.024831537652001516}
+{"step": 20, "tokens": 163840, "kind": "train", "loss": 8.557413816452026, "aux_loss": 0.0, "lr_factor": 0.33694694020338606, "tokens_per_second": 19106.98008877362, "epoch": 0}
+{"step": 40, "tokens": 327680, "kind": "train", "loss": 7.320798850059509, "aux_loss": 0.0, "lr_factor": 0.6916256511679459, "tokens_per_second": 107182.89442810035, "epoch": 0}
+{"step": 60, "tokens": 491520, "kind": "train", "loss": 6.871130108833313, "aux_loss": 0.0, "lr_factor": 0.9999868120848697, "tokens_per_second": 107085.8871132872, "epoch": 0}
+{"step": 80, "tokens": 655360, "kind": "train", "loss": 6.587843608856201, "aux_loss": 0.0, "lr_factor": 0.999011302681498, "tokens_per_second": 106711.12283410254, "epoch": 0}
+{"step": 100, "tokens": 819200, "kind": "train", "loss": 6.385958909988403, "aux_loss": 0.0, "lr_factor": 0.9964919771069799, "tokens_per_second": 106763.04736248055, "epoch": 0}
+{"step": 113, "tokens": 925696, "kind": "eval", "eval_seconds": 4.615929841995239, "val_loss": 6.2247976314408975, "val_perplexity": 505.12081154221175, "dataset_fraction": 0.10019673584748816, "test_bleu": 0.31804382599605047}
+{"step": 120, "tokens": 983040, "kind": "train", "loss": 6.265087223052978, "aux_loss": 0.0, "lr_factor": 0.9924374974496396, "tokens_per_second": 12991.597657290085, "epoch": 0}
+{"step": 140, "tokens": 1146880, "kind": "train", "loss": 6.105484986305237, "aux_loss": 0.0, "lr_factor": 0.9868618040517874, "tokens_per_second": 106879.46408063675, "epoch": 0}
+{"step": 160, "tokens": 1310720, "kind": "train", "loss": 6.071258020401001, "aux_loss": 0.0, "lr_factor": 0.979784067579243, "tokens_per_second": 106793.49278299249, "epoch": 0}
+{"step": 180, "tokens": 1474560, "kind": "train", "loss": 6.012303209304809, "aux_loss": 0.0, "lr_factor": 0.9712286231076629, "tokens_per_second": 106728.59115789134, "epoch": 0}
+{"step": 200, "tokens": 1638400, "kind": "train", "loss": 5.922639966011047, "aux_loss": 0.0, "lr_factor": 0.9612248864523011, "tokens_per_second": 106670.12623245137, "epoch": 0}
+{"step": 220, "tokens": 1802240, "kind": "train", "loss": 5.893012428283692, "aux_loss": 0.0, "lr_factor": 0.9498072530288814, "tokens_per_second": 106398.1786078578, "epoch": 0}
+{"step": 226, "tokens": 1851392, "kind": "eval", "eval_seconds": 4.617531061172485, "val_loss": 5.839202262667328, "val_perplexity": 343.50520440961736, "dataset_fraction": 0.20039347169497632, "test_bleu": 0.5057114089488732}
+{"step": 240, "tokens": 1966080, "kind": "train", "loss": 5.821803998947144, "aux_loss": 0.0, "lr_factor": 0.9370149795933251, "tokens_per_second": 12907.849543592658, "epoch": 0}
+{"step": 260, "tokens": 2129920, "kind": "train", "loss": 5.829473328590393, "aux_loss": 0.0, "lr_factor": 0.9228920492669425, "tokens_per_second": 106688.3596182726, "epoch": 0}
+{"step": 280, "tokens": 2293760, "kind": "train", "loss": 5.758775520324707, "aux_loss": 0.0, "lr_factor": 0.9074870203111662, "tokens_per_second": 106604.6798383605, "epoch": 0}
+{"step": 300, "tokens": 2457600, "kind": "train", "loss": 5.7181058645248415, "aux_loss": 0.0, "lr_factor": 0.8908528591717783, "tokens_per_second": 106491.18742452233, "epoch": 0}
+{"step": 320, "tokens": 2621440, "kind": "train", "loss": 5.682852959632873, "aux_loss": 0.0, "lr_factor": 0.8730467583666665, "tokens_per_second": 106272.5676062293, "epoch": 0}
+{"step": 339, "tokens": 2777088, "kind": "eval", "eval_seconds": 4.635813474655151, "val_loss": 5.639590691200829, "val_perplexity": 281.34753688199754, "dataset_fraction": 0.30059020754246446, "test_bleu": 0.491543956566813}
+{"step": 340, "tokens": 2785280, "kind": "train", "loss": 5.67443196773529, "aux_loss": 0.0, "lr_factor": 0.85412993984326, "tokens_per_second": 12875.625241431522, "epoch": 0}
+{"step": 360, "tokens": 2949120, "kind": "train", "loss": 5.644741439819336, "aux_loss": 0.0, "lr_factor": 0.8341674444817523, "tokens_per_second": 106200.86200149968, "epoch": 0}
+{"step": 380, "tokens": 3112960, "kind": "train", "loss": 5.617793345451355, "aux_loss": 0.0, "lr_factor": 0.8132279084678504, "tokens_per_second": 106738.13983144503, "epoch": 0}
+{"step": 400, "tokens": 3276800, "kind": "train", "loss": 5.583799481391907, "aux_loss": 0.0, "lr_factor": 0.7913833273039428, "tokens_per_second": 106643.07745214216, "epoch": 0}
+{"step": 420, "tokens": 3440640, "kind": "train", "loss": 5.569909262657165, "aux_loss": 0.0, "lr_factor": 0.7687088082700718, "tokens_per_second": 106339.33451991108, "epoch": 0}
+{"step": 440, "tokens": 3604480, "kind": "train", "loss": 5.517045879364014, "aux_loss": 0.0, "lr_factor": 0.7452823121858171, "tokens_per_second": 105269.0159094986, "epoch": 0}
+{"step": 452, "tokens": 3702784, "kind": "eval", "eval_seconds": 4.641818046569824, "val_loss": 5.492206712956485, "val_perplexity": 242.79238925491802, "dataset_fraction": 0.40078694338995263, "test_bleu": 0.5018909340710206}
+{"step": 460, "tokens": 3768320, "kind": "train", "loss": 5.510415768623352, "aux_loss": 0.0, "lr_factor": 0.7211843853609754, "tokens_per_second": 13213.006268688605, "epoch": 0}
+{"step": 480, "tokens": 3932160, "kind": "train", "loss": 5.470386910438537, "aux_loss": 0.0, "lr_factor": 0.696497882656663, "tokens_per_second": 106796.18144175322, "epoch": 0}
+{"step": 500, "tokens": 4096000, "kind": "train", "loss": 5.469701075553894, "aux_loss": 0.0, "lr_factor": 0.6713076826090247, "tokens_per_second": 106657.11329517736, "epoch": 0}
+{"step": 520, "tokens": 4259840, "kind": "train", "loss": 5.447249221801758, "aux_loss": 0.0, "lr_factor": 0.645700395595028, "tokens_per_second": 106569.63163566556, "epoch": 0}
+{"step": 540, "tokens": 4423680, "kind": "train", "loss": 5.429160165786743, "aux_loss": 0.0, "lr_factor": 0.6197640660437446, "tokens_per_second": 105846.24227730109, "epoch": 0}
+{"step": 560, "tokens": 4587520, "kind": "train", "loss": 5.407112455368042, "aux_loss": 0.0, "lr_factor": 0.5935878697169924, "tokens_per_second": 104748.56150412276, "epoch": 0}
+{"step": 564, "tokens": 4620288, "kind": "eval", "eval_seconds": 4.6409924030303955, "val_loss": 5.380170371692643, "val_perplexity": 217.05925302723165, "dataset_fraction": 0.5000969824600294, "test_bleu": 0.6745106309283598}
+{"step": 580, "tokens": 4751360, "kind": "train", "loss": 5.372203803062439, "aux_loss": 0.0, "lr_factor": 0.5672618071001645, "tokens_per_second": 12958.403572865742, "epoch": 0}
+{"step": 600, "tokens": 4915200, "kind": "train", "loss": 5.343112802505493, "aux_loss": 0.0, "lr_factor": 0.5408763939574474, "tokens_per_second": 106724.76222187986, "epoch": 0}
+{"step": 620, "tokens": 5079040, "kind": "train", "loss": 5.345380353927612, "aux_loss": 0.0, "lr_factor": 0.5145223501153768, "tokens_per_second": 106631.89114774774, "epoch": 0}
+{"step": 640, "tokens": 5242880, "kind": "train", "loss": 5.335080981254578, "aux_loss": 0.0, "lr_factor": 0.4882902875447779, "tokens_per_second": 106212.9594336667, "epoch": 0}
+{"step": 660, "tokens": 5406720, "kind": "train", "loss": 5.315628504753112, "aux_loss": 0.0, "lr_factor": 0.46227039881353715, "tokens_per_second": 105674.6857776506, "epoch": 0}
+{"step": 677, "tokens": 5545984, "kind": "eval", "eval_seconds": 4.632675409317017, "val_loss": 5.282425171772953, "val_perplexity": 196.84668396826459, "dataset_fraction": 0.6002937183075175, "test_bleu": 0.6124747705492068}
+{"step": 680, "tokens": 5570560, "kind": "train", "loss": 5.272585248947143, "aux_loss": 0.0, "lr_factor": 0.43655214698139233, "tokens_per_second": 12673.567924976627, "epoch": 0}
+{"step": 700, "tokens": 5734400, "kind": "train", "loss": 5.275763940811157, "aux_loss": 0.0, "lr_factor": 0.4112239580029574, "tokens_per_second": 106987.28961393361, "epoch": 0}
+{"step": 720, "tokens": 5898240, "kind": "train", "loss": 5.240960264205933, "aux_loss": 0.0, "lr_factor": 0.3863729166965816, "tokens_per_second": 106683.85451452965, "epoch": 0}
+{"step": 740, "tokens": 6062080, "kind": "train", "loss": 5.255446362495422, "aux_loss": 0.0, "lr_factor": 0.36208446732437927, "tokens_per_second": 106660.49039365347, "epoch": 0}
+{"step": 760, "tokens": 6225920, "kind": "train", "loss": 5.222750639915466, "aux_loss": 0.0, "lr_factor": 0.3384421198129132, "tokens_per_second": 106270.69408034324, "epoch": 0}
+{"step": 780, "tokens": 6389760, "kind": "train", "loss": 5.21322660446167, "aux_loss": 0.0, "lr_factor": 0.31552716262462127, "tokens_per_second": 105143.11129281111, "epoch": 0}
+{"step": 790, "tokens": 6471680, "kind": "eval", "eval_seconds": 4.634413719177246, "val_loss": 5.203483915140506, "val_perplexity": 181.90488037269748, "dataset_fraction": 0.7004904541550057, "test_bleu": 0.6996609366581781}
+{"step": 800, "tokens": 6553600, "kind": "train", "loss": 5.190594172477722, "aux_loss": 0.0, "lr_factor": 0.2934183832672054, "tokens_per_second": 12988.926220239242, "epoch": 0}
+{"step": 820, "tokens": 6717440, "kind": "train", "loss": 5.198034763336182, "aux_loss": 0.0, "lr_factor": 0.27219179740194743, "tokens_per_second": 106455.38922273497, "epoch": 0}
+{"step": 840, "tokens": 6881280, "kind": "train", "loss": 5.195906090736389, "aux_loss": 0.0, "lr_factor": 0.25192038748234297, "tokens_per_second": 106632.30480064712, "epoch": 0}
+{"step": 860, "tokens": 7045120, "kind": "train", "loss": 5.188484644889831, "aux_loss": 0.0, "lr_factor": 0.23267385182168396, "tokens_per_second": 106648.68803577911, "epoch": 0}
+{"step": 880, "tokens": 7208960, "kind": "train", "loss": 5.158581495285034, "aux_loss": 0.0, "lr_factor": 0.2145183649523572, "tokens_per_second": 105832.59831479708, "epoch": 0}
+{"step": 900, "tokens": 7372800, "kind": "train", "loss": 5.161272668838501, "aux_loss": 0.0, "lr_factor": 0.1975163501008086, "tokens_per_second": 104447.49822018406, "epoch": 0}
+{"step": 903, "tokens": 7397376, "kind": "eval", "eval_seconds": 4.642587184906006, "val_loss": 5.138879172886784, "val_perplexity": 170.52453265061726, "dataset_fraction": 0.8006871900024939, "test_bleu": 0.6030571054674874}
+{"step": 920, "tokens": 7536640, "kind": "train", "loss": 5.124852085113526, "aux_loss": 0.0, "lr_factor": 0.18172626456045823, "tokens_per_second": 12864.638715155766, "epoch": 0}
+{"step": 940, "tokens": 7700480, "kind": "train", "loss": 5.136901950836181, "aux_loss": 0.0, "lr_factor": 0.16720239870051368, "tokens_per_second": 106742.84848691015, "epoch": 0}
+{"step": 960, "tokens": 7864320, "kind": "train", "loss": 5.090995359420776, "aux_loss": 0.0, "lr_factor": 0.1539946893017382, "tokens_per_second": 106606.84630905114, "epoch": 0}
+{"step": 980, "tokens": 8028160, "kind": "train", "loss": 5.1030231475830075, "aux_loss": 0.0, "lr_factor": 0.14214854786097333, "tokens_per_second": 106045.95679119034, "epoch": 0}
+{"step": 1000, "tokens": 8192000, "kind": "train", "loss": 5.1001627206802365, "aux_loss": 0.0, "lr_factor": 0.13170470445475124, "tokens_per_second": 104874.40269208171, "epoch": 0}
+{"step": 1016, "tokens": 8323072, "kind": "eval", "eval_seconds": 4.647707462310791, "val_loss": 5.092280646086682, "val_perplexity": 162.76063850111402, "dataset_fraction": 0.900883925849982, "test_bleu": 0.5537054802078557}
+{"step": 1020, "tokens": 8355840, "kind": "train", "loss": 5.089758825302124, "aux_loss": 0.0, "lr_factor": 0.12269906769883077, "tokens_per_second": 12912.606045678578, "epoch": 0}
+{"step": 1040, "tokens": 8519680, "kind": "train", "loss": 5.112429690361023, "aux_loss": 0.0, "lr_factor": 0.11516260128515528, "tokens_per_second": 106896.20603622832, "epoch": 0}
+{"step": 1060, "tokens": 8683520, "kind": "train", "loss": 5.090072250366211, "aux_loss": 0.0, "lr_factor": 0.10912121752072734, "tokens_per_second": 106667.72538812089, "epoch": 0}
+{"step": 1080, "tokens": 8847360, "kind": "train", "loss": 5.075993251800537, "aux_loss": 0.0, "lr_factor": 0.10459568823444292, "tokens_per_second": 106589.68233799237, "epoch": 0}
+{"step": 1100, "tokens": 9011200, "kind": "train", "loss": 5.080230784416199, "aux_loss": 0.0, "lr_factor": 0.10160157335820881, "tokens_per_second": 105488.33124079445, "epoch": 0}
+{"step": 1120, "tokens": 9175040, "kind": "train", "loss": 5.049615454673767, "aux_loss": 0.0, "lr_factor": 0.10014916742790117, "tokens_per_second": 104577.42263387975, "epoch": 0}
+{"step": 1128, "tokens": 9238784, "kind": "eval", "eval_seconds": 4.648599624633789, "val_loss": 5.062403151169125, "val_perplexity": 157.9696855749481, "dataset_fraction": 1.0, "test_bleu": 0.6773226735276665}

part2/mars-half/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9995f2b9da542fe67a6955c338a5f35c6085b1c6f07b3d2c214279c734a195ae
+size 147370362