Upload folder using huggingface_hub

part2/mars/config.json

@@ -0,0 +1,54 @@
+{
+  "model": {
+    "d_model": 512,
+    "n_layers": 6,
+    "n_heads": 8,
+    "n_ctx": 256,
+    "d_ff": 1536,
+    "dropout": 0.0
+  },
+  "seed": 0,
+  "data": {
+    "name": "browndw/human-ai-parallel-corpus",
+    "fractions": {
+      "val": 0.05,
+      "test": 0.05
+    },
+    "seed": 0,
+    "limit": null
+  },
+  "tokenizer": {
+    "file": "unigram-16k.json",
+    "vocab_size": 16000
+  },
+  "train": {
+    "batch_size": 32,
+    "grad_clip": 0.0,
+    "warmup_fraction": 0.05,
+    "min_lr_ratio": 0.1,
+    "eval_every_fraction": 0.1,
+    "log_every": 20
+  },
+  "evaluate": {
+    "batch_size": 64,
+    "prompt_tokens": 128,
+    "reference_tokens": 64
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "anlp-assignment-2"
+  },
+  "name": "mars",
+  "optimizer": {
+    "name": "mars",
+    "lr": 0.004,
+    "betas": [
+      0.9,
+      0.95
+    ],
+    "gamma": 0.025,
+    "eps": 1e-10,
+    "weight_decay": 0.1,
+    "grad_norm": 1.0
+  }
+}

part2/mars/metrics.jsonl

@@ -0,0 +1,67 @@
+{"step": 0, "tokens": 0, "kind": "eval", "eval_seconds": 27.44230055809021, "val_loss": 9.769027694883082, "val_perplexity": 17483.759402580894, "dataset_fraction": 0.0, "test_bleu": 0.02477986831865296}
+{"step": 20, "tokens": 163840, "kind": "train", "loss": 8.306053757667542, "aux_loss": 0.0, "lr_factor": 0.33694694020338606, "tokens_per_second": 4210.119365818618, "epoch": 0}
+{"step": 40, "tokens": 327680, "kind": "train", "loss": 7.293674659729004, "aux_loss": 0.0, "lr_factor": 0.6916256511679459, "tokens_per_second": 16879.047424982477, "epoch": 0}
+{"step": 60, "tokens": 491520, "kind": "train", "loss": 6.862284922599793, "aux_loss": 0.0, "lr_factor": 0.9999868120848697, "tokens_per_second": 16850.83555622035, "epoch": 0}
+{"step": 80, "tokens": 655360, "kind": "train", "loss": 6.6230392217636105, "aux_loss": 0.0, "lr_factor": 0.999011302681498, "tokens_per_second": 16836.42835384905, "epoch": 0}
+{"step": 100, "tokens": 819200, "kind": "train", "loss": 6.43754813671112, "aux_loss": 0.0, "lr_factor": 0.9964919771069799, "tokens_per_second": 16809.588822144953, "epoch": 0}
+{"step": 113, "tokens": 925696, "kind": "eval", "eval_seconds": 27.133575677871704, "val_loss": 6.294938824393532, "val_perplexity": 541.8226990469889, "dataset_fraction": 0.10019673584748816, "test_bleu": 0.27667881427287944}
+{"step": 120, "tokens": 983040, "kind": "train", "loss": 6.329708504676819, "aux_loss": 0.0, "lr_factor": 0.9924374974496396, "tokens_per_second": 3772.4031640558824, "epoch": 0}
+{"step": 140, "tokens": 1146880, "kind": "train", "loss": 6.189028644561768, "aux_loss": 0.0, "lr_factor": 0.9868618040517874, "tokens_per_second": 16838.38670168973, "epoch": 0}
+{"step": 160, "tokens": 1310720, "kind": "train", "loss": 6.159878754615784, "aux_loss": 0.0, "lr_factor": 0.979784067579243, "tokens_per_second": 16782.96752624917, "epoch": 0}
+{"step": 180, "tokens": 1474560, "kind": "train", "loss": 6.1087634563446045, "aux_loss": 0.0, "lr_factor": 0.9712286231076629, "tokens_per_second": 16803.383906475643, "epoch": 0}
+{"step": 200, "tokens": 1638400, "kind": "train", "loss": 6.0258612632751465, "aux_loss": 0.0, "lr_factor": 0.9612248864523011, "tokens_per_second": 16812.411239603298, "epoch": 0}
+{"step": 220, "tokens": 1802240, "kind": "train", "loss": 6.004348373413086, "aux_loss": 0.0, "lr_factor": 0.9498072530288814, "tokens_per_second": 16793.43872420506, "epoch": 0}
+{"step": 226, "tokens": 1851392, "kind": "eval", "eval_seconds": 27.300244569778442, "val_loss": 5.960691815779614, "val_perplexity": 387.8783718163244, "dataset_fraction": 0.20039347169497632, "test_bleu": 0.4456650364387113}
+{"step": 240, "tokens": 1966080, "kind": "train", "loss": 5.937868237495422, "aux_loss": 0.0, "lr_factor": 0.9370149795933251, "tokens_per_second": 3791.0808547601923, "epoch": 0}
+{"step": 260, "tokens": 2129920, "kind": "train", "loss": 5.949131917953491, "aux_loss": 0.0, "lr_factor": 0.9228920492669425, "tokens_per_second": 16772.705878355853, "epoch": 0}
+{"step": 280, "tokens": 2293760, "kind": "train", "loss": 5.888448548316956, "aux_loss": 0.0, "lr_factor": 0.9074870203111662, "tokens_per_second": 16770.02937095829, "epoch": 0}
+{"step": 300, "tokens": 2457600, "kind": "train", "loss": 5.846836400032044, "aux_loss": 0.0, "lr_factor": 0.8908528591717783, "tokens_per_second": 16790.640313696116, "epoch": 0}
+{"step": 320, "tokens": 2621440, "kind": "train", "loss": 5.817968821525573, "aux_loss": 0.0, "lr_factor": 0.8730467583666665, "tokens_per_second": 16800.41788570656, "epoch": 0}
+{"step": 339, "tokens": 2777088, "kind": "eval", "eval_seconds": 27.37646245956421, "val_loss": 5.780085669204651, "val_perplexity": 323.7869277976871, "dataset_fraction": 0.30059020754246446, "test_bleu": 0.6361282128923249}
+{"step": 340, "tokens": 2785280, "kind": "train", "loss": 5.809471154212952, "aux_loss": 0.0, "lr_factor": 0.85412993984326, "tokens_per_second": 3772.499276360288, "epoch": 0}
+{"step": 360, "tokens": 2949120, "kind": "train", "loss": 5.785085439682007, "aux_loss": 0.0, "lr_factor": 0.8341674444817523, "tokens_per_second": 16818.42689437698, "epoch": 0}
+{"step": 380, "tokens": 3112960, "kind": "train", "loss": 5.764096903800964, "aux_loss": 0.0, "lr_factor": 0.8132279084678504, "tokens_per_second": 16812.3149911994, "epoch": 0}
+{"step": 400, "tokens": 3276800, "kind": "train", "loss": 5.728936243057251, "aux_loss": 0.0, "lr_factor": 0.7913833273039428, "tokens_per_second": 16791.432554921194, "epoch": 0}
+{"step": 420, "tokens": 3440640, "kind": "train", "loss": 5.719450616836548, "aux_loss": 0.0, "lr_factor": 0.7687088082700718, "tokens_per_second": 16761.72041149654, "epoch": 0}
+{"step": 440, "tokens": 3604480, "kind": "train", "loss": 5.66547658443451, "aux_loss": 0.0, "lr_factor": 0.7452823121858171, "tokens_per_second": 16761.805042575725, "epoch": 0}
+{"step": 452, "tokens": 3702784, "kind": "eval", "eval_seconds": 27.338379383087158, "val_loss": 5.646855951769079, "val_perplexity": 283.39904338845673, "dataset_fraction": 0.40078694338995263, "test_bleu": 0.6161190050087185}
+{"step": 460, "tokens": 3768320, "kind": "train", "loss": 5.664773559570312, "aux_loss": 0.0, "lr_factor": 0.7211843853609754, "tokens_per_second": 3758.4106282361367, "epoch": 0}
+{"step": 480, "tokens": 3932160, "kind": "train", "loss": 5.625240468978882, "aux_loss": 0.0, "lr_factor": 0.696497882656663, "tokens_per_second": 16806.72131534042, "epoch": 0}
+{"step": 500, "tokens": 4096000, "kind": "train", "loss": 5.629225945472717, "aux_loss": 0.0, "lr_factor": 0.6713076826090247, "tokens_per_second": 16794.595700389462, "epoch": 0}
+{"step": 520, "tokens": 4259840, "kind": "train", "loss": 5.605629730224609, "aux_loss": 0.0, "lr_factor": 0.645700395595028, "tokens_per_second": 16790.659595738336, "epoch": 0}
+{"step": 540, "tokens": 4423680, "kind": "train", "loss": 5.591954755783081, "aux_loss": 0.0, "lr_factor": 0.6197640660437446, "tokens_per_second": 16782.02362366211, "epoch": 0}
+{"step": 560, "tokens": 4587520, "kind": "train", "loss": 5.5699930667877195, "aux_loss": 0.0, "lr_factor": 0.5935878697169924, "tokens_per_second": 16770.009317773158, "epoch": 0}
+{"step": 564, "tokens": 4620288, "kind": "eval", "eval_seconds": 27.293628454208374, "val_loss": 5.547484450660676, "val_perplexity": 256.59127535807363, "dataset_fraction": 0.5000969824600294, "test_bleu": 0.6350725756391062}
+{"step": 580, "tokens": 4751360, "kind": "train", "loss": 5.5332095384597775, "aux_loss": 0.0, "lr_factor": 0.5672618071001645, "tokens_per_second": 3790.949934884202, "epoch": 0}
+{"step": 600, "tokens": 4915200, "kind": "train", "loss": 5.5118821144104, "aux_loss": 0.0, "lr_factor": 0.5408763939574474, "tokens_per_second": 16806.49771149128, "epoch": 0}
+{"step": 620, "tokens": 5079040, "kind": "train", "loss": 5.512618613243103, "aux_loss": 0.0, "lr_factor": 0.5145223501153768, "tokens_per_second": 16818.836872633885, "epoch": 0}
+{"step": 640, "tokens": 5242880, "kind": "train", "loss": 5.507871794700622, "aux_loss": 0.0, "lr_factor": 0.4882902875447779, "tokens_per_second": 16788.437529362276, "epoch": 0}
+{"step": 660, "tokens": 5406720, "kind": "train", "loss": 5.486001205444336, "aux_loss": 0.0, "lr_factor": 0.46227039881353715, "tokens_per_second": 16764.278937184437, "epoch": 0}
+{"step": 677, "tokens": 5545984, "kind": "eval", "eval_seconds": 27.263744115829468, "val_loss": 5.451136651246444, "val_perplexity": 233.02288118303446, "dataset_fraction": 0.6002937183075175, "test_bleu": 0.6252807028223081}
+{"step": 680, "tokens": 5570560, "kind": "train", "loss": 5.441760516166687, "aux_loss": 0.0, "lr_factor": 0.43655214698139233, "tokens_per_second": 3778.686909497318, "epoch": 0}
+{"step": 700, "tokens": 5734400, "kind": "train", "loss": 5.447092461585998, "aux_loss": 0.0, "lr_factor": 0.4112239580029574, "tokens_per_second": 16785.685476605104, "epoch": 0}
+{"step": 720, "tokens": 5898240, "kind": "train", "loss": 5.41042857170105, "aux_loss": 0.0, "lr_factor": 0.3863729166965816, "tokens_per_second": 16826.183233094926, "epoch": 0}
+{"step": 740, "tokens": 6062080, "kind": "train", "loss": 5.422480678558349, "aux_loss": 0.0, "lr_factor": 0.36208446732437927, "tokens_per_second": 16815.417706035583, "epoch": 0}
+{"step": 760, "tokens": 6225920, "kind": "train", "loss": 5.398648762702942, "aux_loss": 0.0, "lr_factor": 0.3384421198129132, "tokens_per_second": 16775.585131347827, "epoch": 0}
+{"step": 780, "tokens": 6389760, "kind": "train", "loss": 5.385588312149048, "aux_loss": 0.0, "lr_factor": 0.31552716262462127, "tokens_per_second": 16758.36406784292, "epoch": 0}
+{"step": 790, "tokens": 6471680, "kind": "eval", "eval_seconds": 27.26255512237549, "val_loss": 5.375435691577173, "val_perplexity": 216.03397599275786, "dataset_fraction": 0.7004904541550057, "test_bleu": 0.6234028809293407}
+{"step": 800, "tokens": 6553600, "kind": "train", "loss": 5.363969850540161, "aux_loss": 0.0, "lr_factor": 0.2934183832672054, "tokens_per_second": 3747.5851688296702, "epoch": 0}
+{"step": 820, "tokens": 6717440, "kind": "train", "loss": 5.371731472015381, "aux_loss": 0.0, "lr_factor": 0.27219179740194743, "tokens_per_second": 16795.277894432907, "epoch": 0}
+{"step": 840, "tokens": 6881280, "kind": "train", "loss": 5.3678457021713255, "aux_loss": 0.0, "lr_factor": 0.25192038748234297, "tokens_per_second": 16812.824626689206, "epoch": 0}
+{"step": 860, "tokens": 7045120, "kind": "train", "loss": 5.355131363868713, "aux_loss": 0.0, "lr_factor": 0.23267385182168396, "tokens_per_second": 16803.52771526059, "epoch": 0}
+{"step": 880, "tokens": 7208960, "kind": "train", "loss": 5.33197865486145, "aux_loss": 0.0, "lr_factor": 0.2145183649523572, "tokens_per_second": 16761.483694320603, "epoch": 0}
+{"step": 900, "tokens": 7372800, "kind": "train", "loss": 5.331108903884887, "aux_loss": 0.0, "lr_factor": 0.1975163501008086, "tokens_per_second": 16769.65819018089, "epoch": 0}
+{"step": 903, "tokens": 7397376, "kind": "eval", "eval_seconds": 27.32157278060913, "val_loss": 5.307742224380433, "val_perplexity": 201.89388233323126, "dataset_fraction": 0.8006871900024939, "test_bleu": 0.7753154935950357}
+{"step": 920, "tokens": 7536640, "kind": "train", "loss": 5.296346044540405, "aux_loss": 0.0, "lr_factor": 0.18172626456045823, "tokens_per_second": 3796.727388118689, "epoch": 0}
+{"step": 940, "tokens": 7700480, "kind": "train", "loss": 5.303777384757995, "aux_loss": 0.0, "lr_factor": 0.16720239870051368, "tokens_per_second": 16796.053742013046, "epoch": 0}
+{"step": 960, "tokens": 7864320, "kind": "train", "loss": 5.261468553543091, "aux_loss": 0.0, "lr_factor": 0.1539946893017382, "tokens_per_second": 16794.41797782817, "epoch": 0}
+{"step": 980, "tokens": 8028160, "kind": "train", "loss": 5.270877385139466, "aux_loss": 0.0, "lr_factor": 0.14214854786097333, "tokens_per_second": 16806.411806447457, "epoch": 0}
+{"step": 1000, "tokens": 8192000, "kind": "train", "loss": 5.270149993896484, "aux_loss": 0.0, "lr_factor": 0.13170470445475124, "tokens_per_second": 16762.00006527278, "epoch": 0}
+{"step": 1016, "tokens": 8323072, "kind": "eval", "eval_seconds": 27.33437418937683, "val_loss": 5.260519317958666, "val_perplexity": 192.58147635215735, "dataset_fraction": 0.900883925849982, "test_bleu": 0.6158698931109332}
+{"step": 1020, "tokens": 8355840, "kind": "train", "loss": 5.262749409675598, "aux_loss": 0.0, "lr_factor": 0.12269906769883077, "tokens_per_second": 3778.869327622783, "epoch": 0}
+{"step": 1040, "tokens": 8519680, "kind": "train", "loss": 5.283662724494934, "aux_loss": 0.0, "lr_factor": 0.11516260128515528, "tokens_per_second": 16802.364578194112, "epoch": 0}
+{"step": 1060, "tokens": 8683520, "kind": "train", "loss": 5.261514925956726, "aux_loss": 0.0, "lr_factor": 0.10912121752072734, "tokens_per_second": 16809.517687920317, "epoch": 0}
+{"step": 1080, "tokens": 8847360, "kind": "train", "loss": 5.245190525054932, "aux_loss": 0.0, "lr_factor": 0.10459568823444292, "tokens_per_second": 16784.28990616822, "epoch": 0}
+{"step": 1100, "tokens": 9011200, "kind": "train", "loss": 5.249981927871704, "aux_loss": 0.0, "lr_factor": 0.10160157335820881, "tokens_per_second": 16734.30248280788, "epoch": 0}
+{"step": 1120, "tokens": 9175040, "kind": "train", "loss": 5.221726107597351, "aux_loss": 0.0, "lr_factor": 0.10014916742790117, "tokens_per_second": 16773.912407436746, "epoch": 0}
+{"step": 1128, "tokens": 9238784, "kind": "eval", "eval_seconds": 27.34925866127014, "val_loss": 5.229855695732026, "val_perplexity": 186.76585046616051, "dataset_fraction": 1.0, "test_bleu": 0.713095208361588}

part2/mars/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2872c07baedcc145e9ac208f292b089f236a688a20e5a50caa9d5c06d88dd72b
+size 147370362