Upload folder using huggingface_hub

part2/adamw/config.json

@@ -0,0 +1,53 @@
+{
+  "model": {
+    "d_model": 512,
+    "n_layers": 6,
+    "n_heads": 8,
+    "n_ctx": 256,
+    "d_ff": 1536,
+    "dropout": 0.0
+  },
+  "seed": 0,
+  "data": {
+    "name": "browndw/human-ai-parallel-corpus",
+    "fractions": {
+      "val": 0.05,
+      "test": 0.05
+    },
+    "seed": 0,
+    "limit": null
+  },
+  "tokenizer": {
+    "file": "unigram-16k.json",
+    "vocab_size": 16000
+  },
+  "train": {
+    "batch_size": 32,
+    "grad_clip": 0.0,
+    "warmup_fraction": 0.05,
+    "min_lr_ratio": 0.1,
+    "eval_every_fraction": 0.1,
+    "log_every": 20
+  },
+  "evaluate": {
+    "batch_size": 64,
+    "prompt_tokens": 128,
+    "reference_tokens": 64
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "anlp-assignment-2"
+  },
+  "name": "adamw",
+  "optimizer": {
+    "name": "adamw",
+    "lr": 0.002,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "eps": 1e-10,
+    "weight_decay": 0.1,
+    "grad_norm": 1.0
+  }
+}

part2/adamw/metrics.jsonl

@@ -0,0 +1,67 @@
+{"step": 0, "tokens": 0, "kind": "eval", "eval_seconds": 27.136680364608765, "val_loss": 9.769027694883082, "val_perplexity": 17483.759402580894, "dataset_fraction": 0.0, "test_bleu": 0.02477986831865296}
+{"step": 20, "tokens": 163840, "kind": "train", "loss": 8.539702153205871, "aux_loss": 0.0, "lr_factor": 0.33694694020338606, "tokens_per_second": 4252.270724040367, "epoch": 0}
+{"step": 40, "tokens": 327680, "kind": "train", "loss": 7.467249512672424, "aux_loss": 0.0, "lr_factor": 0.6916256511679459, "tokens_per_second": 17098.40298804332, "epoch": 0}
+{"step": 60, "tokens": 491520, "kind": "train", "loss": 6.969194602966309, "aux_loss": 0.0, "lr_factor": 0.9999868120848697, "tokens_per_second": 17016.87264395803, "epoch": 0}
+{"step": 80, "tokens": 655360, "kind": "train", "loss": 6.692301321029663, "aux_loss": 0.0, "lr_factor": 0.999011302681498, "tokens_per_second": 16994.414239726142, "epoch": 0}
+{"step": 100, "tokens": 819200, "kind": "train", "loss": 6.4899917364120485, "aux_loss": 0.0, "lr_factor": 0.9964919771069799, "tokens_per_second": 16983.345551780298, "epoch": 0}
+{"step": 113, "tokens": 925696, "kind": "eval", "eval_seconds": 27.15604591369629, "val_loss": 6.330227212943578, "val_perplexity": 561.2841103806425, "dataset_fraction": 0.10019673584748816, "test_bleu": 0.17977685496362175}
+{"step": 120, "tokens": 983040, "kind": "train", "loss": 6.36898889541626, "aux_loss": 0.0, "lr_factor": 0.9924374974496396, "tokens_per_second": 3934.476313665344, "epoch": 0}
+{"step": 140, "tokens": 1146880, "kind": "train", "loss": 6.223147892951966, "aux_loss": 0.0, "lr_factor": 0.9868618040517874, "tokens_per_second": 16977.931543974522, "epoch": 0}
+{"step": 160, "tokens": 1310720, "kind": "train", "loss": 6.182667899131775, "aux_loss": 0.0, "lr_factor": 0.979784067579243, "tokens_per_second": 16981.08646322567, "epoch": 0}
+{"step": 180, "tokens": 1474560, "kind": "train", "loss": 6.127523374557495, "aux_loss": 0.0, "lr_factor": 0.9712286231076629, "tokens_per_second": 16940.56381495133, "epoch": 0}
+{"step": 200, "tokens": 1638400, "kind": "train", "loss": 6.038010859489441, "aux_loss": 0.0, "lr_factor": 0.9612248864523011, "tokens_per_second": 16967.826917168546, "epoch": 0}
+{"step": 220, "tokens": 1802240, "kind": "train", "loss": 6.012081980705261, "aux_loss": 0.0, "lr_factor": 0.9498072530288814, "tokens_per_second": 16917.993032156297, "epoch": 0}
+{"step": 226, "tokens": 1851392, "kind": "eval", "eval_seconds": 27.215065240859985, "val_loss": 5.96611026341736, "val_perplexity": 389.9857747339705, "dataset_fraction": 0.20039347169497632, "test_bleu": 0.4501581752031005}
+{"step": 240, "tokens": 1966080, "kind": "train", "loss": 5.94084587097168, "aux_loss": 0.0, "lr_factor": 0.9370149795933251, "tokens_per_second": 3930.343507031379, "epoch": 0}
+{"step": 260, "tokens": 2129920, "kind": "train", "loss": 5.956671118736267, "aux_loss": 0.0, "lr_factor": 0.9228920492669425, "tokens_per_second": 16949.247134212448, "epoch": 0}
+{"step": 280, "tokens": 2293760, "kind": "train", "loss": 5.891137194633484, "aux_loss": 0.0, "lr_factor": 0.9074870203111662, "tokens_per_second": 16917.866832774205, "epoch": 0}
+{"step": 300, "tokens": 2457600, "kind": "train", "loss": 5.845809960365296, "aux_loss": 0.0, "lr_factor": 0.8908528591717783, "tokens_per_second": 16945.30548732229, "epoch": 0}
+{"step": 320, "tokens": 2621440, "kind": "train", "loss": 5.8100899934768675, "aux_loss": 0.0, "lr_factor": 0.8730467583666665, "tokens_per_second": 16922.036573744146, "epoch": 0}
+{"step": 339, "tokens": 2777088, "kind": "eval", "eval_seconds": 27.3556067943573, "val_loss": 5.773949257469931, "val_perplexity": 321.8061216311937, "dataset_fraction": 0.30059020754246446, "test_bleu": 0.38954443751471024}
+{"step": 340, "tokens": 2785280, "kind": "train", "loss": 5.799847888946533, "aux_loss": 0.0, "lr_factor": 0.85412993984326, "tokens_per_second": 3916.438095531304, "epoch": 0}
+{"step": 360, "tokens": 2949120, "kind": "train", "loss": 5.771744179725647, "aux_loss": 0.0, "lr_factor": 0.8341674444817523, "tokens_per_second": 16958.90273681806, "epoch": 0}
+{"step": 380, "tokens": 3112960, "kind": "train", "loss": 5.746690392494202, "aux_loss": 0.0, "lr_factor": 0.8132279084678504, "tokens_per_second": 16910.513452573374, "epoch": 0}
+{"step": 400, "tokens": 3276800, "kind": "train", "loss": 5.710587573051453, "aux_loss": 0.0, "lr_factor": 0.7913833273039428, "tokens_per_second": 16920.05038923922, "epoch": 0}
+{"step": 420, "tokens": 3440640, "kind": "train", "loss": 5.695873260498047, "aux_loss": 0.0, "lr_factor": 0.7687088082700718, "tokens_per_second": 16913.799473024807, "epoch": 0}
+{"step": 440, "tokens": 3604480, "kind": "train", "loss": 5.64167046546936, "aux_loss": 0.0, "lr_factor": 0.7452823121858171, "tokens_per_second": 16942.5356038805, "epoch": 0}
+{"step": 452, "tokens": 3702784, "kind": "eval", "eval_seconds": 27.384177684783936, "val_loss": 5.622878325315332, "val_perplexity": 276.6846265662517, "dataset_fraction": 0.40078694338995263, "test_bleu": 0.8003497686628434}
+{"step": 460, "tokens": 3768320, "kind": "train", "loss": 5.638845586776734, "aux_loss": 0.0, "lr_factor": 0.7211843853609754, "tokens_per_second": 3942.8526399185876, "epoch": 0}
+{"step": 480, "tokens": 3932160, "kind": "train", "loss": 5.597468113899231, "aux_loss": 0.0, "lr_factor": 0.696497882656663, "tokens_per_second": 16940.95387615156, "epoch": 0}
+{"step": 500, "tokens": 4096000, "kind": "train", "loss": 5.596441316604614, "aux_loss": 0.0, "lr_factor": 0.6713076826090247, "tokens_per_second": 16928.070066584085, "epoch": 0}
+{"step": 520, "tokens": 4259840, "kind": "train", "loss": 5.574360585212707, "aux_loss": 0.0, "lr_factor": 0.645700395595028, "tokens_per_second": 16907.94837239705, "epoch": 0}
+{"step": 540, "tokens": 4423680, "kind": "train", "loss": 5.553585648536682, "aux_loss": 0.0, "lr_factor": 0.6197640660437446, "tokens_per_second": 16926.320936375538, "epoch": 0}
+{"step": 560, "tokens": 4587520, "kind": "train", "loss": 5.5327983617782595, "aux_loss": 0.0, "lr_factor": 0.5935878697169924, "tokens_per_second": 16892.2777332839, "epoch": 0}
+{"step": 564, "tokens": 4620288, "kind": "eval", "eval_seconds": 27.318898677825928, "val_loss": 5.519348807956861, "val_perplexity": 249.47252975730333, "dataset_fraction": 0.5000969824600294, "test_bleu": 0.44349244129811083}
+{"step": 580, "tokens": 4751360, "kind": "train", "loss": 5.500571179389953, "aux_loss": 0.0, "lr_factor": 0.5672618071001645, "tokens_per_second": 3922.8011479509646, "epoch": 0}
+{"step": 600, "tokens": 4915200, "kind": "train", "loss": 5.47302930355072, "aux_loss": 0.0, "lr_factor": 0.5408763939574474, "tokens_per_second": 16929.15933815549, "epoch": 0}
+{"step": 620, "tokens": 5079040, "kind": "train", "loss": 5.474884438514709, "aux_loss": 0.0, "lr_factor": 0.5145223501153768, "tokens_per_second": 16970.988559708254, "epoch": 0}
+{"step": 640, "tokens": 5242880, "kind": "train", "loss": 5.467896938323975, "aux_loss": 0.0, "lr_factor": 0.4882902875447779, "tokens_per_second": 16936.939276568326, "epoch": 0}
+{"step": 660, "tokens": 5406720, "kind": "train", "loss": 5.444820261001587, "aux_loss": 0.0, "lr_factor": 0.46227039881353715, "tokens_per_second": 16899.4656452544, "epoch": 0}
+{"step": 677, "tokens": 5545984, "kind": "eval", "eval_seconds": 27.302202701568604, "val_loss": 5.415750543119408, "val_perplexity": 224.92129547442175, "dataset_fraction": 0.6002937183075175, "test_bleu": 0.5114023727772138}
+{"step": 680, "tokens": 5570560, "kind": "train", "loss": 5.401749300956726, "aux_loss": 0.0, "lr_factor": 0.43655214698139233, "tokens_per_second": 3934.6879166090825, "epoch": 0}
+{"step": 700, "tokens": 5734400, "kind": "train", "loss": 5.405132365226746, "aux_loss": 0.0, "lr_factor": 0.4112239580029574, "tokens_per_second": 16919.51382027287, "epoch": 0}
+{"step": 720, "tokens": 5898240, "kind": "train", "loss": 5.370532035827637, "aux_loss": 0.0, "lr_factor": 0.3863729166965816, "tokens_per_second": 16925.66223968068, "epoch": 0}
+{"step": 740, "tokens": 6062080, "kind": "train", "loss": 5.379682898521423, "aux_loss": 0.0, "lr_factor": 0.36208446732437927, "tokens_per_second": 16943.455455681335, "epoch": 0}
+{"step": 760, "tokens": 6225920, "kind": "train", "loss": 5.35442373752594, "aux_loss": 0.0, "lr_factor": 0.3384421198129132, "tokens_per_second": 16907.30233621234, "epoch": 0}
+{"step": 780, "tokens": 6389760, "kind": "train", "loss": 5.345157361030578, "aux_loss": 0.0, "lr_factor": 0.31552716262462127, "tokens_per_second": 16888.863925249596, "epoch": 0}
+{"step": 790, "tokens": 6471680, "kind": "eval", "eval_seconds": 27.321674585342407, "val_loss": 5.335231718809708, "val_perplexity": 207.5208297167502, "dataset_fraction": 0.7004904541550057, "test_bleu": 0.6208649547744031}
+{"step": 800, "tokens": 6553600, "kind": "train", "loss": 5.321095895767212, "aux_loss": 0.0, "lr_factor": 0.2934183832672054, "tokens_per_second": 3953.0381210901073, "epoch": 0}
+{"step": 820, "tokens": 6717440, "kind": "train", "loss": 5.32721107006073, "aux_loss": 0.0, "lr_factor": 0.27219179740194743, "tokens_per_second": 16919.95582065439, "epoch": 0}
+{"step": 840, "tokens": 6881280, "kind": "train", "loss": 5.323449492454529, "aux_loss": 0.0, "lr_factor": 0.25192038748234297, "tokens_per_second": 16893.975394838835, "epoch": 0}
+{"step": 860, "tokens": 7045120, "kind": "train", "loss": 5.31880247592926, "aux_loss": 0.0, "lr_factor": 0.23267385182168396, "tokens_per_second": 16919.510904230163, "epoch": 0}
+{"step": 880, "tokens": 7208960, "kind": "train", "loss": 5.28803174495697, "aux_loss": 0.0, "lr_factor": 0.2145183649523572, "tokens_per_second": 16897.618545307072, "epoch": 0}
+{"step": 900, "tokens": 7372800, "kind": "train", "loss": 5.2870094060897825, "aux_loss": 0.0, "lr_factor": 0.1975163501008086, "tokens_per_second": 16887.249043480697, "epoch": 0}
+{"step": 903, "tokens": 7397376, "kind": "eval", "eval_seconds": 27.32434368133545, "val_loss": 5.270200944229548, "val_perplexity": 194.4550331365617, "dataset_fraction": 0.8006871900024939, "test_bleu": 0.5423850877050456}
+{"step": 920, "tokens": 7536640, "kind": "train", "loss": 5.255075693130493, "aux_loss": 0.0, "lr_factor": 0.18172626456045823, "tokens_per_second": 3942.528508278134, "epoch": 0}
+{"step": 940, "tokens": 7700480, "kind": "train", "loss": 5.2652702808380125, "aux_loss": 0.0, "lr_factor": 0.16720239870051368, "tokens_per_second": 16919.097252927782, "epoch": 0}
+{"step": 960, "tokens": 7864320, "kind": "train", "loss": 5.221942353248596, "aux_loss": 0.0, "lr_factor": 0.1539946893017382, "tokens_per_second": 16922.81792576436, "epoch": 0}
+{"step": 980, "tokens": 8028160, "kind": "train", "loss": 5.229254984855652, "aux_loss": 0.0, "lr_factor": 0.14214854786097333, "tokens_per_second": 16929.27861606922, "epoch": 0}
+{"step": 1000, "tokens": 8192000, "kind": "train", "loss": 5.23237669467926, "aux_loss": 0.0, "lr_factor": 0.13170470445475124, "tokens_per_second": 16889.587837831194, "epoch": 0}
+{"step": 1016, "tokens": 8323072, "kind": "eval", "eval_seconds": 27.36286497116089, "val_loss": 5.22599207082756, "val_perplexity": 186.04564946416497, "dataset_fraction": 0.900883925849982, "test_bleu": 0.6196968514578995}
+{"step": 1020, "tokens": 8355840, "kind": "train", "loss": 5.222024822235108, "aux_loss": 0.0, "lr_factor": 0.12269906769883077, "tokens_per_second": 3896.7369952528966, "epoch": 0}
+{"step": 1040, "tokens": 8519680, "kind": "train", "loss": 5.245731401443481, "aux_loss": 0.0, "lr_factor": 0.11516260128515528, "tokens_per_second": 16919.902496103492, "epoch": 0}
+{"step": 1060, "tokens": 8683520, "kind": "train", "loss": 5.221320581436157, "aux_loss": 0.0, "lr_factor": 0.10912121752072734, "tokens_per_second": 16915.399450895173, "epoch": 0}
+{"step": 1080, "tokens": 8847360, "kind": "train", "loss": 5.205885696411133, "aux_loss": 0.0, "lr_factor": 0.10459568823444292, "tokens_per_second": 16941.96210469389, "epoch": 0}
+{"step": 1100, "tokens": 9011200, "kind": "train", "loss": 5.2134154558181764, "aux_loss": 0.0, "lr_factor": 0.10160157335820881, "tokens_per_second": 16934.124972576094, "epoch": 0}
+{"step": 1120, "tokens": 9175040, "kind": "train", "loss": 5.180478119850159, "aux_loss": 0.0, "lr_factor": 0.10014916742790117, "tokens_per_second": 16911.175548517982, "epoch": 0}
+{"step": 1128, "tokens": 9238784, "kind": "eval", "eval_seconds": 27.32951784133911, "val_loss": 5.195787243221117, "val_perplexity": 180.51019230115048, "dataset_fraction": 1.0, "test_bleu": 0.6201012684965325}

part2/adamw/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e54bbdb97ccf254195ea4b7271355347e68e26157a89a6316e4cc786fb73826f
+size 147370362