Upload folder using huggingface_hub

part2/adamw-half/config.json

@@ -0,0 +1,53 @@
+{
+  "model": {
+    "d_model": 512,
+    "n_layers": 6,
+    "n_heads": 8,
+    "n_ctx": 256,
+    "d_ff": 1536,
+    "dropout": 0.0
+  },
+  "seed": 0,
+  "data": {
+    "name": "browndw/human-ai-parallel-corpus",
+    "fractions": {
+      "val": 0.05,
+      "test": 0.05
+    },
+    "seed": 0,
+    "limit": null
+  },
+  "tokenizer": {
+    "file": "unigram-16k.json",
+    "vocab_size": 16000
+  },
+  "train": {
+    "batch_size": 32,
+    "grad_clip": 0.0,
+    "warmup_fraction": 0.05,
+    "min_lr_ratio": 0.1,
+    "eval_every_fraction": 0.1,
+    "log_every": 20
+  },
+  "evaluate": {
+    "batch_size": 64,
+    "prompt_tokens": 128,
+    "reference_tokens": 64
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "anlp-assignment-2"
+  },
+  "name": "adamw-half",
+  "optimizer": {
+    "name": "adamw",
+    "lr": 0.001,
+    "betas": [
+      0.9,
+      0.98
+    ],
+    "eps": 1e-10,
+    "weight_decay": 0.1,
+    "grad_norm": 1.0
+  }
+}

part2/adamw-half/metrics.jsonl

@@ -0,0 +1,67 @@
+{"step": 0, "tokens": 0, "kind": "eval", "eval_seconds": 5.46994161605835, "val_loss": 9.76902662435539, "val_perplexity": 17483.740685742323, "dataset_fraction": 0.0, "test_bleu": 0.024831537652001516}
+{"step": 20, "tokens": 163840, "kind": "train", "loss": 8.851418828964233, "aux_loss": 0.0, "lr_factor": 0.33694694020338606, "tokens_per_second": 18087.505643439486, "epoch": 0}
+{"step": 40, "tokens": 327680, "kind": "train", "loss": 7.417070364952087, "aux_loss": 0.0, "lr_factor": 0.6916256511679459, "tokens_per_second": 111187.2806995403, "epoch": 0}
+{"step": 60, "tokens": 491520, "kind": "train", "loss": 6.971866750717163, "aux_loss": 0.0, "lr_factor": 0.9999868120848697, "tokens_per_second": 110728.38747496427, "epoch": 0}
+{"step": 80, "tokens": 655360, "kind": "train", "loss": 6.649613237380981, "aux_loss": 0.0, "lr_factor": 0.999011302681498, "tokens_per_second": 110492.06119428806, "epoch": 0}
+{"step": 100, "tokens": 819200, "kind": "train", "loss": 6.432142233848571, "aux_loss": 0.0, "lr_factor": 0.9964919771069799, "tokens_per_second": 110534.71533233693, "epoch": 0}
+{"step": 113, "tokens": 925696, "kind": "eval", "eval_seconds": 4.645138740539551, "val_loss": 6.261759823961221, "val_perplexity": 524.1405240106404, "dataset_fraction": 0.10019673584748816, "test_bleu": 0.27986835783663494}
+{"step": 120, "tokens": 983040, "kind": "train", "loss": 6.299322032928467, "aux_loss": 0.0, "lr_factor": 0.9924374974496396, "tokens_per_second": 14934.685298335542, "epoch": 0}
+{"step": 140, "tokens": 1146880, "kind": "train", "loss": 6.146497631072998, "aux_loss": 0.0, "lr_factor": 0.9868618040517874, "tokens_per_second": 110576.13911588296, "epoch": 0}
+{"step": 160, "tokens": 1310720, "kind": "train", "loss": 6.10483992099762, "aux_loss": 0.0, "lr_factor": 0.979784067579243, "tokens_per_second": 110405.25517592838, "epoch": 0}
+{"step": 180, "tokens": 1474560, "kind": "train", "loss": 6.043229031562805, "aux_loss": 0.0, "lr_factor": 0.9712286231076629, "tokens_per_second": 110327.2108693038, "epoch": 0}
+{"step": 200, "tokens": 1638400, "kind": "train", "loss": 5.95453360080719, "aux_loss": 0.0, "lr_factor": 0.9612248864523011, "tokens_per_second": 110237.10901942768, "epoch": 0}
+{"step": 220, "tokens": 1802240, "kind": "train", "loss": 5.915622639656067, "aux_loss": 0.0, "lr_factor": 0.9498072530288814, "tokens_per_second": 110108.53870980715, "epoch": 0}
+{"step": 226, "tokens": 1851392, "kind": "eval", "eval_seconds": 5.639220952987671, "val_loss": 5.867844969858765, "val_perplexity": 353.48638504077053, "dataset_fraction": 0.20039347169497632, "test_bleu": 0.33434165344501693}
+{"step": 240, "tokens": 1966080, "kind": "train", "loss": 5.847331714630127, "aux_loss": 0.0, "lr_factor": 0.9370149795933251, "tokens_per_second": 13628.71958315157, "epoch": 0}
+{"step": 260, "tokens": 2129920, "kind": "train", "loss": 5.851085090637207, "aux_loss": 0.0, "lr_factor": 0.9228920492669425, "tokens_per_second": 110238.5060579504, "epoch": 0}
+{"step": 280, "tokens": 2293760, "kind": "train", "loss": 5.782818841934204, "aux_loss": 0.0, "lr_factor": 0.9074870203111662, "tokens_per_second": 110058.20979477478, "epoch": 0}
+{"step": 300, "tokens": 2457600, "kind": "train", "loss": 5.731809210777283, "aux_loss": 0.0, "lr_factor": 0.8908528591717783, "tokens_per_second": 109986.90412196347, "epoch": 0}
+{"step": 320, "tokens": 2621440, "kind": "train", "loss": 5.695348501205444, "aux_loss": 0.0, "lr_factor": 0.8730467583666665, "tokens_per_second": 109979.42309015724, "epoch": 0}
+{"step": 339, "tokens": 2777088, "kind": "eval", "eval_seconds": 4.6588380336761475, "val_loss": 5.65422322156401, "val_perplexity": 285.49463050009194, "dataset_fraction": 0.30059020754246446, "test_bleu": 0.7636503408561441}
+{"step": 340, "tokens": 2785280, "kind": "train", "loss": 5.685166883468628, "aux_loss": 0.0, "lr_factor": 0.85412993984326, "tokens_per_second": 14868.758243466129, "epoch": 0}
+{"step": 360, "tokens": 2949120, "kind": "train", "loss": 5.659188318252563, "aux_loss": 0.0, "lr_factor": 0.8341674444817523, "tokens_per_second": 109883.59755113776, "epoch": 0}
+{"step": 380, "tokens": 3112960, "kind": "train", "loss": 5.630069541931152, "aux_loss": 0.0, "lr_factor": 0.8132279084678504, "tokens_per_second": 110166.27785839741, "epoch": 0}
+{"step": 400, "tokens": 3276800, "kind": "train", "loss": 5.588517808914185, "aux_loss": 0.0, "lr_factor": 0.7913833273039428, "tokens_per_second": 110132.23779769747, "epoch": 0}
+{"step": 420, "tokens": 3440640, "kind": "train", "loss": 5.576625037193298, "aux_loss": 0.0, "lr_factor": 0.7687088082700718, "tokens_per_second": 110128.81376966812, "epoch": 0}
+{"step": 440, "tokens": 3604480, "kind": "train", "loss": 5.525609564781189, "aux_loss": 0.0, "lr_factor": 0.7452823121858171, "tokens_per_second": 109741.68784384367, "epoch": 0}
+{"step": 452, "tokens": 3702784, "kind": "eval", "eval_seconds": 4.685971975326538, "val_loss": 5.49735704429536, "val_perplexity": 244.04607618641953, "dataset_fraction": 0.40078694338995263, "test_bleu": 0.4354434909724848}
+{"step": 460, "tokens": 3768320, "kind": "train", "loss": 5.515602397918701, "aux_loss": 0.0, "lr_factor": 0.7211843853609754, "tokens_per_second": 14884.031187740871, "epoch": 0}
+{"step": 480, "tokens": 3932160, "kind": "train", "loss": 5.475169014930725, "aux_loss": 0.0, "lr_factor": 0.696497882656663, "tokens_per_second": 110270.0461077362, "epoch": 0}
+{"step": 500, "tokens": 4096000, "kind": "train", "loss": 5.471340560913086, "aux_loss": 0.0, "lr_factor": 0.6713076826090247, "tokens_per_second": 109982.83783489645, "epoch": 0}
+{"step": 520, "tokens": 4259840, "kind": "train", "loss": 5.45366051197052, "aux_loss": 0.0, "lr_factor": 0.645700395595028, "tokens_per_second": 109938.39233394648, "epoch": 0}
+{"step": 540, "tokens": 4423680, "kind": "train", "loss": 5.42756724357605, "aux_loss": 0.0, "lr_factor": 0.6197640660437446, "tokens_per_second": 109300.8837279138, "epoch": 0}
+{"step": 560, "tokens": 4587520, "kind": "train", "loss": 5.407470440864563, "aux_loss": 0.0, "lr_factor": 0.5935878697169924, "tokens_per_second": 108255.12539778784, "epoch": 0}
+{"step": 564, "tokens": 4620288, "kind": "eval", "eval_seconds": 4.682809352874756, "val_loss": 5.385529305152742, "val_perplexity": 218.2255814681239, "dataset_fraction": 0.5000969824600294, "test_bleu": 0.5309161278590233}
+{"step": 580, "tokens": 4751360, "kind": "train", "loss": 5.375206971168518, "aux_loss": 0.0, "lr_factor": 0.5672618071001645, "tokens_per_second": 14906.770097416224, "epoch": 0}
+{"step": 600, "tokens": 4915200, "kind": "train", "loss": 5.344594025611878, "aux_loss": 0.0, "lr_factor": 0.5408763939574474, "tokens_per_second": 110253.02674458454, "epoch": 0}
+{"step": 620, "tokens": 5079040, "kind": "train", "loss": 5.346733713150025, "aux_loss": 0.0, "lr_factor": 0.5145223501153768, "tokens_per_second": 110067.55261043238, "epoch": 0}
+{"step": 640, "tokens": 5242880, "kind": "train", "loss": 5.337068104743958, "aux_loss": 0.0, "lr_factor": 0.4882902875447779, "tokens_per_second": 109518.8349861156, "epoch": 0}
+{"step": 660, "tokens": 5406720, "kind": "train", "loss": 5.319228553771973, "aux_loss": 0.0, "lr_factor": 0.46227039881353715, "tokens_per_second": 108527.4561962353, "epoch": 0}
+{"step": 677, "tokens": 5545984, "kind": "eval", "eval_seconds": 4.677881956100464, "val_loss": 5.2870833035043105, "val_perplexity": 197.76576068141227, "dataset_fraction": 0.6002937183075175, "test_bleu": 0.6260475229758603}
+{"step": 680, "tokens": 5570560, "kind": "train", "loss": 5.275702977180481, "aux_loss": 0.0, "lr_factor": 0.43655214698139233, "tokens_per_second": 14931.633638106548, "epoch": 0}
+{"step": 700, "tokens": 5734400, "kind": "train", "loss": 5.281583619117737, "aux_loss": 0.0, "lr_factor": 0.4112239580029574, "tokens_per_second": 109026.4613869344, "epoch": 0}
+{"step": 720, "tokens": 5898240, "kind": "train", "loss": 5.242434763908387, "aux_loss": 0.0, "lr_factor": 0.3863729166965816, "tokens_per_second": 110044.86818375583, "epoch": 0}
+{"step": 740, "tokens": 6062080, "kind": "train", "loss": 5.256624722480774, "aux_loss": 0.0, "lr_factor": 0.36208446732437927, "tokens_per_second": 109899.90546169267, "epoch": 0}
+{"step": 760, "tokens": 6225920, "kind": "train", "loss": 5.230032873153687, "aux_loss": 0.0, "lr_factor": 0.3384421198129132, "tokens_per_second": 109043.27717530062, "epoch": 0}
+{"step": 780, "tokens": 6389760, "kind": "train", "loss": 5.220384311676026, "aux_loss": 0.0, "lr_factor": 0.31552716262462127, "tokens_per_second": 108096.6565380335, "epoch": 0}
+{"step": 790, "tokens": 6471680, "kind": "eval", "eval_seconds": 4.698574066162109, "val_loss": 5.210817442581116, "val_perplexity": 183.24378826276453, "dataset_fraction": 0.7004904541550057, "test_bleu": 0.5643420009188618}
+{"step": 800, "tokens": 6553600, "kind": "train", "loss": 5.197488307952881, "aux_loss": 0.0, "lr_factor": 0.2934183832672054, "tokens_per_second": 14804.83499086486, "epoch": 0}
+{"step": 820, "tokens": 6717440, "kind": "train", "loss": 5.205226087570191, "aux_loss": 0.0, "lr_factor": 0.27219179740194743, "tokens_per_second": 110074.37563070598, "epoch": 0}
+{"step": 840, "tokens": 6881280, "kind": "train", "loss": 5.205312347412109, "aux_loss": 0.0, "lr_factor": 0.25192038748234297, "tokens_per_second": 110004.47534653194, "epoch": 0}
+{"step": 860, "tokens": 7045120, "kind": "train", "loss": 5.198543071746826, "aux_loss": 0.0, "lr_factor": 0.23267385182168396, "tokens_per_second": 109378.42235704337, "epoch": 0}
+{"step": 880, "tokens": 7208960, "kind": "train", "loss": 5.168464398384094, "aux_loss": 0.0, "lr_factor": 0.2145183649523572, "tokens_per_second": 108647.08485572689, "epoch": 0}
+{"step": 900, "tokens": 7372800, "kind": "train", "loss": 5.170079088211059, "aux_loss": 0.0, "lr_factor": 0.1975163501008086, "tokens_per_second": 106876.32243115002, "epoch": 0}
+{"step": 903, "tokens": 7397376, "kind": "eval", "eval_seconds": 4.701461315155029, "val_loss": 5.150407706324762, "val_perplexity": 172.501806059157, "dataset_fraction": 0.8006871900024939, "test_bleu": 0.5952705742652565}
+{"step": 920, "tokens": 7536640, "kind": "train", "loss": 5.133412408828735, "aux_loss": 0.0, "lr_factor": 0.18172626456045823, "tokens_per_second": 14883.859686239264, "epoch": 0}
+{"step": 940, "tokens": 7700480, "kind": "train", "loss": 5.1498253583908085, "aux_loss": 0.0, "lr_factor": 0.16720239870051368, "tokens_per_second": 110115.3315230611, "epoch": 0}
+{"step": 960, "tokens": 7864320, "kind": "train", "loss": 5.104320335388183, "aux_loss": 0.0, "lr_factor": 0.1539946893017382, "tokens_per_second": 109871.51032747299, "epoch": 0}
+{"step": 980, "tokens": 8028160, "kind": "train", "loss": 5.112407684326172, "aux_loss": 0.0, "lr_factor": 0.14214854786097333, "tokens_per_second": 108862.69261373996, "epoch": 0}
+{"step": 1000, "tokens": 8192000, "kind": "train", "loss": 5.1148988723754885, "aux_loss": 0.0, "lr_factor": 0.13170470445475124, "tokens_per_second": 106852.47518988539, "epoch": 0}
+{"step": 1016, "tokens": 8323072, "kind": "eval", "eval_seconds": 4.714008569717407, "val_loss": 5.108053429795819, "val_perplexity": 165.34817956737285, "dataset_fraction": 0.900883925849982, "test_bleu": 0.47462801592194687}
+{"step": 1020, "tokens": 8355840, "kind": "train", "loss": 5.106530237197876, "aux_loss": 0.0, "lr_factor": 0.12269906769883077, "tokens_per_second": 14753.358810837592, "epoch": 0}
+{"step": 1040, "tokens": 8519680, "kind": "train", "loss": 5.126446461677551, "aux_loss": 0.0, "lr_factor": 0.11516260128515528, "tokens_per_second": 110240.69894932049, "epoch": 0}
+{"step": 1060, "tokens": 8683520, "kind": "train", "loss": 5.105200219154358, "aux_loss": 0.0, "lr_factor": 0.10912121752072734, "tokens_per_second": 109951.37388423785, "epoch": 0}
+{"step": 1080, "tokens": 8847360, "kind": "train", "loss": 5.093526148796082, "aux_loss": 0.0, "lr_factor": 0.10459568823444292, "tokens_per_second": 109511.03355330792, "epoch": 0}
+{"step": 1100, "tokens": 9011200, "kind": "train", "loss": 5.093413996696472, "aux_loss": 0.0, "lr_factor": 0.10160157335820881, "tokens_per_second": 108088.41034900931, "epoch": 0}
+{"step": 1120, "tokens": 9175040, "kind": "train", "loss": 5.070134782791138, "aux_loss": 0.0, "lr_factor": 0.10014916742790117, "tokens_per_second": 106477.44268736732, "epoch": 0}
+{"step": 1128, "tokens": 9238784, "kind": "eval", "eval_seconds": 4.7203755378723145, "val_loss": 5.079115294656264, "val_perplexity": 160.63188116285397, "dataset_fraction": 1.0, "test_bleu": 0.6989427479545056}

part2/adamw-half/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19185f958dea444dc11f7447c740ee38ff0a2d6eb0f059004ca25ebe71cac16
+size 147370362