Upload folder using huggingface_hub

part2/scion/config.json

@@ -0,0 +1,51 @@
+{
+  "model": {
+    "d_model": 512,
+    "n_layers": 6,
+    "n_heads": 8,
+    "n_ctx": 256,
+    "d_ff": 1536,
+    "dropout": 0.0
+  },
+  "seed": 0,
+  "data": {
+    "name": "browndw/human-ai-parallel-corpus",
+    "fractions": {
+      "val": 0.05,
+      "test": 0.05
+    },
+    "seed": 0,
+    "limit": null
+  },
+  "tokenizer": {
+    "file": "unigram-16k.json",
+    "vocab_size": 16000
+  },
+  "train": {
+    "batch_size": 32,
+    "grad_clip": 0.0,
+    "warmup_fraction": 0.05,
+    "min_lr_ratio": 0.1,
+    "eval_every_fraction": 0.1,
+    "log_every": 20
+  },
+  "evaluate": {
+    "batch_size": 64,
+    "prompt_tokens": 128,
+    "reference_tokens": 64
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "anlp-assignment-2"
+  },
+  "name": "scion",
+  "optimizer": {
+    "name": "scion",
+    "lr": 0.004,
+    "sign_lr": 0.0008,
+    "beta": 0.95,
+    "sign_beta": 0.95,
+    "weight_decay": 0.1,
+    "grad_norm": 1.0
+  }
+}

part2/scion/metrics.jsonl

@@ -0,0 +1,67 @@
+{"step": 0, "tokens": 0, "kind": "eval", "eval_seconds": 27.442217111587524, "val_loss": 9.769027694883082, "val_perplexity": 17483.759402580894, "dataset_fraction": 0.0, "test_bleu": 0.02477986831865296}
+{"step": 20, "tokens": 163840, "kind": "train", "loss": 9.397862482070924, "aux_loss": 0.0, "lr_factor": 0.33694694020338606, "tokens_per_second": 4113.522812743444, "epoch": 0}
+{"step": 40, "tokens": 327680, "kind": "train", "loss": 7.653070712089539, "aux_loss": 0.0, "lr_factor": 0.6916256511679459, "tokens_per_second": 15486.434885344184, "epoch": 0}
+{"step": 60, "tokens": 491520, "kind": "train", "loss": 7.010008835792542, "aux_loss": 0.0, "lr_factor": 0.9999868120848697, "tokens_per_second": 15450.519567247256, "epoch": 0}
+{"step": 80, "tokens": 655360, "kind": "train", "loss": 6.491313743591308, "aux_loss": 0.0, "lr_factor": 0.999011302681498, "tokens_per_second": 15389.735187043141, "epoch": 0}
+{"step": 100, "tokens": 819200, "kind": "train", "loss": 6.282988095283509, "aux_loss": 0.0, "lr_factor": 0.9964919771069799, "tokens_per_second": 15395.776917884928, "epoch": 0}
+{"step": 113, "tokens": 925696, "kind": "eval", "eval_seconds": 27.346315145492554, "val_loss": 6.161676105303256, "val_perplexity": 474.2222555210794, "dataset_fraction": 0.10019673584748816, "test_bleu": 0.16697033235923003}
+{"step": 120, "tokens": 983040, "kind": "train", "loss": 6.192653179168701, "aux_loss": 0.0, "lr_factor": 0.9924374974496396, "tokens_per_second": 3966.5544380263864, "epoch": 0}
+{"step": 140, "tokens": 1146880, "kind": "train", "loss": 6.079783701896668, "aux_loss": 0.0, "lr_factor": 0.9868618040517874, "tokens_per_second": 15386.826862845724, "epoch": 0}
+{"step": 160, "tokens": 1310720, "kind": "train", "loss": 6.069482064247131, "aux_loss": 0.0, "lr_factor": 0.979784067579243, "tokens_per_second": 15393.792481181717, "epoch": 0}
+{"step": 180, "tokens": 1474560, "kind": "train", "loss": 6.024078965187073, "aux_loss": 0.0, "lr_factor": 0.9712286231076629, "tokens_per_second": 15381.976129820569, "epoch": 0}
+{"step": 200, "tokens": 1638400, "kind": "train", "loss": 5.951095080375671, "aux_loss": 0.0, "lr_factor": 0.9612248864523011, "tokens_per_second": 15391.12185222901, "epoch": 0}
+{"step": 220, "tokens": 1802240, "kind": "train", "loss": 5.915898036956787, "aux_loss": 0.0, "lr_factor": 0.9498072530288814, "tokens_per_second": 15268.361546800785, "epoch": 0}
+{"step": 226, "tokens": 1851392, "kind": "eval", "eval_seconds": 27.350701808929443, "val_loss": 5.874929501605128, "val_perplexity": 355.99956238837166, "dataset_fraction": 0.20039347169497632, "test_bleu": 0.5558464983944039}
+{"step": 240, "tokens": 1966080, "kind": "train", "loss": 5.846077799797058, "aux_loss": 0.0, "lr_factor": 0.9370149795933251, "tokens_per_second": 3972.262066366808, "epoch": 0}
+{"step": 260, "tokens": 2129920, "kind": "train", "loss": 5.844623041152954, "aux_loss": 0.0, "lr_factor": 0.9228920492669425, "tokens_per_second": 15396.997700604892, "epoch": 0}
+{"step": 280, "tokens": 2293760, "kind": "train", "loss": 5.770828342437744, "aux_loss": 0.0, "lr_factor": 0.9074870203111662, "tokens_per_second": 15409.196834794124, "epoch": 0}
+{"step": 300, "tokens": 2457600, "kind": "train", "loss": 5.713491988182068, "aux_loss": 0.0, "lr_factor": 0.8908528591717783, "tokens_per_second": 15390.64029985639, "epoch": 0}
+{"step": 320, "tokens": 2621440, "kind": "train", "loss": 5.668173861503601, "aux_loss": 0.0, "lr_factor": 0.8730467583666665, "tokens_per_second": 15287.218645797051, "epoch": 0}
+{"step": 339, "tokens": 2777088, "kind": "eval", "eval_seconds": 27.342163801193237, "val_loss": 5.598623890179419, "val_perplexity": 270.05452692403975, "dataset_fraction": 0.30059020754246446, "test_bleu": 0.36923910094379}
+{"step": 340, "tokens": 2785280, "kind": "train", "loss": 5.63374764919281, "aux_loss": 0.0, "lr_factor": 0.85412993984326, "tokens_per_second": 3982.242120912871, "epoch": 0}
+{"step": 360, "tokens": 2949120, "kind": "train", "loss": 5.583894920349121, "aux_loss": 0.0, "lr_factor": 0.8341674444817523, "tokens_per_second": 15383.128952254094, "epoch": 0}
+{"step": 380, "tokens": 3112960, "kind": "train", "loss": 5.546555376052856, "aux_loss": 0.0, "lr_factor": 0.8132279084678504, "tokens_per_second": 15412.453755782517, "epoch": 0}
+{"step": 400, "tokens": 3276800, "kind": "train", "loss": 5.497678899765015, "aux_loss": 0.0, "lr_factor": 0.7913833273039428, "tokens_per_second": 15382.064961247064, "epoch": 0}
+{"step": 420, "tokens": 3440640, "kind": "train", "loss": 5.467425990104675, "aux_loss": 0.0, "lr_factor": 0.7687088082700718, "tokens_per_second": 15362.055290541442, "epoch": 0}
+{"step": 440, "tokens": 3604480, "kind": "train", "loss": 5.411876082420349, "aux_loss": 0.0, "lr_factor": 0.7452823121858171, "tokens_per_second": 15260.902964657284, "epoch": 0}
+{"step": 452, "tokens": 3702784, "kind": "eval", "eval_seconds": 27.310315370559692, "val_loss": 5.380671495505473, "val_perplexity": 217.16805384678258, "dataset_fraction": 0.40078694338995263, "test_bleu": 0.583130236757636}
+{"step": 460, "tokens": 3768320, "kind": "train", "loss": 5.390134572982788, "aux_loss": 0.0, "lr_factor": 0.7211843853609754, "tokens_per_second": 3969.8941510246814, "epoch": 0}
+{"step": 480, "tokens": 3932160, "kind": "train", "loss": 5.346011734008789, "aux_loss": 0.0, "lr_factor": 0.696497882656663, "tokens_per_second": 15391.350746670974, "epoch": 0}
+{"step": 500, "tokens": 4096000, "kind": "train", "loss": 5.336445212364197, "aux_loss": 0.0, "lr_factor": 0.6713076826090247, "tokens_per_second": 15409.014399465865, "epoch": 0}
+{"step": 520, "tokens": 4259840, "kind": "train", "loss": 5.313250207901001, "aux_loss": 0.0, "lr_factor": 0.645700395595028, "tokens_per_second": 15380.925035840044, "epoch": 0}
+{"step": 540, "tokens": 4423680, "kind": "train", "loss": 5.28004138469696, "aux_loss": 0.0, "lr_factor": 0.6197640660437446, "tokens_per_second": 15295.764110811531, "epoch": 0}
+{"step": 560, "tokens": 4587520, "kind": "train", "loss": 5.2514889478683475, "aux_loss": 0.0, "lr_factor": 0.5935878697169924, "tokens_per_second": 15270.825833083003, "epoch": 0}
+{"step": 564, "tokens": 4620288, "kind": "eval", "eval_seconds": 27.358331441879272, "val_loss": 5.240570686551422, "val_perplexity": 188.77780462494525, "dataset_fraction": 0.5000969824600294, "test_bleu": 0.4887113751863503}
+{"step": 580, "tokens": 4751360, "kind": "train", "loss": 5.22217116355896, "aux_loss": 0.0, "lr_factor": 0.5672618071001645, "tokens_per_second": 3970.82082824869, "epoch": 0}
+{"step": 600, "tokens": 4915200, "kind": "train", "loss": 5.186874413490296, "aux_loss": 0.0, "lr_factor": 0.5408763939574474, "tokens_per_second": 15390.589974719107, "epoch": 0}
+{"step": 620, "tokens": 5079040, "kind": "train", "loss": 5.1733482599258425, "aux_loss": 0.0, "lr_factor": 0.5145223501153768, "tokens_per_second": 15394.764632597738, "epoch": 0}
+{"step": 640, "tokens": 5242880, "kind": "train", "loss": 5.162998819351197, "aux_loss": 0.0, "lr_factor": 0.4882902875447779, "tokens_per_second": 15365.132558592646, "epoch": 0}
+{"step": 660, "tokens": 5406720, "kind": "train", "loss": 5.144989919662476, "aux_loss": 0.0, "lr_factor": 0.46227039881353715, "tokens_per_second": 15265.286969809911, "epoch": 0}
+{"step": 677, "tokens": 5545984, "kind": "eval", "eval_seconds": 27.36740255355835, "val_loss": 5.122200698249425, "val_perplexity": 167.70402978219434, "dataset_fraction": 0.6002937183075175, "test_bleu": 0.686868796646868}
+{"step": 680, "tokens": 5570560, "kind": "train", "loss": 5.096845960617065, "aux_loss": 0.0, "lr_factor": 0.43655214698139233, "tokens_per_second": 3974.3882695522825, "epoch": 0}
+{"step": 700, "tokens": 5734400, "kind": "train", "loss": 5.1002647399902346, "aux_loss": 0.0, "lr_factor": 0.4112239580029574, "tokens_per_second": 15403.088292510281, "epoch": 0}
+{"step": 720, "tokens": 5898240, "kind": "train", "loss": 5.061530590057373, "aux_loss": 0.0, "lr_factor": 0.3863729166965816, "tokens_per_second": 15375.375865246244, "epoch": 0}
+{"step": 740, "tokens": 6062080, "kind": "train", "loss": 5.073472118377685, "aux_loss": 0.0, "lr_factor": 0.36208446732437927, "tokens_per_second": 15387.744727015031, "epoch": 0}
+{"step": 760, "tokens": 6225920, "kind": "train", "loss": 5.032696413993835, "aux_loss": 0.0, "lr_factor": 0.3384421198129132, "tokens_per_second": 15357.898008030597, "epoch": 0}
+{"step": 780, "tokens": 6389760, "kind": "train", "loss": 5.026140356063843, "aux_loss": 0.0, "lr_factor": 0.31552716262462127, "tokens_per_second": 15278.852574336175, "epoch": 0}
+{"step": 790, "tokens": 6471680, "kind": "eval", "eval_seconds": 27.288350343704224, "val_loss": 5.027266704046679, "val_perplexity": 152.51557226489277, "dataset_fraction": 0.7004904541550057, "test_bleu": 0.7064008371025176}
+{"step": 800, "tokens": 6553600, "kind": "train", "loss": 5.001676607131958, "aux_loss": 0.0, "lr_factor": 0.2934183832672054, "tokens_per_second": 3988.9275036625622, "epoch": 0}
+{"step": 820, "tokens": 6717440, "kind": "train", "loss": 5.005000352859497, "aux_loss": 0.0, "lr_factor": 0.27219179740194743, "tokens_per_second": 15390.410047810521, "epoch": 0}
+{"step": 840, "tokens": 6881280, "kind": "train", "loss": 5.004054355621338, "aux_loss": 0.0, "lr_factor": 0.25192038748234297, "tokens_per_second": 15399.354610643783, "epoch": 0}
+{"step": 860, "tokens": 7045120, "kind": "train", "loss": 4.99724657535553, "aux_loss": 0.0, "lr_factor": 0.23267385182168396, "tokens_per_second": 15397.988198087127, "epoch": 0}
+{"step": 880, "tokens": 7208960, "kind": "train", "loss": 4.955174112319947, "aux_loss": 0.0, "lr_factor": 0.2145183649523572, "tokens_per_second": 15349.143116392956, "epoch": 0}
+{"step": 900, "tokens": 7372800, "kind": "train", "loss": 4.956913542747498, "aux_loss": 0.0, "lr_factor": 0.1975163501008086, "tokens_per_second": 15271.208627143002, "epoch": 0}
+{"step": 903, "tokens": 7397376, "kind": "eval", "eval_seconds": 27.327828645706177, "val_loss": 4.953729829298178, "val_perplexity": 141.70250564679176, "dataset_fraction": 0.8006871900024939, "test_bleu": 0.5498246766100552}
+{"step": 920, "tokens": 7536640, "kind": "train", "loss": 4.927284550666809, "aux_loss": 0.0, "lr_factor": 0.18172626456045823, "tokens_per_second": 3970.6836242034356, "epoch": 0}
+{"step": 940, "tokens": 7700480, "kind": "train", "loss": 4.932329297065735, "aux_loss": 0.0, "lr_factor": 0.16720239870051368, "tokens_per_second": 15394.920519309213, "epoch": 0}
+{"step": 960, "tokens": 7864320, "kind": "train", "loss": 4.882231831550598, "aux_loss": 0.0, "lr_factor": 0.1539946893017382, "tokens_per_second": 15380.508149717047, "epoch": 0}
+{"step": 980, "tokens": 8028160, "kind": "train", "loss": 4.890003633499146, "aux_loss": 0.0, "lr_factor": 0.14214854786097333, "tokens_per_second": 15385.881549662028, "epoch": 0}
+{"step": 1000, "tokens": 8192000, "kind": "train", "loss": 4.891906309127807, "aux_loss": 0.0, "lr_factor": 0.13170470445475124, "tokens_per_second": 15321.398486010456, "epoch": 0}
+{"step": 1016, "tokens": 8323072, "kind": "eval", "eval_seconds": 27.322156190872192, "val_loss": 4.900510599490682, "val_perplexity": 134.35836548649394, "dataset_fraction": 0.900883925849982, "test_bleu": 0.6689536149889149}
+{"step": 1020, "tokens": 8355840, "kind": "train", "loss": 4.879666876792908, "aux_loss": 0.0, "lr_factor": 0.12269906769883077, "tokens_per_second": 3960.582540194513, "epoch": 0}
+{"step": 1040, "tokens": 8519680, "kind": "train", "loss": 4.906302547454834, "aux_loss": 0.0, "lr_factor": 0.11516260128515528, "tokens_per_second": 15398.806981341622, "epoch": 0}
+{"step": 1060, "tokens": 8683520, "kind": "train", "loss": 4.876235580444336, "aux_loss": 0.0, "lr_factor": 0.10912121752072734, "tokens_per_second": 15382.058075053286, "epoch": 0}
+{"step": 1080, "tokens": 8847360, "kind": "train", "loss": 4.861716532707215, "aux_loss": 0.0, "lr_factor": 0.10459568823444292, "tokens_per_second": 15391.76580757248, "epoch": 0}
+{"step": 1100, "tokens": 9011200, "kind": "train", "loss": 4.87034764289856, "aux_loss": 0.0, "lr_factor": 0.10160157335820881, "tokens_per_second": 15370.35117113596, "epoch": 0}
+{"step": 1120, "tokens": 9175040, "kind": "train", "loss": 4.8377938032150265, "aux_loss": 0.0, "lr_factor": 0.10014916742790117, "tokens_per_second": 15272.751196609832, "epoch": 0}
+{"step": 1128, "tokens": 9238784, "kind": "eval", "eval_seconds": 27.39917826652527, "val_loss": 4.867142322977541, "val_perplexity": 129.94903342120568, "dataset_fraction": 1.0, "test_bleu": 0.7066685081665655}

part2/scion/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1ecb731b3c89c7163cc190838c0426e17858e2c9baf49ef8a88082b9165de69
+size 147370362