{ "summary": { "reference_ppl_mean": 27.426984312057495, "reference_ppl_median": 24.0395565032959, "n_samples": 1000, "model": "EleutherAI/pythia-160m", "max_new_tokens": 70, "beam_w1_equals_greedy_frac": 0.997, "reference_judge_ppl_median": 16.601248741149902 }, "results": [ { "precision": 0.156788215552466, "recall": 0.10761634910100995, "f1": 0.11682790615891263, "accuracy": 0.013836985105826744, "bleu": 0.45975591061692944, "self_ppl_mean": 2.129623557711418, "self_ppl_median": 1.9777633712182454, "distinct_1": 0.037811864720014784, "distinct_2": 0.11828823642295144, "rep_4gram": 0.3296551592928797, "avg_words": 27.055, "name": "greedy", "strategy": "greedy", "params": { "name": "greedy", "strategy": "greedy" }, "time_s": 25.106160163879395, "ms_per_prompt": 25.106160163879395, "gen_tokens_mean": 69.995, "judge_ppl_median": 3.7136934995651245, "judge_ppl_mean": 3.8431873861551287, "bertscore_p": 0.8487190008163452, "bertscore_r": 0.8463276028633118, "bertscore_f1": 0.847396731376648 }, { "precision": 0.1568385962211188, "recall": 0.10770206338672424, "f1": 0.11688205526981743, "accuracy": 0.013886833130142854, "bleu": 0.4603869452185167, "self_ppl_mean": 2.1293727944369447, "self_ppl_median": 1.977726615504968, "distinct_1": 0.03780348102435239, "distinct_2": 0.11810751697939449, "rep_4gram": 0.3296463873630552, "avg_words": 27.061, "name": "beam_w1", "strategy": "beam", "params": { "name": "beam_w1", "strategy": "beam", "width": 1 }, "time_s": 106.85117244720459, "ms_per_prompt": 106.85117244720459, "gen_tokens_mean": 69.995, "judge_ppl_median": 3.7136934995651245, "judge_ppl_mean": 3.839161422610283, "bertscore_p": 0.8486897945404053, "bertscore_r": 0.8463218808174133, "bertscore_f1": 0.8473787903785706 }, { "precision": 0.20124718334613445, "recall": 0.08660508798842184, "f1": 0.11012341441029258, "accuracy": 0.010045733414015445, "bleu": 0.29556628142570907, "self_ppl_mean": 1.8974243870210339, "self_ppl_median": 1.8302103567597376, "distinct_1": 0.045782342657342656, "distinct_2": 0.13661581137309292, "rep_4gram": 0.20882754144393487, "avg_words": 18.304, "name": "beam_w2", "strategy": "beam", "params": { "name": "beam_w2", "strategy": "beam", "width": 2 }, "time_s": 104.73980450630188, "ms_per_prompt": 104.73980450630188, "gen_tokens_mean": 69.827, "judge_ppl_median": 4.08447003364563, "judge_ppl_mean": 5.812772500276566, "bertscore_p": 0.8609820604324341, "bertscore_r": 0.8465735912322998, "bertscore_f1": 0.8535703420639038 }, { "precision": 0.1976862812247924, "recall": 0.07404912467040367, "f1": 0.09568651928951545, "accuracy": 0.008065531697039307, "bleu": 0.340259807839598, "self_ppl_mean": 1.7576852720858818, "self_ppl_median": 1.739890666494589, "distinct_1": 0.040265558374264585, "distinct_2": 0.11690534603754207, "rep_4gram": 0.23782374503889867, "avg_words": 18.527, "name": "beam_w4", "strategy": "beam", "params": { "name": "beam_w4", "strategy": "beam", "width": 4 }, "time_s": 117.64894247055054, "ms_per_prompt": 117.64894247055054, "gen_tokens_mean": 69.253, "judge_ppl_median": 3.7220958471298218, "judge_ppl_mean": 549.211596192956, "bertscore_p": 0.8594744801521301, "bertscore_r": 0.8445522785186768, "bertscore_f1": 0.8517692685127258 }, { "precision": 0.20828856855773425, "recall": 0.16455016427602215, "f1": 0.16573516044843375, "accuracy": 0.015756478767379786, "bleu": 0.5589188575413241, "self_ppl_mean": 5.8339501863641905, "self_ppl_median": 5.838575948334771, "distinct_1": 0.07212438521339044, "distinct_2": 0.35802064558413893, "rep_4gram": 0.028893427447938293, "avg_words": 31.515, "name": "topk_k5", "strategy": "top_k", "params": { "name": "topk_k5", "strategy": "top_k", "k": 5 }, "time_s": 27.3364098072052, "ms_per_prompt": 27.3364098072052, "gen_tokens_mean": 69.697, "judge_ppl_median": 8.458185195922852, "judge_ppl_mean": 9.329122072458267, "bertscore_p": 0.8579514026641846, "bertscore_r": 0.8530384302139282, "bertscore_f1": 0.8553735613822937 }, { "precision": 0.1919000986912784, "recall": 0.1684283187083047, "f1": 0.1622514196784391, "accuracy": 0.013794441209097598, "bleu": 0.44804585179427503, "self_ppl_mean": 10.859201539564154, "self_ppl_median": 9.966973243847942, "distinct_1": 0.09244218671549662, "distinct_2": 0.4732998630762209, "rep_4gram": 0.004260111395545372, "avg_words": 33.859, "name": "topk_k20", "strategy": "top_k", "params": { "name": "topk_k20", "strategy": "top_k", "k": 20 }, "time_s": 27.339534044265747, "ms_per_prompt": 27.339534044265747, "gen_tokens_mean": 69.165, "judge_ppl_median": 14.761709213256836, "judge_ppl_mean": 50.361770061254504 }, { "precision": 0.17549255996014995, "recall": 0.16199041954465634, "f1": 0.15203213911919755, "accuracy": 0.011857919480724241, "bleu": 0.44096547944277925, "self_ppl_mean": 15.653030294977679, "self_ppl_median": 13.604209219791796, "distinct_1": 0.106938636169668, "distinct_2": 0.5367388758782201, "rep_4gram": 0.002577424448111978, "avg_words": 35.151, "name": "topk_k50", "strategy": "top_k", "params": { "name": "topk_k50", "strategy": "top_k", "k": 50 }, "time_s": 27.421632289886475, "ms_per_prompt": 27.421632289886475, "gen_tokens_mean": 68.635, "judge_ppl_median": 20.48440647125244, "judge_ppl_mean": 72.74511349225044 }, { "precision": 0.20069336987864045, "recall": 0.14639978279849664, "f1": 0.1519355828806502, "accuracy": 0.01273002667646736, "bleu": 0.5404882348932311, "self_ppl_mean": 7.576500820452772, "self_ppl_median": 6.75727633465444, "distinct_1": 0.09244264507422402, "distinct_2": 0.43183742591024554, "rep_4gram": 0.021025283581782708, "avg_words": 28.158, "name": "topk_k50_t0.7", "strategy": "top_k", "params": { "name": "topk_k50_t0.7", "strategy": "top_k", "k": 50, "temperature": 0.7 }, "time_s": 27.444777011871338, "ms_per_prompt": 27.444777011871338, "gen_tokens_mean": 69.563, "judge_ppl_median": 10.548301219940186, "judge_ppl_mean": 1104.2365723810196 }, { "precision": 0.21224675405441254, "recall": 0.14685590502611967, "f1": 0.15687408460192412, "accuracy": 0.013971389777601836, "bleu": 0.5397175501288309, "self_ppl_mean": 6.038022332522205, "self_ppl_median": 5.858416877967823, "distinct_1": 0.1009201217053411, "distinct_2": 0.4334855403348554, "rep_4gram": 0.02484618782785406, "avg_words": 27.279, "name": "topp_p0.5", "strategy": "top_p", "params": { "name": "topp_p0.5", "strategy": "top_p", "p": 0.5 }, "time_s": 31.041442394256592, "ms_per_prompt": 31.041442394256592, "gen_tokens_mean": 69.823, "judge_ppl_median": 9.4617280960083, "judge_ppl_mean": 1095.229723884821 }, { "precision": 0.1754558860783798, "recall": 0.15128935460540213, "f1": 0.14627172467982152, "accuracy": 0.010996779233892341, "bleu": 0.3233499868741911, "self_ppl_mean": 17.14533789994608, "self_ppl_median": 14.304176726370176, "distinct_1": 0.13880592419250545, "distinct_2": 0.5860382392174299, "rep_4gram": 0.003378257483535917, "avg_words": 32.477, "name": "topp_p0.8", "strategy": "top_p", "params": { "name": "topp_p0.8", "strategy": "top_p", "p": 0.8 }, "time_s": 30.962740898132324, "ms_per_prompt": 30.962740898132324, "gen_tokens_mean": 69.009, "judge_ppl_median": 23.211604118347168, "judge_ppl_mean": 74.37143414640427 }, { "precision": 0.15602106748659775, "recall": 0.14042348676499086, "f1": 0.13287360175699992, "accuracy": 0.00936425787467936, "bleu": 0.16536685767069867, "self_ppl_mean": 39.594243798258624, "self_ppl_median": 33.30959383748847, "distinct_1": 0.18891279361056262, "distinct_2": 0.701587687076895, "rep_4gram": 0.0007256760411977545, "avg_words": 33.931, "name": "topp_p0.95", "strategy": "top_p", "params": { "name": "topp_p0.95", "strategy": "top_p", "p": 0.95 }, "time_s": 30.961236238479614, "ms_per_prompt": 30.961236238479614, "gen_tokens_mean": 68.636, "judge_ppl_median": 54.274309158325195, "judge_ppl_mean": 297.4285540444851 }, { "precision": 0.19970724391003492, "recall": 0.14364941747564147, "f1": 0.15108641777377704, "accuracy": 0.01395932104782926, "bleu": 0.5577671411688147, "self_ppl_mean": 5.73871449144444, "self_ppl_median": 5.589571006013263, "distinct_1": 0.09601266733841946, "distinct_2": 0.408413901228116, "rep_4gram": 0.034714608915333796, "avg_words": 27.788, "name": "topp_p0.9_t0.7", "strategy": "top_p", "params": { "name": "topp_p0.9_t0.7", "strategy": "top_p", "p": 0.9, "temperature": 0.7 }, "time_s": 30.83731961250305, "ms_per_prompt": 30.83731961250305, "gen_tokens_mean": 69.865, "judge_ppl_median": 9.213471412658691, "judge_ppl_mean": 1094.454825058937 } ], "beam_timing": [ { "mode": "natural", "width": 1, "mean_s_per_prompt": 0.7533104753494263, "std_s": 0.017264227186726558, "mean_ms_per_token": 10.76157821927752, "mean_output_tokens": 70.0, "total_s": 75.33104753494263, "n_prompts": 100 }, { "mode": "natural", "width": 2, "mean_s_per_prompt": 0.7676023697853088, "std_s": 0.022590433953289194, "mean_ms_per_token": 10.987960138362206, "mean_output_tokens": 69.86, "total_s": 76.76023697853088, "n_prompts": 100 }, { "mode": "natural", "width": 4, "mean_s_per_prompt": 0.8113761901855469, "std_s": 0.03146275351297224, "mean_ms_per_token": 11.6640359597942, "mean_output_tokens": 69.59, "total_s": 81.13761901855469, "n_prompts": 100 }, { "mode": "natural", "width": 8, "mean_s_per_prompt": 0.8519934725761413, "std_s": 0.040385070552722405, "mean_ms_per_token": 12.310079472418668, "mean_output_tokens": 69.34, "total_s": 85.19934725761414, "n_prompts": 100 }, { "mode": "fixed", "width": 1, "mean_s_per_prompt": 0.7492986965179443, "std_s": 0.015134158474454455, "mean_ms_per_token": 10.704267093113488, "mean_output_tokens": 70.0, "total_s": 74.92986965179443, "n_prompts": 100 }, { "mode": "fixed", "width": 2, "mean_s_per_prompt": 0.7675032258033753, "std_s": 0.015123762200423012, "mean_ms_per_token": 10.964331797191074, "mean_output_tokens": 70.0, "total_s": 76.75032258033752, "n_prompts": 100 }, { "mode": "fixed", "width": 4, "mean_s_per_prompt": 0.8134244823455811, "std_s": 0.012396015269615033, "mean_ms_per_token": 11.620349747794014, "mean_output_tokens": 70.0, "total_s": 81.3424482345581, "n_prompts": 100 }, { "mode": "fixed", "width": 8, "mean_s_per_prompt": 0.8602101182937623, "std_s": 0.015484862038058947, "mean_ms_per_token": 12.288715975625175, "mean_output_tokens": 70.0, "total_s": 86.02101182937622, "n_prompts": 100 } ], "config": { "model_name": "EleutherAI/pythia-160m", "dataset": "hamishivi/ROCStories", "split": "test", "n_samples": 1000, "seed": 0, "max_new_tokens": 70, "batch_size": 32, "beam_batch_size": 8, "dtype": "float32", "length_penalty": 1.0, "configs": [ { "name": "greedy", "strategy": "greedy" }, { "name": "beam_w1", "strategy": "beam", "width": 1 }, { "name": "beam_w2", "strategy": "beam", "width": 2 }, { "name": "beam_w4", "strategy": "beam", "width": 4 }, { "name": "topk_k5", "strategy": "top_k", "k": 5 }, { "name": "topk_k20", "strategy": "top_k", "k": 20 }, { "name": "topk_k50", "strategy": "top_k", "k": 50 }, { "name": "topk_k50_t0.7", "strategy": "top_k", "k": 50, "temperature": 0.7 }, { "name": "topp_p0.5", "strategy": "top_p", "p": 0.5 }, { "name": "topp_p0.8", "strategy": "top_p", "p": 0.8 }, { "name": "topp_p0.95", "strategy": "top_p", "p": 0.95 }, { "name": "topp_p0.9_t0.7", "strategy": "top_p", "p": 0.9, "temperature": 0.7 } ], "timing_widths": [ 1, 2, 4, 8 ], "timing_samples": 100, "judge_model": "EleutherAI/pythia-410m", "use_bertscore": true, "out_dir": "/kaggle/working/outputs/decoding", "hf_user": null, "repo_name": "anlp-a2-decoding", "push_to_hub": true, "private": false } }