Download decoding_results.json from usrnotfound101/anlp-a2-decoding: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/usrnotfound101/anlp-a2-decoding/resolve/main/decoding_results.json
- Command line
-
hf download hf://usrnotfound101/anlp-a2-decoding/decoding_results.json
-
curl -L -o decoding_results.json https://huggingface.co/usrnotfound101/anlp-a2-decoding/resolve/main/decoding_results.json
14.4 kB
| { | |
| "summary": { | |
| "reference_ppl_mean": 27.426984312057495, | |
| "reference_ppl_median": 24.0395565032959, | |
| "n_samples": 1000, | |
| "model": "EleutherAI/pythia-160m", | |
| "max_new_tokens": 70, | |
| "beam_w1_equals_greedy_frac": 0.997, | |
| "reference_judge_ppl_median": 16.601248741149902 | |
| }, | |
| "results": [ | |
| { | |
| "precision": 0.156788215552466, | |
| "recall": 0.10761634910100995, | |
| "f1": 0.11682790615891263, | |
| "accuracy": 0.013836985105826744, | |
| "bleu": 0.45975591061692944, | |
| "self_ppl_mean": 2.129623557711418, | |
| "self_ppl_median": 1.9777633712182454, | |
| "distinct_1": 0.037811864720014784, | |
| "distinct_2": 0.11828823642295144, | |
| "rep_4gram": 0.3296551592928797, | |
| "avg_words": 27.055, | |
| "name": "greedy", | |
| "strategy": "greedy", | |
| "params": { | |
| "name": "greedy", | |
| "strategy": "greedy" | |
| }, | |
| "time_s": 25.106160163879395, | |
| "ms_per_prompt": 25.106160163879395, | |
| "gen_tokens_mean": 69.995, | |
| "judge_ppl_median": 3.7136934995651245, | |
| "judge_ppl_mean": 3.8431873861551287, | |
| "bertscore_p": 0.8487190008163452, | |
| "bertscore_r": 0.8463276028633118, | |
| "bertscore_f1": 0.847396731376648 | |
| }, | |
| { | |
| "precision": 0.1568385962211188, | |
| "recall": 0.10770206338672424, | |
| "f1": 0.11688205526981743, | |
| "accuracy": 0.013886833130142854, | |
| "bleu": 0.4603869452185167, | |
| "self_ppl_mean": 2.1293727944369447, | |
| "self_ppl_median": 1.977726615504968, | |
| "distinct_1": 0.03780348102435239, | |
| "distinct_2": 0.11810751697939449, | |
| "rep_4gram": 0.3296463873630552, | |
| "avg_words": 27.061, | |
| "name": "beam_w1", | |
| "strategy": "beam", | |
| "params": { | |
| "name": "beam_w1", | |
| "strategy": "beam", | |
| "width": 1 | |
| }, | |
| "time_s": 106.85117244720459, | |
| "ms_per_prompt": 106.85117244720459, | |
| "gen_tokens_mean": 69.995, | |
| "judge_ppl_median": 3.7136934995651245, | |
| "judge_ppl_mean": 3.839161422610283, | |
| "bertscore_p": 0.8486897945404053, | |
| "bertscore_r": 0.8463218808174133, | |
| "bertscore_f1": 0.8473787903785706 | |
| }, | |
| { | |
| "precision": 0.20124718334613445, | |
| "recall": 0.08660508798842184, | |
| "f1": 0.11012341441029258, | |
| "accuracy": 0.010045733414015445, | |
| "bleu": 0.29556628142570907, | |
| "self_ppl_mean": 1.8974243870210339, | |
| "self_ppl_median": 1.8302103567597376, | |
| "distinct_1": 0.045782342657342656, | |
| "distinct_2": 0.13661581137309292, | |
| "rep_4gram": 0.20882754144393487, | |
| "avg_words": 18.304, | |
| "name": "beam_w2", | |
| "strategy": "beam", | |
| "params": { | |
| "name": "beam_w2", | |
| "strategy": "beam", | |
| "width": 2 | |
| }, | |
| "time_s": 104.73980450630188, | |
| "ms_per_prompt": 104.73980450630188, | |
| "gen_tokens_mean": 69.827, | |
| "judge_ppl_median": 4.08447003364563, | |
| "judge_ppl_mean": 5.812772500276566, | |
| "bertscore_p": 0.8609820604324341, | |
| "bertscore_r": 0.8465735912322998, | |
| "bertscore_f1": 0.8535703420639038 | |
| }, | |
| { | |
| "precision": 0.1976862812247924, | |
| "recall": 0.07404912467040367, | |
| "f1": 0.09568651928951545, | |
| "accuracy": 0.008065531697039307, | |
| "bleu": 0.340259807839598, | |
| "self_ppl_mean": 1.7576852720858818, | |
| "self_ppl_median": 1.739890666494589, | |
| "distinct_1": 0.040265558374264585, | |
| "distinct_2": 0.11690534603754207, | |
| "rep_4gram": 0.23782374503889867, | |
| "avg_words": 18.527, | |
| "name": "beam_w4", | |
| "strategy": "beam", | |
| "params": { | |
| "name": "beam_w4", | |
| "strategy": "beam", | |
| "width": 4 | |
| }, | |
| "time_s": 117.64894247055054, | |
| "ms_per_prompt": 117.64894247055054, | |
| "gen_tokens_mean": 69.253, | |
| "judge_ppl_median": 3.7220958471298218, | |
| "judge_ppl_mean": 549.211596192956, | |
| "bertscore_p": 0.8594744801521301, | |
| "bertscore_r": 0.8445522785186768, | |
| "bertscore_f1": 0.8517692685127258 | |
| }, | |
| { | |
| "precision": 0.20828856855773425, | |
| "recall": 0.16455016427602215, | |
| "f1": 0.16573516044843375, | |
| "accuracy": 0.015756478767379786, | |
| "bleu": 0.5589188575413241, | |
| "self_ppl_mean": 5.8339501863641905, | |
| "self_ppl_median": 5.838575948334771, | |
| "distinct_1": 0.07212438521339044, | |
| "distinct_2": 0.35802064558413893, | |
| "rep_4gram": 0.028893427447938293, | |
| "avg_words": 31.515, | |
| "name": "topk_k5", | |
| "strategy": "top_k", | |
| "params": { | |
| "name": "topk_k5", | |
| "strategy": "top_k", | |
| "k": 5 | |
| }, | |
| "time_s": 27.3364098072052, | |
| "ms_per_prompt": 27.3364098072052, | |
| "gen_tokens_mean": 69.697, | |
| "judge_ppl_median": 8.458185195922852, | |
| "judge_ppl_mean": 9.329122072458267, | |
| "bertscore_p": 0.8579514026641846, | |
| "bertscore_r": 0.8530384302139282, | |
| "bertscore_f1": 0.8553735613822937 | |
| }, | |
| { | |
| "precision": 0.1919000986912784, | |
| "recall": 0.1684283187083047, | |
| "f1": 0.1622514196784391, | |
| "accuracy": 0.013794441209097598, | |
| "bleu": 0.44804585179427503, | |
| "self_ppl_mean": 10.859201539564154, | |
| "self_ppl_median": 9.966973243847942, | |
| "distinct_1": 0.09244218671549662, | |
| "distinct_2": 0.4732998630762209, | |
| "rep_4gram": 0.004260111395545372, | |
| "avg_words": 33.859, | |
| "name": "topk_k20", | |
| "strategy": "top_k", | |
| "params": { | |
| "name": "topk_k20", | |
| "strategy": "top_k", | |
| "k": 20 | |
| }, | |
| "time_s": 27.339534044265747, | |
| "ms_per_prompt": 27.339534044265747, | |
| "gen_tokens_mean": 69.165, | |
| "judge_ppl_median": 14.761709213256836, | |
| "judge_ppl_mean": 50.361770061254504 | |
| }, | |
| { | |
| "precision": 0.17549255996014995, | |
| "recall": 0.16199041954465634, | |
| "f1": 0.15203213911919755, | |
| "accuracy": 0.011857919480724241, | |
| "bleu": 0.44096547944277925, | |
| "self_ppl_mean": 15.653030294977679, | |
| "self_ppl_median": 13.604209219791796, | |
| "distinct_1": 0.106938636169668, | |
| "distinct_2": 0.5367388758782201, | |
| "rep_4gram": 0.002577424448111978, | |
| "avg_words": 35.151, | |
| "name": "topk_k50", | |
| "strategy": "top_k", | |
| "params": { | |
| "name": "topk_k50", | |
| "strategy": "top_k", | |
| "k": 50 | |
| }, | |
| "time_s": 27.421632289886475, | |
| "ms_per_prompt": 27.421632289886475, | |
| "gen_tokens_mean": 68.635, | |
| "judge_ppl_median": 20.48440647125244, | |
| "judge_ppl_mean": 72.74511349225044 | |
| }, | |
| { | |
| "precision": 0.20069336987864045, | |
| "recall": 0.14639978279849664, | |
| "f1": 0.1519355828806502, | |
| "accuracy": 0.01273002667646736, | |
| "bleu": 0.5404882348932311, | |
| "self_ppl_mean": 7.576500820452772, | |
| "self_ppl_median": 6.75727633465444, | |
| "distinct_1": 0.09244264507422402, | |
| "distinct_2": 0.43183742591024554, | |
| "rep_4gram": 0.021025283581782708, | |
| "avg_words": 28.158, | |
| "name": "topk_k50_t0.7", | |
| "strategy": "top_k", | |
| "params": { | |
| "name": "topk_k50_t0.7", | |
| "strategy": "top_k", | |
| "k": 50, | |
| "temperature": 0.7 | |
| }, | |
| "time_s": 27.444777011871338, | |
| "ms_per_prompt": 27.444777011871338, | |
| "gen_tokens_mean": 69.563, | |
| "judge_ppl_median": 10.548301219940186, | |
| "judge_ppl_mean": 1104.2365723810196 | |
| }, | |
| { | |
| "precision": 0.21224675405441254, | |
| "recall": 0.14685590502611967, | |
| "f1": 0.15687408460192412, | |
| "accuracy": 0.013971389777601836, | |
| "bleu": 0.5397175501288309, | |
| "self_ppl_mean": 6.038022332522205, | |
| "self_ppl_median": 5.858416877967823, | |
| "distinct_1": 0.1009201217053411, | |
| "distinct_2": 0.4334855403348554, | |
| "rep_4gram": 0.02484618782785406, | |
| "avg_words": 27.279, | |
| "name": "topp_p0.5", | |
| "strategy": "top_p", | |
| "params": { | |
| "name": "topp_p0.5", | |
| "strategy": "top_p", | |
| "p": 0.5 | |
| }, | |
| "time_s": 31.041442394256592, | |
| "ms_per_prompt": 31.041442394256592, | |
| "gen_tokens_mean": 69.823, | |
| "judge_ppl_median": 9.4617280960083, | |
| "judge_ppl_mean": 1095.229723884821 | |
| }, | |
| { | |
| "precision": 0.1754558860783798, | |
| "recall": 0.15128935460540213, | |
| "f1": 0.14627172467982152, | |
| "accuracy": 0.010996779233892341, | |
| "bleu": 0.3233499868741911, | |
| "self_ppl_mean": 17.14533789994608, | |
| "self_ppl_median": 14.304176726370176, | |
| "distinct_1": 0.13880592419250545, | |
| "distinct_2": 0.5860382392174299, | |
| "rep_4gram": 0.003378257483535917, | |
| "avg_words": 32.477, | |
| "name": "topp_p0.8", | |
| "strategy": "top_p", | |
| "params": { | |
| "name": "topp_p0.8", | |
| "strategy": "top_p", | |
| "p": 0.8 | |
| }, | |
| "time_s": 30.962740898132324, | |
| "ms_per_prompt": 30.962740898132324, | |
| "gen_tokens_mean": 69.009, | |
| "judge_ppl_median": 23.211604118347168, | |
| "judge_ppl_mean": 74.37143414640427 | |
| }, | |
| { | |
| "precision": 0.15602106748659775, | |
| "recall": 0.14042348676499086, | |
| "f1": 0.13287360175699992, | |
| "accuracy": 0.00936425787467936, | |
| "bleu": 0.16536685767069867, | |
| "self_ppl_mean": 39.594243798258624, | |
| "self_ppl_median": 33.30959383748847, | |
| "distinct_1": 0.18891279361056262, | |
| "distinct_2": 0.701587687076895, | |
| "rep_4gram": 0.0007256760411977545, | |
| "avg_words": 33.931, | |
| "name": "topp_p0.95", | |
| "strategy": "top_p", | |
| "params": { | |
| "name": "topp_p0.95", | |
| "strategy": "top_p", | |
| "p": 0.95 | |
| }, | |
| "time_s": 30.961236238479614, | |
| "ms_per_prompt": 30.961236238479614, | |
| "gen_tokens_mean": 68.636, | |
| "judge_ppl_median": 54.274309158325195, | |
| "judge_ppl_mean": 297.4285540444851 | |
| }, | |
| { | |
| "precision": 0.19970724391003492, | |
| "recall": 0.14364941747564147, | |
| "f1": 0.15108641777377704, | |
| "accuracy": 0.01395932104782926, | |
| "bleu": 0.5577671411688147, | |
| "self_ppl_mean": 5.73871449144444, | |
| "self_ppl_median": 5.589571006013263, | |
| "distinct_1": 0.09601266733841946, | |
| "distinct_2": 0.408413901228116, | |
| "rep_4gram": 0.034714608915333796, | |
| "avg_words": 27.788, | |
| "name": "topp_p0.9_t0.7", | |
| "strategy": "top_p", | |
| "params": { | |
| "name": "topp_p0.9_t0.7", | |
| "strategy": "top_p", | |
| "p": 0.9, | |
| "temperature": 0.7 | |
| }, | |
| "time_s": 30.83731961250305, | |
| "ms_per_prompt": 30.83731961250305, | |
| "gen_tokens_mean": 69.865, | |
| "judge_ppl_median": 9.213471412658691, | |
| "judge_ppl_mean": 1094.454825058937 | |
| } | |
| ], | |
| "beam_timing": [ | |
| { | |
| "mode": "natural", | |
| "width": 1, | |
| "mean_s_per_prompt": 0.7533104753494263, | |
| "std_s": 0.017264227186726558, | |
| "mean_ms_per_token": 10.76157821927752, | |
| "mean_output_tokens": 70.0, | |
| "total_s": 75.33104753494263, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "natural", | |
| "width": 2, | |
| "mean_s_per_prompt": 0.7676023697853088, | |
| "std_s": 0.022590433953289194, | |
| "mean_ms_per_token": 10.987960138362206, | |
| "mean_output_tokens": 69.86, | |
| "total_s": 76.76023697853088, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "natural", | |
| "width": 4, | |
| "mean_s_per_prompt": 0.8113761901855469, | |
| "std_s": 0.03146275351297224, | |
| "mean_ms_per_token": 11.6640359597942, | |
| "mean_output_tokens": 69.59, | |
| "total_s": 81.13761901855469, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "natural", | |
| "width": 8, | |
| "mean_s_per_prompt": 0.8519934725761413, | |
| "std_s": 0.040385070552722405, | |
| "mean_ms_per_token": 12.310079472418668, | |
| "mean_output_tokens": 69.34, | |
| "total_s": 85.19934725761414, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "fixed", | |
| "width": 1, | |
| "mean_s_per_prompt": 0.7492986965179443, | |
| "std_s": 0.015134158474454455, | |
| "mean_ms_per_token": 10.704267093113488, | |
| "mean_output_tokens": 70.0, | |
| "total_s": 74.92986965179443, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "fixed", | |
| "width": 2, | |
| "mean_s_per_prompt": 0.7675032258033753, | |
| "std_s": 0.015123762200423012, | |
| "mean_ms_per_token": 10.964331797191074, | |
| "mean_output_tokens": 70.0, | |
| "total_s": 76.75032258033752, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "fixed", | |
| "width": 4, | |
| "mean_s_per_prompt": 0.8134244823455811, | |
| "std_s": 0.012396015269615033, | |
| "mean_ms_per_token": 11.620349747794014, | |
| "mean_output_tokens": 70.0, | |
| "total_s": 81.3424482345581, | |
| "n_prompts": 100 | |
| }, | |
| { | |
| "mode": "fixed", | |
| "width": 8, | |
| "mean_s_per_prompt": 0.8602101182937623, | |
| "std_s": 0.015484862038058947, | |
| "mean_ms_per_token": 12.288715975625175, | |
| "mean_output_tokens": 70.0, | |
| "total_s": 86.02101182937622, | |
| "n_prompts": 100 | |
| } | |
| ], | |
| "config": { | |
| "model_name": "EleutherAI/pythia-160m", | |
| "dataset": "hamishivi/ROCStories", | |
| "split": "test", | |
| "n_samples": 1000, | |
| "seed": 0, | |
| "max_new_tokens": 70, | |
| "batch_size": 32, | |
| "beam_batch_size": 8, | |
| "dtype": "float32", | |
| "length_penalty": 1.0, | |
| "configs": [ | |
| { | |
| "name": "greedy", | |
| "strategy": "greedy" | |
| }, | |
| { | |
| "name": "beam_w1", | |
| "strategy": "beam", | |
| "width": 1 | |
| }, | |
| { | |
| "name": "beam_w2", | |
| "strategy": "beam", | |
| "width": 2 | |
| }, | |
| { | |
| "name": "beam_w4", | |
| "strategy": "beam", | |
| "width": 4 | |
| }, | |
| { | |
| "name": "topk_k5", | |
| "strategy": "top_k", | |
| "k": 5 | |
| }, | |
| { | |
| "name": "topk_k20", | |
| "strategy": "top_k", | |
| "k": 20 | |
| }, | |
| { | |
| "name": "topk_k50", | |
| "strategy": "top_k", | |
| "k": 50 | |
| }, | |
| { | |
| "name": "topk_k50_t0.7", | |
| "strategy": "top_k", | |
| "k": 50, | |
| "temperature": 0.7 | |
| }, | |
| { | |
| "name": "topp_p0.5", | |
| "strategy": "top_p", | |
| "p": 0.5 | |
| }, | |
| { | |
| "name": "topp_p0.8", | |
| "strategy": "top_p", | |
| "p": 0.8 | |
| }, | |
| { | |
| "name": "topp_p0.95", | |
| "strategy": "top_p", | |
| "p": 0.95 | |
| }, | |
| { | |
| "name": "topp_p0.9_t0.7", | |
| "strategy": "top_p", | |
| "p": 0.9, | |
| "temperature": 0.7 | |
| } | |
| ], | |
| "timing_widths": [ | |
| 1, | |
| 2, | |
| 4, | |
| 8 | |
| ], | |
| "timing_samples": 100, | |
| "judge_model": "EleutherAI/pythia-410m", | |
| "use_bertscore": true, | |
| "out_dir": "/kaggle/working/outputs/decoding", | |
| "hf_user": null, | |
| "repo_name": "anlp-a2-decoding", | |
| "push_to_hub": true, | |
| "private": false | |
| } | |
| } |