anlp-a2-decoding / decoding_results.json
usrnotfound101's picture
decoding results
bcc2233 verified
Raw History Blame Contribute Delete
14.4 kB
{
"summary": {
"reference_ppl_mean": 27.426984312057495,
"reference_ppl_median": 24.0395565032959,
"n_samples": 1000,
"model": "EleutherAI/pythia-160m",
"max_new_tokens": 70,
"beam_w1_equals_greedy_frac": 0.997,
"reference_judge_ppl_median": 16.601248741149902
},
"results": [
{
"precision": 0.156788215552466,
"recall": 0.10761634910100995,
"f1": 0.11682790615891263,
"accuracy": 0.013836985105826744,
"bleu": 0.45975591061692944,
"self_ppl_mean": 2.129623557711418,
"self_ppl_median": 1.9777633712182454,
"distinct_1": 0.037811864720014784,
"distinct_2": 0.11828823642295144,
"rep_4gram": 0.3296551592928797,
"avg_words": 27.055,
"name": "greedy",
"strategy": "greedy",
"params": {
"name": "greedy",
"strategy": "greedy"
},
"time_s": 25.106160163879395,
"ms_per_prompt": 25.106160163879395,
"gen_tokens_mean": 69.995,
"judge_ppl_median": 3.7136934995651245,
"judge_ppl_mean": 3.8431873861551287,
"bertscore_p": 0.8487190008163452,
"bertscore_r": 0.8463276028633118,
"bertscore_f1": 0.847396731376648
},
{
"precision": 0.1568385962211188,
"recall": 0.10770206338672424,
"f1": 0.11688205526981743,
"accuracy": 0.013886833130142854,
"bleu": 0.4603869452185167,
"self_ppl_mean": 2.1293727944369447,
"self_ppl_median": 1.977726615504968,
"distinct_1": 0.03780348102435239,
"distinct_2": 0.11810751697939449,
"rep_4gram": 0.3296463873630552,
"avg_words": 27.061,
"name": "beam_w1",
"strategy": "beam",
"params": {
"name": "beam_w1",
"strategy": "beam",
"width": 1
},
"time_s": 106.85117244720459,
"ms_per_prompt": 106.85117244720459,
"gen_tokens_mean": 69.995,
"judge_ppl_median": 3.7136934995651245,
"judge_ppl_mean": 3.839161422610283,
"bertscore_p": 0.8486897945404053,
"bertscore_r": 0.8463218808174133,
"bertscore_f1": 0.8473787903785706
},
{
"precision": 0.20124718334613445,
"recall": 0.08660508798842184,
"f1": 0.11012341441029258,
"accuracy": 0.010045733414015445,
"bleu": 0.29556628142570907,
"self_ppl_mean": 1.8974243870210339,
"self_ppl_median": 1.8302103567597376,
"distinct_1": 0.045782342657342656,
"distinct_2": 0.13661581137309292,
"rep_4gram": 0.20882754144393487,
"avg_words": 18.304,
"name": "beam_w2",
"strategy": "beam",
"params": {
"name": "beam_w2",
"strategy": "beam",
"width": 2
},
"time_s": 104.73980450630188,
"ms_per_prompt": 104.73980450630188,
"gen_tokens_mean": 69.827,
"judge_ppl_median": 4.08447003364563,
"judge_ppl_mean": 5.812772500276566,
"bertscore_p": 0.8609820604324341,
"bertscore_r": 0.8465735912322998,
"bertscore_f1": 0.8535703420639038
},
{
"precision": 0.1976862812247924,
"recall": 0.07404912467040367,
"f1": 0.09568651928951545,
"accuracy": 0.008065531697039307,
"bleu": 0.340259807839598,
"self_ppl_mean": 1.7576852720858818,
"self_ppl_median": 1.739890666494589,
"distinct_1": 0.040265558374264585,
"distinct_2": 0.11690534603754207,
"rep_4gram": 0.23782374503889867,
"avg_words": 18.527,
"name": "beam_w4",
"strategy": "beam",
"params": {
"name": "beam_w4",
"strategy": "beam",
"width": 4
},
"time_s": 117.64894247055054,
"ms_per_prompt": 117.64894247055054,
"gen_tokens_mean": 69.253,
"judge_ppl_median": 3.7220958471298218,
"judge_ppl_mean": 549.211596192956,
"bertscore_p": 0.8594744801521301,
"bertscore_r": 0.8445522785186768,
"bertscore_f1": 0.8517692685127258
},
{
"precision": 0.20828856855773425,
"recall": 0.16455016427602215,
"f1": 0.16573516044843375,
"accuracy": 0.015756478767379786,
"bleu": 0.5589188575413241,
"self_ppl_mean": 5.8339501863641905,
"self_ppl_median": 5.838575948334771,
"distinct_1": 0.07212438521339044,
"distinct_2": 0.35802064558413893,
"rep_4gram": 0.028893427447938293,
"avg_words": 31.515,
"name": "topk_k5",
"strategy": "top_k",
"params": {
"name": "topk_k5",
"strategy": "top_k",
"k": 5
},
"time_s": 27.3364098072052,
"ms_per_prompt": 27.3364098072052,
"gen_tokens_mean": 69.697,
"judge_ppl_median": 8.458185195922852,
"judge_ppl_mean": 9.329122072458267,
"bertscore_p": 0.8579514026641846,
"bertscore_r": 0.8530384302139282,
"bertscore_f1": 0.8553735613822937
},
{
"precision": 0.1919000986912784,
"recall": 0.1684283187083047,
"f1": 0.1622514196784391,
"accuracy": 0.013794441209097598,
"bleu": 0.44804585179427503,
"self_ppl_mean": 10.859201539564154,
"self_ppl_median": 9.966973243847942,
"distinct_1": 0.09244218671549662,
"distinct_2": 0.4732998630762209,
"rep_4gram": 0.004260111395545372,
"avg_words": 33.859,
"name": "topk_k20",
"strategy": "top_k",
"params": {
"name": "topk_k20",
"strategy": "top_k",
"k": 20
},
"time_s": 27.339534044265747,
"ms_per_prompt": 27.339534044265747,
"gen_tokens_mean": 69.165,
"judge_ppl_median": 14.761709213256836,
"judge_ppl_mean": 50.361770061254504
},
{
"precision": 0.17549255996014995,
"recall": 0.16199041954465634,
"f1": 0.15203213911919755,
"accuracy": 0.011857919480724241,
"bleu": 0.44096547944277925,
"self_ppl_mean": 15.653030294977679,
"self_ppl_median": 13.604209219791796,
"distinct_1": 0.106938636169668,
"distinct_2": 0.5367388758782201,
"rep_4gram": 0.002577424448111978,
"avg_words": 35.151,
"name": "topk_k50",
"strategy": "top_k",
"params": {
"name": "topk_k50",
"strategy": "top_k",
"k": 50
},
"time_s": 27.421632289886475,
"ms_per_prompt": 27.421632289886475,
"gen_tokens_mean": 68.635,
"judge_ppl_median": 20.48440647125244,
"judge_ppl_mean": 72.74511349225044
},
{
"precision": 0.20069336987864045,
"recall": 0.14639978279849664,
"f1": 0.1519355828806502,
"accuracy": 0.01273002667646736,
"bleu": 0.5404882348932311,
"self_ppl_mean": 7.576500820452772,
"self_ppl_median": 6.75727633465444,
"distinct_1": 0.09244264507422402,
"distinct_2": 0.43183742591024554,
"rep_4gram": 0.021025283581782708,
"avg_words": 28.158,
"name": "topk_k50_t0.7",
"strategy": "top_k",
"params": {
"name": "topk_k50_t0.7",
"strategy": "top_k",
"k": 50,
"temperature": 0.7
},
"time_s": 27.444777011871338,
"ms_per_prompt": 27.444777011871338,
"gen_tokens_mean": 69.563,
"judge_ppl_median": 10.548301219940186,
"judge_ppl_mean": 1104.2365723810196
},
{
"precision": 0.21224675405441254,
"recall": 0.14685590502611967,
"f1": 0.15687408460192412,
"accuracy": 0.013971389777601836,
"bleu": 0.5397175501288309,
"self_ppl_mean": 6.038022332522205,
"self_ppl_median": 5.858416877967823,
"distinct_1": 0.1009201217053411,
"distinct_2": 0.4334855403348554,
"rep_4gram": 0.02484618782785406,
"avg_words": 27.279,
"name": "topp_p0.5",
"strategy": "top_p",
"params": {
"name": "topp_p0.5",
"strategy": "top_p",
"p": 0.5
},
"time_s": 31.041442394256592,
"ms_per_prompt": 31.041442394256592,
"gen_tokens_mean": 69.823,
"judge_ppl_median": 9.4617280960083,
"judge_ppl_mean": 1095.229723884821
},
{
"precision": 0.1754558860783798,
"recall": 0.15128935460540213,
"f1": 0.14627172467982152,
"accuracy": 0.010996779233892341,
"bleu": 0.3233499868741911,
"self_ppl_mean": 17.14533789994608,
"self_ppl_median": 14.304176726370176,
"distinct_1": 0.13880592419250545,
"distinct_2": 0.5860382392174299,
"rep_4gram": 0.003378257483535917,
"avg_words": 32.477,
"name": "topp_p0.8",
"strategy": "top_p",
"params": {
"name": "topp_p0.8",
"strategy": "top_p",
"p": 0.8
},
"time_s": 30.962740898132324,
"ms_per_prompt": 30.962740898132324,
"gen_tokens_mean": 69.009,
"judge_ppl_median": 23.211604118347168,
"judge_ppl_mean": 74.37143414640427
},
{
"precision": 0.15602106748659775,
"recall": 0.14042348676499086,
"f1": 0.13287360175699992,
"accuracy": 0.00936425787467936,
"bleu": 0.16536685767069867,
"self_ppl_mean": 39.594243798258624,
"self_ppl_median": 33.30959383748847,
"distinct_1": 0.18891279361056262,
"distinct_2": 0.701587687076895,
"rep_4gram": 0.0007256760411977545,
"avg_words": 33.931,
"name": "topp_p0.95",
"strategy": "top_p",
"params": {
"name": "topp_p0.95",
"strategy": "top_p",
"p": 0.95
},
"time_s": 30.961236238479614,
"ms_per_prompt": 30.961236238479614,
"gen_tokens_mean": 68.636,
"judge_ppl_median": 54.274309158325195,
"judge_ppl_mean": 297.4285540444851
},
{
"precision": 0.19970724391003492,
"recall": 0.14364941747564147,
"f1": 0.15108641777377704,
"accuracy": 0.01395932104782926,
"bleu": 0.5577671411688147,
"self_ppl_mean": 5.73871449144444,
"self_ppl_median": 5.589571006013263,
"distinct_1": 0.09601266733841946,
"distinct_2": 0.408413901228116,
"rep_4gram": 0.034714608915333796,
"avg_words": 27.788,
"name": "topp_p0.9_t0.7",
"strategy": "top_p",
"params": {
"name": "topp_p0.9_t0.7",
"strategy": "top_p",
"p": 0.9,
"temperature": 0.7
},
"time_s": 30.83731961250305,
"ms_per_prompt": 30.83731961250305,
"gen_tokens_mean": 69.865,
"judge_ppl_median": 9.213471412658691,
"judge_ppl_mean": 1094.454825058937
}
],
"beam_timing": [
{
"mode": "natural",
"width": 1,
"mean_s_per_prompt": 0.7533104753494263,
"std_s": 0.017264227186726558,
"mean_ms_per_token": 10.76157821927752,
"mean_output_tokens": 70.0,
"total_s": 75.33104753494263,
"n_prompts": 100
},
{
"mode": "natural",
"width": 2,
"mean_s_per_prompt": 0.7676023697853088,
"std_s": 0.022590433953289194,
"mean_ms_per_token": 10.987960138362206,
"mean_output_tokens": 69.86,
"total_s": 76.76023697853088,
"n_prompts": 100
},
{
"mode": "natural",
"width": 4,
"mean_s_per_prompt": 0.8113761901855469,
"std_s": 0.03146275351297224,
"mean_ms_per_token": 11.6640359597942,
"mean_output_tokens": 69.59,
"total_s": 81.13761901855469,
"n_prompts": 100
},
{
"mode": "natural",
"width": 8,
"mean_s_per_prompt": 0.8519934725761413,
"std_s": 0.040385070552722405,
"mean_ms_per_token": 12.310079472418668,
"mean_output_tokens": 69.34,
"total_s": 85.19934725761414,
"n_prompts": 100
},
{
"mode": "fixed",
"width": 1,
"mean_s_per_prompt": 0.7492986965179443,
"std_s": 0.015134158474454455,
"mean_ms_per_token": 10.704267093113488,
"mean_output_tokens": 70.0,
"total_s": 74.92986965179443,
"n_prompts": 100
},
{
"mode": "fixed",
"width": 2,
"mean_s_per_prompt": 0.7675032258033753,
"std_s": 0.015123762200423012,
"mean_ms_per_token": 10.964331797191074,
"mean_output_tokens": 70.0,
"total_s": 76.75032258033752,
"n_prompts": 100
},
{
"mode": "fixed",
"width": 4,
"mean_s_per_prompt": 0.8134244823455811,
"std_s": 0.012396015269615033,
"mean_ms_per_token": 11.620349747794014,
"mean_output_tokens": 70.0,
"total_s": 81.3424482345581,
"n_prompts": 100
},
{
"mode": "fixed",
"width": 8,
"mean_s_per_prompt": 0.8602101182937623,
"std_s": 0.015484862038058947,
"mean_ms_per_token": 12.288715975625175,
"mean_output_tokens": 70.0,
"total_s": 86.02101182937622,
"n_prompts": 100
}
],
"config": {
"model_name": "EleutherAI/pythia-160m",
"dataset": "hamishivi/ROCStories",
"split": "test",
"n_samples": 1000,
"seed": 0,
"max_new_tokens": 70,
"batch_size": 32,
"beam_batch_size": 8,
"dtype": "float32",
"length_penalty": 1.0,
"configs": [
{
"name": "greedy",
"strategy": "greedy"
},
{
"name": "beam_w1",
"strategy": "beam",
"width": 1
},
{
"name": "beam_w2",
"strategy": "beam",
"width": 2
},
{
"name": "beam_w4",
"strategy": "beam",
"width": 4
},
{
"name": "topk_k5",
"strategy": "top_k",
"k": 5
},
{
"name": "topk_k20",
"strategy": "top_k",
"k": 20
},
{
"name": "topk_k50",
"strategy": "top_k",
"k": 50
},
{
"name": "topk_k50_t0.7",
"strategy": "top_k",
"k": 50,
"temperature": 0.7
},
{
"name": "topp_p0.5",
"strategy": "top_p",
"p": 0.5
},
{
"name": "topp_p0.8",
"strategy": "top_p",
"p": 0.8
},
{
"name": "topp_p0.95",
"strategy": "top_p",
"p": 0.95
},
{
"name": "topp_p0.9_t0.7",
"strategy": "top_p",
"p": 0.9,
"temperature": 0.7
}
],
"timing_widths": [
1,
2,
4,
8
],
"timing_samples": 100,
"judge_model": "EleutherAI/pythia-410m",
"use_bertscore": true,
"out_dir": "/kaggle/working/outputs/decoding",
"hf_user": null,
"repo_name": "anlp-a2-decoding",
"push_to_hub": true,
"private": false
}
}