decoding results
Browse files- decoding_results.json +60 -60
decoding_results.json
CHANGED
|
@@ -27,8 +27,8 @@
|
|
| 27 |
"name": "greedy",
|
| 28 |
"strategy": "greedy"
|
| 29 |
},
|
| 30 |
-
"time_s":
|
| 31 |
-
"ms_per_prompt":
|
| 32 |
"gen_tokens_mean": 69.995,
|
| 33 |
"judge_ppl_median": 3.7136934995651245,
|
| 34 |
"judge_ppl_mean": 3.8431873861551287,
|
|
@@ -55,8 +55,8 @@
|
|
| 55 |
"strategy": "beam",
|
| 56 |
"width": 1
|
| 57 |
},
|
| 58 |
-
"time_s":
|
| 59 |
-
"ms_per_prompt":
|
| 60 |
"gen_tokens_mean": 69.995,
|
| 61 |
"judge_ppl_median": 3.7136934995651245,
|
| 62 |
"judge_ppl_mean": 3.839161422610283,
|
|
@@ -83,8 +83,8 @@
|
|
| 83 |
"strategy": "beam",
|
| 84 |
"width": 2
|
| 85 |
},
|
| 86 |
-
"time_s":
|
| 87 |
-
"ms_per_prompt":
|
| 88 |
"gen_tokens_mean": 69.827,
|
| 89 |
"judge_ppl_median": 4.08447003364563,
|
| 90 |
"judge_ppl_mean": 5.812772500276566,
|
|
@@ -111,14 +111,14 @@
|
|
| 111 |
"strategy": "beam",
|
| 112 |
"width": 4
|
| 113 |
},
|
| 114 |
-
"time_s":
|
| 115 |
-
"ms_per_prompt":
|
| 116 |
"gen_tokens_mean": 69.253,
|
| 117 |
"judge_ppl_median": 3.7220958471298218,
|
| 118 |
"judge_ppl_mean": 549.211596192956,
|
| 119 |
-
"bertscore_p": 0.
|
| 120 |
-
"bertscore_r": 0.
|
| 121 |
-
"bertscore_f1": 0.
|
| 122 |
},
|
| 123 |
{
|
| 124 |
"precision": 0.20828856855773425,
|
|
@@ -139,13 +139,13 @@
|
|
| 139 |
"strategy": "top_k",
|
| 140 |
"k": 5
|
| 141 |
},
|
| 142 |
-
"time_s":
|
| 143 |
-
"ms_per_prompt":
|
| 144 |
"gen_tokens_mean": 69.697,
|
| 145 |
"judge_ppl_median": 8.458185195922852,
|
| 146 |
"judge_ppl_mean": 9.329122072458267,
|
| 147 |
"bertscore_p": 0.8579514026641846,
|
| 148 |
-
"bertscore_r": 0.
|
| 149 |
"bertscore_f1": 0.8553735613822937
|
| 150 |
},
|
| 151 |
{
|
|
@@ -167,8 +167,8 @@
|
|
| 167 |
"strategy": "top_k",
|
| 168 |
"k": 20
|
| 169 |
},
|
| 170 |
-
"time_s":
|
| 171 |
-
"ms_per_prompt":
|
| 172 |
"gen_tokens_mean": 69.165,
|
| 173 |
"judge_ppl_median": 14.761709213256836,
|
| 174 |
"judge_ppl_mean": 50.361770061254504
|
|
@@ -192,8 +192,8 @@
|
|
| 192 |
"strategy": "top_k",
|
| 193 |
"k": 50
|
| 194 |
},
|
| 195 |
-
"time_s":
|
| 196 |
-
"ms_per_prompt":
|
| 197 |
"gen_tokens_mean": 68.635,
|
| 198 |
"judge_ppl_median": 20.48440647125244,
|
| 199 |
"judge_ppl_mean": 72.74511349225044
|
|
@@ -218,8 +218,8 @@
|
|
| 218 |
"k": 50,
|
| 219 |
"temperature": 0.7
|
| 220 |
},
|
| 221 |
-
"time_s":
|
| 222 |
-
"ms_per_prompt":
|
| 223 |
"gen_tokens_mean": 69.563,
|
| 224 |
"judge_ppl_median": 10.548301219940186,
|
| 225 |
"judge_ppl_mean": 1104.2365723810196
|
|
@@ -243,8 +243,8 @@
|
|
| 243 |
"strategy": "top_p",
|
| 244 |
"p": 0.5
|
| 245 |
},
|
| 246 |
-
"time_s":
|
| 247 |
-
"ms_per_prompt":
|
| 248 |
"gen_tokens_mean": 69.823,
|
| 249 |
"judge_ppl_median": 9.4617280960083,
|
| 250 |
"judge_ppl_mean": 1095.229723884821
|
|
@@ -268,8 +268,8 @@
|
|
| 268 |
"strategy": "top_p",
|
| 269 |
"p": 0.8
|
| 270 |
},
|
| 271 |
-
"time_s":
|
| 272 |
-
"ms_per_prompt":
|
| 273 |
"gen_tokens_mean": 69.009,
|
| 274 |
"judge_ppl_median": 23.211604118347168,
|
| 275 |
"judge_ppl_mean": 74.37143414640427
|
|
@@ -293,8 +293,8 @@
|
|
| 293 |
"strategy": "top_p",
|
| 294 |
"p": 0.95
|
| 295 |
},
|
| 296 |
-
"time_s":
|
| 297 |
-
"ms_per_prompt":
|
| 298 |
"gen_tokens_mean": 68.636,
|
| 299 |
"judge_ppl_median": 54.274309158325195,
|
| 300 |
"judge_ppl_mean": 297.4285540444851
|
|
@@ -319,8 +319,8 @@
|
|
| 319 |
"p": 0.9,
|
| 320 |
"temperature": 0.7
|
| 321 |
},
|
| 322 |
-
"time_s":
|
| 323 |
-
"ms_per_prompt":
|
| 324 |
"gen_tokens_mean": 69.865,
|
| 325 |
"judge_ppl_median": 9.213471412658691,
|
| 326 |
"judge_ppl_mean": 1094.454825058937
|
|
@@ -330,81 +330,81 @@
|
|
| 330 |
{
|
| 331 |
"mode": "natural",
|
| 332 |
"width": 1,
|
| 333 |
-
"mean_s_per_prompt": 0.
|
| 334 |
-
"std_s": 0.
|
| 335 |
-
"mean_ms_per_token":
|
| 336 |
"mean_output_tokens": 70.0,
|
| 337 |
-
"total_s":
|
| 338 |
"n_prompts": 100
|
| 339 |
},
|
| 340 |
{
|
| 341 |
"mode": "natural",
|
| 342 |
"width": 2,
|
| 343 |
-
"mean_s_per_prompt": 0.
|
| 344 |
-
"std_s": 0.
|
| 345 |
-
"mean_ms_per_token":
|
| 346 |
"mean_output_tokens": 69.86,
|
| 347 |
-
"total_s":
|
| 348 |
"n_prompts": 100
|
| 349 |
},
|
| 350 |
{
|
| 351 |
"mode": "natural",
|
| 352 |
"width": 4,
|
| 353 |
-
"mean_s_per_prompt": 0.
|
| 354 |
-
"std_s": 0.
|
| 355 |
-
"mean_ms_per_token":
|
| 356 |
"mean_output_tokens": 69.59,
|
| 357 |
-
"total_s":
|
| 358 |
"n_prompts": 100
|
| 359 |
},
|
| 360 |
{
|
| 361 |
"mode": "natural",
|
| 362 |
"width": 8,
|
| 363 |
-
"mean_s_per_prompt": 0.
|
| 364 |
-
"std_s": 0.
|
| 365 |
-
"mean_ms_per_token":
|
| 366 |
"mean_output_tokens": 69.34,
|
| 367 |
-
"total_s":
|
| 368 |
"n_prompts": 100
|
| 369 |
},
|
| 370 |
{
|
| 371 |
"mode": "fixed",
|
| 372 |
"width": 1,
|
| 373 |
-
"mean_s_per_prompt": 0.
|
| 374 |
-
"std_s": 0.
|
| 375 |
-
"mean_ms_per_token":
|
| 376 |
"mean_output_tokens": 70.0,
|
| 377 |
-
"total_s":
|
| 378 |
"n_prompts": 100
|
| 379 |
},
|
| 380 |
{
|
| 381 |
"mode": "fixed",
|
| 382 |
"width": 2,
|
| 383 |
-
"mean_s_per_prompt": 0.
|
| 384 |
-
"std_s": 0.
|
| 385 |
-
"mean_ms_per_token":
|
| 386 |
"mean_output_tokens": 70.0,
|
| 387 |
-
"total_s":
|
| 388 |
"n_prompts": 100
|
| 389 |
},
|
| 390 |
{
|
| 391 |
"mode": "fixed",
|
| 392 |
"width": 4,
|
| 393 |
-
"mean_s_per_prompt": 0.
|
| 394 |
-
"std_s": 0.
|
| 395 |
-
"mean_ms_per_token":
|
| 396 |
"mean_output_tokens": 70.0,
|
| 397 |
-
"total_s":
|
| 398 |
"n_prompts": 100
|
| 399 |
},
|
| 400 |
{
|
| 401 |
"mode": "fixed",
|
| 402 |
"width": 8,
|
| 403 |
-
"mean_s_per_prompt": 0.
|
| 404 |
-
"std_s": 0.
|
| 405 |
-
"mean_ms_per_token": 12.
|
| 406 |
"mean_output_tokens": 70.0,
|
| 407 |
-
"total_s":
|
| 408 |
"n_prompts": 100
|
| 409 |
}
|
| 410 |
],
|
|
|
|
| 27 |
"name": "greedy",
|
| 28 |
"strategy": "greedy"
|
| 29 |
},
|
| 30 |
+
"time_s": 25.106160163879395,
|
| 31 |
+
"ms_per_prompt": 25.106160163879395,
|
| 32 |
"gen_tokens_mean": 69.995,
|
| 33 |
"judge_ppl_median": 3.7136934995651245,
|
| 34 |
"judge_ppl_mean": 3.8431873861551287,
|
|
|
|
| 55 |
"strategy": "beam",
|
| 56 |
"width": 1
|
| 57 |
},
|
| 58 |
+
"time_s": 106.85117244720459,
|
| 59 |
+
"ms_per_prompt": 106.85117244720459,
|
| 60 |
"gen_tokens_mean": 69.995,
|
| 61 |
"judge_ppl_median": 3.7136934995651245,
|
| 62 |
"judge_ppl_mean": 3.839161422610283,
|
|
|
|
| 83 |
"strategy": "beam",
|
| 84 |
"width": 2
|
| 85 |
},
|
| 86 |
+
"time_s": 104.73980450630188,
|
| 87 |
+
"ms_per_prompt": 104.73980450630188,
|
| 88 |
"gen_tokens_mean": 69.827,
|
| 89 |
"judge_ppl_median": 4.08447003364563,
|
| 90 |
"judge_ppl_mean": 5.812772500276566,
|
|
|
|
| 111 |
"strategy": "beam",
|
| 112 |
"width": 4
|
| 113 |
},
|
| 114 |
+
"time_s": 117.64894247055054,
|
| 115 |
+
"ms_per_prompt": 117.64894247055054,
|
| 116 |
"gen_tokens_mean": 69.253,
|
| 117 |
"judge_ppl_median": 3.7220958471298218,
|
| 118 |
"judge_ppl_mean": 549.211596192956,
|
| 119 |
+
"bertscore_p": 0.8594744801521301,
|
| 120 |
+
"bertscore_r": 0.8445522785186768,
|
| 121 |
+
"bertscore_f1": 0.8517692685127258
|
| 122 |
},
|
| 123 |
{
|
| 124 |
"precision": 0.20828856855773425,
|
|
|
|
| 139 |
"strategy": "top_k",
|
| 140 |
"k": 5
|
| 141 |
},
|
| 142 |
+
"time_s": 27.3364098072052,
|
| 143 |
+
"ms_per_prompt": 27.3364098072052,
|
| 144 |
"gen_tokens_mean": 69.697,
|
| 145 |
"judge_ppl_median": 8.458185195922852,
|
| 146 |
"judge_ppl_mean": 9.329122072458267,
|
| 147 |
"bertscore_p": 0.8579514026641846,
|
| 148 |
+
"bertscore_r": 0.8530384302139282,
|
| 149 |
"bertscore_f1": 0.8553735613822937
|
| 150 |
},
|
| 151 |
{
|
|
|
|
| 167 |
"strategy": "top_k",
|
| 168 |
"k": 20
|
| 169 |
},
|
| 170 |
+
"time_s": 27.339534044265747,
|
| 171 |
+
"ms_per_prompt": 27.339534044265747,
|
| 172 |
"gen_tokens_mean": 69.165,
|
| 173 |
"judge_ppl_median": 14.761709213256836,
|
| 174 |
"judge_ppl_mean": 50.361770061254504
|
|
|
|
| 192 |
"strategy": "top_k",
|
| 193 |
"k": 50
|
| 194 |
},
|
| 195 |
+
"time_s": 27.421632289886475,
|
| 196 |
+
"ms_per_prompt": 27.421632289886475,
|
| 197 |
"gen_tokens_mean": 68.635,
|
| 198 |
"judge_ppl_median": 20.48440647125244,
|
| 199 |
"judge_ppl_mean": 72.74511349225044
|
|
|
|
| 218 |
"k": 50,
|
| 219 |
"temperature": 0.7
|
| 220 |
},
|
| 221 |
+
"time_s": 27.444777011871338,
|
| 222 |
+
"ms_per_prompt": 27.444777011871338,
|
| 223 |
"gen_tokens_mean": 69.563,
|
| 224 |
"judge_ppl_median": 10.548301219940186,
|
| 225 |
"judge_ppl_mean": 1104.2365723810196
|
|
|
|
| 243 |
"strategy": "top_p",
|
| 244 |
"p": 0.5
|
| 245 |
},
|
| 246 |
+
"time_s": 31.041442394256592,
|
| 247 |
+
"ms_per_prompt": 31.041442394256592,
|
| 248 |
"gen_tokens_mean": 69.823,
|
| 249 |
"judge_ppl_median": 9.4617280960083,
|
| 250 |
"judge_ppl_mean": 1095.229723884821
|
|
|
|
| 268 |
"strategy": "top_p",
|
| 269 |
"p": 0.8
|
| 270 |
},
|
| 271 |
+
"time_s": 30.962740898132324,
|
| 272 |
+
"ms_per_prompt": 30.962740898132324,
|
| 273 |
"gen_tokens_mean": 69.009,
|
| 274 |
"judge_ppl_median": 23.211604118347168,
|
| 275 |
"judge_ppl_mean": 74.37143414640427
|
|
|
|
| 293 |
"strategy": "top_p",
|
| 294 |
"p": 0.95
|
| 295 |
},
|
| 296 |
+
"time_s": 30.961236238479614,
|
| 297 |
+
"ms_per_prompt": 30.961236238479614,
|
| 298 |
"gen_tokens_mean": 68.636,
|
| 299 |
"judge_ppl_median": 54.274309158325195,
|
| 300 |
"judge_ppl_mean": 297.4285540444851
|
|
|
|
| 319 |
"p": 0.9,
|
| 320 |
"temperature": 0.7
|
| 321 |
},
|
| 322 |
+
"time_s": 30.83731961250305,
|
| 323 |
+
"ms_per_prompt": 30.83731961250305,
|
| 324 |
"gen_tokens_mean": 69.865,
|
| 325 |
"judge_ppl_median": 9.213471412658691,
|
| 326 |
"judge_ppl_mean": 1094.454825058937
|
|
|
|
| 330 |
{
|
| 331 |
"mode": "natural",
|
| 332 |
"width": 1,
|
| 333 |
+
"mean_s_per_prompt": 0.7533104753494263,
|
| 334 |
+
"std_s": 0.017264227186726558,
|
| 335 |
+
"mean_ms_per_token": 10.76157821927752,
|
| 336 |
"mean_output_tokens": 70.0,
|
| 337 |
+
"total_s": 75.33104753494263,
|
| 338 |
"n_prompts": 100
|
| 339 |
},
|
| 340 |
{
|
| 341 |
"mode": "natural",
|
| 342 |
"width": 2,
|
| 343 |
+
"mean_s_per_prompt": 0.7676023697853088,
|
| 344 |
+
"std_s": 0.022590433953289194,
|
| 345 |
+
"mean_ms_per_token": 10.987960138362206,
|
| 346 |
"mean_output_tokens": 69.86,
|
| 347 |
+
"total_s": 76.76023697853088,
|
| 348 |
"n_prompts": 100
|
| 349 |
},
|
| 350 |
{
|
| 351 |
"mode": "natural",
|
| 352 |
"width": 4,
|
| 353 |
+
"mean_s_per_prompt": 0.8113761901855469,
|
| 354 |
+
"std_s": 0.03146275351297224,
|
| 355 |
+
"mean_ms_per_token": 11.6640359597942,
|
| 356 |
"mean_output_tokens": 69.59,
|
| 357 |
+
"total_s": 81.13761901855469,
|
| 358 |
"n_prompts": 100
|
| 359 |
},
|
| 360 |
{
|
| 361 |
"mode": "natural",
|
| 362 |
"width": 8,
|
| 363 |
+
"mean_s_per_prompt": 0.8519934725761413,
|
| 364 |
+
"std_s": 0.040385070552722405,
|
| 365 |
+
"mean_ms_per_token": 12.310079472418668,
|
| 366 |
"mean_output_tokens": 69.34,
|
| 367 |
+
"total_s": 85.19934725761414,
|
| 368 |
"n_prompts": 100
|
| 369 |
},
|
| 370 |
{
|
| 371 |
"mode": "fixed",
|
| 372 |
"width": 1,
|
| 373 |
+
"mean_s_per_prompt": 0.7492986965179443,
|
| 374 |
+
"std_s": 0.015134158474454455,
|
| 375 |
+
"mean_ms_per_token": 10.704267093113488,
|
| 376 |
"mean_output_tokens": 70.0,
|
| 377 |
+
"total_s": 74.92986965179443,
|
| 378 |
"n_prompts": 100
|
| 379 |
},
|
| 380 |
{
|
| 381 |
"mode": "fixed",
|
| 382 |
"width": 2,
|
| 383 |
+
"mean_s_per_prompt": 0.7675032258033753,
|
| 384 |
+
"std_s": 0.015123762200423012,
|
| 385 |
+
"mean_ms_per_token": 10.964331797191074,
|
| 386 |
"mean_output_tokens": 70.0,
|
| 387 |
+
"total_s": 76.75032258033752,
|
| 388 |
"n_prompts": 100
|
| 389 |
},
|
| 390 |
{
|
| 391 |
"mode": "fixed",
|
| 392 |
"width": 4,
|
| 393 |
+
"mean_s_per_prompt": 0.8134244823455811,
|
| 394 |
+
"std_s": 0.012396015269615033,
|
| 395 |
+
"mean_ms_per_token": 11.620349747794014,
|
| 396 |
"mean_output_tokens": 70.0,
|
| 397 |
+
"total_s": 81.3424482345581,
|
| 398 |
"n_prompts": 100
|
| 399 |
},
|
| 400 |
{
|
| 401 |
"mode": "fixed",
|
| 402 |
"width": 8,
|
| 403 |
+
"mean_s_per_prompt": 0.8602101182937623,
|
| 404 |
+
"std_s": 0.015484862038058947,
|
| 405 |
+
"mean_ms_per_token": 12.288715975625175,
|
| 406 |
"mean_output_tokens": 70.0,
|
| 407 |
+
"total_s": 86.02101182937622,
|
| 408 |
"n_prompts": 100
|
| 409 |
}
|
| 410 |
],
|