| { |
| "reference": { |
| "reference_perplexity": 24.01962701026171, |
| "samples": 1000 |
| }, |
| "greedy": { |
| "name": "greedy", |
| "kind": "greedy", |
| "samples": 1000, |
| "generation_perplexity": 2.1845094968449197, |
| "token_match_rate": 0.017536830158415048, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.987, |
| "total_seconds": 30.55351047298609, |
| "seconds_per_sample": 0.03055351047298609, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.4011543259943025, |
| "rouge_l": 0.10133870138074608, |
| "repeat_rate": 0.991, |
| "distinct_2": 0.37804763470662844 |
| }, |
| "beam-1": { |
| "name": "beam-1", |
| "kind": "beam", |
| "width": 1, |
| "samples": 1000, |
| "generation_perplexity": 2.184142097873099, |
| "token_match_rate": 0.01747016349174838, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.987, |
| "total_seconds": 748.7766567100189, |
| "seconds_per_sample": 0.7487766567100189, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.4017249195346004, |
| "rouge_l": 0.10134016059568843, |
| "repeat_rate": 0.991, |
| "distinct_2": 0.37787066171981404 |
| }, |
| "beam-2": { |
| "name": "beam-2", |
| "kind": "beam", |
| "width": 2, |
| "samples": 1000, |
| "generation_perplexity": 1.9418890830063311, |
| "token_match_rate": 0.012793069344256506, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.82, |
| "total_seconds": 743.5340949810707, |
| "seconds_per_sample": 0.7435340949810707, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.26636636550312737, |
| "rouge_l": 0.08545188908177155, |
| "repeat_rate": 0.997, |
| "distinct_2": 0.35923942153186933 |
| }, |
| "beam-4": { |
| "name": "beam-4", |
| "kind": "beam", |
| "width": 4, |
| "samples": 1000, |
| "generation_perplexity": 1.7872872389121282, |
| "token_match_rate": 0.013918423162327762, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.519, |
| "total_seconds": 745.1537463600471, |
| "seconds_per_sample": 0.745153746360047, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.21235683373778194, |
| "rouge_l": 0.07549435572208057, |
| "repeat_rate": 0.997, |
| "distinct_2": 0.3154023728995544 |
| }, |
| "top-k-5": { |
| "name": "top-k-5", |
| "kind": "top_k", |
| "k": 5, |
| "temperature": 1.0, |
| "samples": 1000, |
| "generation_perplexity": 5.840530230478968, |
| "token_match_rate": 0.01660183385003512, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.671, |
| "total_seconds": 29.016556230009883, |
| "seconds_per_sample": 0.02901655623000988, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.4880453760353093, |
| "rouge_l": 0.1236210191578215, |
| "repeat_rate": 0.393, |
| "distinct_2": 0.8830857095559326 |
| }, |
| "top-k-20": { |
| "name": "top-k-20", |
| "kind": "top_k", |
| "k": 20, |
| "temperature": 1.0, |
| "samples": 1000, |
| "generation_perplexity": 10.019873678669159, |
| "token_match_rate": 0.015090541951915377, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.223, |
| "total_seconds": 28.801222771980974, |
| "seconds_per_sample": 0.028801222771980974, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.4856051512173579, |
| "rouge_l": 0.11775486793844488, |
| "repeat_rate": 0.115, |
| "distinct_2": 0.9596821600145947 |
| }, |
| "top-k-50": { |
| "name": "top-k-50", |
| "kind": "top_k", |
| "k": 50, |
| "temperature": 1.0, |
| "samples": 1000, |
| "generation_perplexity": 13.80108373815267, |
| "token_match_rate": 0.013518684208311345, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 63.13, |
| "total_seconds": 27.95822200999828, |
| "seconds_per_sample": 0.02795822200999828, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.46471149496907277, |
| "rouge_l": 0.11558306869222004, |
| "repeat_rate": 0.053, |
| "distinct_2": 0.9742885973763875 |
| }, |
| "top-p-0.5": { |
| "name": "top-p-0.5", |
| "kind": "top_p", |
| "p": 0.5, |
| "temperature": 1.0, |
| "samples": 1000, |
| "generation_perplexity": 5.867983666604579, |
| "token_match_rate": 0.016987026890306695, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 64.0, |
| "total_seconds": 31.004208463971736, |
| "seconds_per_sample": 0.031004208463971734, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.6045125923332172, |
| "rouge_l": 0.11670411529976443, |
| "repeat_rate": 0.384, |
| "distinct_2": 0.8828828828828829 |
| }, |
| "top-p-0.9": { |
| "name": "top-p-0.9", |
| "kind": "top_p", |
| "p": 0.9, |
| "temperature": 1.0, |
| "samples": 1000, |
| "generation_perplexity": 24.27615581913303, |
| "token_match_rate": 0.012577092470312138, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 62.755, |
| "total_seconds": 30.671832652995363, |
| "seconds_per_sample": 0.030671832652995362, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.21955334969040827, |
| "rouge_l": 0.10558525109265285, |
| "repeat_rate": 0.021, |
| "distinct_2": 0.9854764735858672 |
| }, |
| "top-p-0.95": { |
| "name": "top-p-0.95", |
| "kind": "top_p", |
| "p": 0.95, |
| "temperature": 1.0, |
| "samples": 1000, |
| "generation_perplexity": 34.31498216399583, |
| "token_match_rate": 0.012355755202568324, |
| "first_sentence_exact_match": 0.0, |
| "mean_generated_tokens": 62.609, |
| "total_seconds": 29.93822276096762, |
| "seconds_per_sample": 0.02993822276096762, |
| "reference_perplexity": 24.01962701026171, |
| "bleu": 0.23364194724538673, |
| "rouge_l": 0.1033430667860992, |
| "repeat_rate": 0.022, |
| "distinct_2": 0.987771768176668 |
| } |
| } |
