| { |
| "token_position_grid": { |
| "n": 30, |
| "sbert_cos_mean": 0.9094, |
| "sbert_cos_min": 0.8466, |
| "token_f1_mean": 0.7263, |
| "token_f1_min": 0.5854, |
| "exact_match_rate": 0.0, |
| "examples": [ |
| { |
| "label": "marsh@p8/f2", |
| "orig": "Bridge, cliff, curtain, drawer, engine, fountain, helmet, marsh, kitchen, meadow, pillow, pocket, shovel, statue, trumpet, valley were read aloud twice.", |
| "decoded": "Bridge, crane, slope, drawer, machine, fountain, helmet, swamp, kitchen, lawn, mat, pocket, pillow, statue, trumpet, valley were read loudly two times.", |
| "token_f1": 0.585, |
| "sbert_cos": 0.847 |
| }, |
| { |
| "label": "mushroom@p9/f2", |
| "orig": "Bridge, cliff, curtain, drawer, engine, fountain, helmet, jacket, mushroom, meadow, pillow, pocket, shovel, statue, trumpet, valley were read aloud twice.", |
| "decoded": "Bridge, cliff, curtain, drawer, engine, fountain, helmet, jacket, mosquito, grass, pillow, pocket, pillow, statue, trumpet, valley were read loudly two times.", |
| "token_f1": 0.732, |
| "sbert_cos": 0.871 |
| }, |
| { |
| "label": "think@p14/f0", |
| "orig": "Table, river, candle, mirror, garden, pencil, button, ladder, basket, window, carpet, lantern, ribbon, think, kettle, saddle were on the list.", |
| "decoded": "Table, river, lamp, mirror, garden, pencil, button, scale, basket, window, carpet, lantern, ribbon, thought, kettle, saddle, was on the list.", |
| "token_f1": 0.8, |
| "sbert_cos": 0.972 |
| }, |
| { |
| "label": "marsh@p8/f2", |
| "orig": "Bridge, cliff, curtain, drawer, engine, fountain, helmet, marsh, kitchen, meadow, pillow, pocket, shovel, statue, trumpet, valley were read aloud twice.", |
| "decoded": "Bridge, crane, slope, drawer, machine, fountain, helmet, swamp, kitchen, lawn, mat, pocket, pillow, statue, trumpet, valley were read loudly two times.", |
| "token_f1": 0.585, |
| "sbert_cos": 0.847 |
| }, |
| { |
| "label": "jungle@p3/f1", |
| "orig": "Anchor, barrel, jungle, feather, island, lemon, magnet, needle, orchard, pebble, quilt, raft, sparrow, tunnel, violin, wagon appeared in that order.", |
| "decoded": "Anchor, barrel, vine, jungle, leaf, jar, magnet, needle, jewelry, plank, woodpecker, pitch, funnel, tunnel, violin, wagon appeared in that order.", |
| "token_f1": 0.6, |
| "sbert_cos": 0.86 |
| } |
| ] |
| }, |
| "spec_edit_corpus_base": { |
| "n": 30, |
| "sbert_cos_mean": 0.9817, |
| "sbert_cos_min": 0.87, |
| "token_f1_mean": 0.9445, |
| "token_f1_min": 0.7619, |
| "exact_match_rate": 0.4667, |
| "examples": [ |
| { |
| "label": "word-swap", |
| "orig": "Emily's eyes scanned the yellowed pages of a tattered book, her fingers tracing the faint lines of text as she searched for a specific passage.", |
| "decoded": "Emily's eyes sifted the yellowish pages of a rotted book, her fingers traced the faint lines of text as she searched for a particular passage.", |
| "token_f1": 0.8, |
| "sbert_cos": 0.94 |
| }, |
| { |
| "label": "word-swap", |
| "orig": "In the end, it was a combination of old-fashioned detective work and innovative thinking that led to the eventual breakthrough in the case.", |
| "decoded": "In the end, it was a combination of old-fashioned detective work and innovative thinking that led to the eventual breakthrough in the case.", |
| "token_f1": 1.0, |
| "sbert_cos": 1.0 |
| }, |
| { |
| "label": "word-swap", |
| "orig": "Irving's account portrayed the Dutch as warm, hospitable, and Christmas-loving, emphasizing their role in shaping the holiday traditions of New Amsterdam (prese", |
| "decoded": "Irving's account portrayed the Dutch as warm, hospitable, and Christmas-loving, emphasizing their role in shaping the holiday traditions of New Amsterdam (now N", |
| "token_f1": 0.943, |
| "sbert_cos": 0.999 |
| }, |
| { |
| "label": "delete-clause", |
| "orig": "The Fund, which holds a royalty interest in certain of these restaurants, benefits from the parent company's growth and profitability.", |
| "decoded": "The Fund, which has a royalty interest in some of these restaurants, benefits from the growth and profitability of the mother company.", |
| "token_f1": 0.762, |
| "sbert_cos": 0.939 |
| }, |
| { |
| "label": "register", |
| "orig": "Our volunteers are the backbone of our organisation, and we're grateful for their dedication and commitment.", |
| "decoded": "Our volunteers are the backbone of our organization, and we are thankful for their dedication and commitment.", |
| "token_f1": 0.788, |
| "sbert_cos": 0.964 |
| } |
| ] |
| }, |
| "spec_edit_corpus_gold": { |
| "n": 30, |
| "sbert_cos_mean": 0.9821, |
| "sbert_cos_min": 0.8805, |
| "token_f1_mean": 0.9562, |
| "token_f1_min": 0.8, |
| "exact_match_rate": 0.4333, |
| "examples": [ |
| { |
| "label": "word-swap", |
| "orig": "Emily's eyes scanned the yellowed pages of a tattered letter, her fingers tracing the faint lines of text as she searched for a specific passage.", |
| "decoded": "Emily's eyes sifted the yellowed pages of a broken letter, her fingers traced the faint lines of text as she searched for a particular passage.", |
| "token_f1": 0.84, |
| "sbert_cos": 0.923 |
| }, |
| { |
| "label": "word-swap", |
| "orig": "In the end, it was a combination of new-fashioned detective work and innovative thinking that led to the eventual breakthrough in the case.", |
| "decoded": "In the end, it was a combination of new-style detective work and innovative thinking that led to the eventual breakthrough in the case.", |
| "token_f1": 0.958, |
| "sbert_cos": 0.99 |
| }, |
| { |
| "label": "word-swap", |
| "orig": "Irving's account portrayed the Dutch as warm, hospitable, and Christmas-loving, emphasizing their role in shaping the holiday traditions of New Amsterdam (prese", |
| "decoded": "Irving's account portrayed the Dutch as warm, hospitable, and Christmas-loving, emphasizing their role in shaping the holiday traditions of New Amsterdam (now N", |
| "token_f1": 0.943, |
| "sbert_cos": 0.994 |
| }, |
| { |
| "label": "negation", |
| "orig": "With some salary cap considerations in mind, the Saints will not be monitoring the early starter pieces and any bold draft moves to get added cushion.", |
| "decoded": "With some salary cap considerations in mind, The Saints won't be monitoring the early starters and any daring draft moves to get extra cushion.", |
| "token_f1": 0.8, |
| "sbert_cos": 0.959 |
| }, |
| { |
| "label": "word-swap", |
| "orig": "Emily's eyes scanned the yellowed pages of a tattered letter, her fingers tracing the faint lines of text as she searched for a specific passage.", |
| "decoded": "Emily's eyes sifted the yellowed pages of a broken letter, her fingers traced the faint lines of text as she searched for a particular passage.", |
| "token_f1": 0.84, |
| "sbert_cos": 0.923 |
| } |
| ] |
| }, |
| "paraphrase_bank_a": { |
| "n": 15, |
| "sbert_cos_mean": 0.968, |
| "sbert_cos_min": 0.8289, |
| "token_f1_mean": 0.9486, |
| "token_f1_min": 0.8571, |
| "exact_match_rate": 0.6, |
| "examples": [ |
| { |
| "label": "mined_orbit", |
| "orig": "As the world continues to evolve, policymakers will need to navigate the challenges and opportunities presented by future trade agreements.", |
| "decoded": "As the world continues to evolve, policymakers will need to navigate the challenges and opportunities presented by future trade agreements.", |
| "token_f1": 1.0, |
| "sbert_cos": 1.0 |
| }, |
| { |
| "label": "template_active_passive", |
| "orig": "The fuzzy ladder scoured the shallow hand.", |
| "decoded": "The fuzzy ladder swept the shallow hand.", |
| "token_f1": 0.857, |
| "sbert_cos": 0.891 |
| }, |
| { |
| "label": "template_cleft_passive", |
| "orig": "It was the sticky waiter that stacked the full number.", |
| "decoded": "It was the sticky waiter who stacked the full number.", |
| "token_f1": 0.9, |
| "sbert_cos": 0.995 |
| }, |
| { |
| "label": "template_active_passive", |
| "orig": "The fuzzy ladder scoured the shallow hand.", |
| "decoded": "The fuzzy ladder swept the shallow hand.", |
| "token_f1": 0.857, |
| "sbert_cos": 0.891 |
| }, |
| { |
| "label": "template_active_passive", |
| "orig": "The hairy jungle hauled the velvet violin.", |
| "decoded": "The hairy jungle dragged the velvet violin.", |
| "token_f1": 0.857, |
| "sbert_cos": 0.943 |
| } |
| ] |
| }, |
| "paraphrase_bank_b": { |
| "n": 15, |
| "sbert_cos_mean": 0.9631, |
| "sbert_cos_min": 0.8221, |
| "token_f1_mean": 0.9475, |
| "token_f1_min": 0.8387, |
| "exact_match_rate": 0.5333, |
| "examples": [ |
| { |
| "label": "mined_orbit", |
| "orig": "As the world progresses, policymakers will have to address the challenges and opportunities that future trade agreements present.", |
| "decoded": "As the world progresses, policymakers will have to address the challenges and opportunities that future trade agreements present.", |
| "token_f1": 1.0, |
| "sbert_cos": 1.0 |
| }, |
| { |
| "label": "template_active_passive", |
| "orig": "The shallow hand was scoured by the fuzzy ladder.", |
| "decoded": "The shallow hand was scrubbed by the fuzzy ladder.", |
| "token_f1": 0.889, |
| "sbert_cos": 0.886 |
| }, |
| { |
| "label": "template_cleft_passive", |
| "orig": "The full number was stacked by the sticky waiter.", |
| "decoded": "The full number was stacked by the sticky waiter.", |
| "token_f1": 1.0, |
| "sbert_cos": 1.0 |
| }, |
| { |
| "label": "mined_orbit", |
| "orig": "For further information about the conference, including registration and the schedule, kindly visit the [NEA website].", |
| "decoded": "For further information about the conference, including registration and the schedule, please visit [NEA's website].", |
| "token_f1": 0.839, |
| "sbert_cos": 0.982 |
| }, |
| { |
| "label": "template_adverb_move", |
| "orig": "Steadily, the slippery cabin packed the room.", |
| "decoded": "Steadily, the slippery cabin filled the room.", |
| "token_f1": 0.857, |
| "sbert_cos": 0.926 |
| } |
| ] |
| }, |
| "construction_specs": { |
| "n": 30, |
| "sbert_cos_mean": 0.9052, |
| "sbert_cos_min": 0.6537, |
| "token_f1_mean": 0.8748, |
| "token_f1_min": 0.6667, |
| "exact_match_rate": 0.2667, |
| "examples": [ |
| { |
| "label": "L04_002", |
| "orig": "The quarry laughed noisily.", |
| "decoded": "The quarry laughed loudly.", |
| "token_f1": 0.75, |
| "sbert_cos": 0.806 |
| }, |
| { |
| "label": "L04_039", |
| "orig": "The mattress smiled hard.", |
| "decoded": "The mattress smiled hard.", |
| "token_f1": 1.0, |
| "sbert_cos": 1.0 |
| }, |
| { |
| "label": "L04_042", |
| "orig": "The year relaxed hard.", |
| "decoded": "The year relaxed hard.", |
| "token_f1": 1.0, |
| "sbert_cos": 1.0 |
| }, |
| { |
| "label": "L06_008", |
| "orig": "The clever dell plastered the donkey.", |
| "decoded": "The smart dell gelled the donkey.", |
| "token_f1": 0.667, |
| "sbert_cos": 0.839 |
| }, |
| { |
| "label": "L12_043", |
| "orig": "The plump jungle followed the rotten tine abroad near the dank pavement.", |
| "decoded": "The thick jungle followed the rotten tin to abroad near the wet pavement.", |
| "token_f1": 0.72, |
| "sbert_cos": 0.882 |
| } |
| ], |
| "content_word_recall_mean": 0.8208 |
| } |
| } |
