| { |
| "summary": { |
| "total_decisions": 244, |
| "jev_failures_count": 22, |
| "laya_failures_count": 44, |
| "laya_better_count": 13, |
| "jev_better_count": 35 |
| }, |
| "laya_better_than_jev": [ |
| { |
| "case_idx": 2, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.84, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 6, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Dec 22' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.83, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 10, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.78, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 11, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.84, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 14, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.8, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 24, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.85, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 44, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Tokyo to Kyoto on Feb 28' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.76, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 45, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.82, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 54, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.76, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 56, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Feb 28' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.73, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 57, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.8, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 64, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.8, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 68, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.78, |
| "jev_correct": false |
| } |
| ], |
| "jev_better_than_laya": [ |
| { |
| "case_idx": 1, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.34, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 2, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Nov 5'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.94, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 3, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.42, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 6, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Dec 22'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.92, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 7, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.95, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 8, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.46, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 9, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.33, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 10, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.202, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.91, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 11, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Oct 12'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.9, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 13, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.39, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 14, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.93, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 17, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.94, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 24, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Berlin to Madrid on Oct 12'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.87, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 25, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.94, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 26, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.93, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 27, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Dec 22'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.25, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 31, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.94, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 36, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.94, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 38, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.32, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 39, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.95, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 42, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.45, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 44, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Tokyo to Kyoto on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.91, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 45, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Berlin to Madrid on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.88, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 49, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.94, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 50, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.48, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 51, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.25, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 54, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Oct 12'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.89, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 55, |
| "url": "https://en.wikipedia.org/wiki/Main_Page", |
| "title": "Wikipedia, the free encyclopedia", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.37, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 56, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.202, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.89, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 57, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.93, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 58, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.95, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 60, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.95, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 64, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Nov 5'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.93, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 67, |
| "url": "https://www.google.com/travel/flights/search?q=zurich+to+london", |
| "title": "Flights from Zurich to London from $129", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.95, |
| "jev_correct": true |
| }, |
| { |
| "case_idx": 68, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "click_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Oct 12'.", |
| "gold": "1", |
| "laya_pred": "5", |
| "laya_conf": 0.201, |
| "laya_correct": false, |
| "jev_pred": "1", |
| "jev_conf": 0.91, |
| "jev_correct": true |
| } |
| ], |
| "jev_failures": [ |
| { |
| "case_idx": 2, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.84, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 6, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Dec 22' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.83, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 10, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.78, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 11, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.84, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 12, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Berlin to Madrid on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.24, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 14, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.8, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 16, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.32, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 23, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Berlin to Madrid on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.31, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 24, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.85, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 32, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Berlin to Madrid on Dec 22'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.31, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 34, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.24, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 43, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Sydney to Melbourne on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.32, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 44, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Tokyo to Kyoto on Feb 28' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.76, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 45, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.82, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 47, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.335, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.35, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 54, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.76, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 56, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Feb 28' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.73, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 57, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.8, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 61, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Feb 28'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.3, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 62, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Form Filled", |
| "qid": "type_text_target", |
| "qtype": "choice", |
| "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Jan 15'.", |
| "gold": "1", |
| "laya_pred": "3", |
| "laya_conf": 0.334, |
| "laya_correct": false, |
| "jev_pred": "3", |
| "jev_conf": 0.25, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 64, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.873, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.8, |
| "jev_correct": false |
| }, |
| { |
| "case_idx": 68, |
| "url": "https://www.google.com/travel/flights?hl=en", |
| "title": "Google Flights - Search Flights", |
| "qid": "operation", |
| "qtype": "choice", |
| "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.", |
| "gold": "TYPE_TEXT", |
| "laya_pred": "TYPE_TEXT", |
| "laya_conf": 0.874, |
| "laya_correct": true, |
| "jev_pred": "CLICK", |
| "jev_conf": 0.78, |
| "jev_correct": false |
| } |
| ] |
| } |
