{
  "summary": {
    "total_decisions": 244,
    "jev_failures_count": 22,
    "laya_failures_count": 44,
    "laya_better_count": 13,
    "jev_better_count": 35
  },
  "laya_better_than_jev": [
    {
      "case_idx": 2,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.84,
      "jev_correct": false
    },
    {
      "case_idx": 6,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Dec 22' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.83,
      "jev_correct": false
    },
    {
      "case_idx": 10,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.78,
      "jev_correct": false
    },
    {
      "case_idx": 11,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.84,
      "jev_correct": false
    },
    {
      "case_idx": 14,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.8,
      "jev_correct": false
    },
    {
      "case_idx": 24,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.85,
      "jev_correct": false
    },
    {
      "case_idx": 44,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Tokyo to Kyoto on Feb 28' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.76,
      "jev_correct": false
    },
    {
      "case_idx": 45,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.82,
      "jev_correct": false
    },
    {
      "case_idx": 54,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.76,
      "jev_correct": false
    },
    {
      "case_idx": 56,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Feb 28' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.73,
      "jev_correct": false
    },
    {
      "case_idx": 57,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.8,
      "jev_correct": false
    },
    {
      "case_idx": 64,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.8,
      "jev_correct": false
    },
    {
      "case_idx": 68,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.78,
      "jev_correct": false
    }
  ],
  "jev_better_than_laya": [
    {
      "case_idx": 1,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.34,
      "jev_correct": true
    },
    {
      "case_idx": 2,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Nov 5'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.94,
      "jev_correct": true
    },
    {
      "case_idx": 3,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.42,
      "jev_correct": true
    },
    {
      "case_idx": 6,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Dec 22'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.92,
      "jev_correct": true
    },
    {
      "case_idx": 7,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.95,
      "jev_correct": true
    },
    {
      "case_idx": 8,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.46,
      "jev_correct": true
    },
    {
      "case_idx": 9,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.33,
      "jev_correct": true
    },
    {
      "case_idx": 10,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Jan 15'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.202,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.91,
      "jev_correct": true
    },
    {
      "case_idx": 11,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Oct 12'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.9,
      "jev_correct": true
    },
    {
      "case_idx": 13,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.39,
      "jev_correct": true
    },
    {
      "case_idx": 14,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.93,
      "jev_correct": true
    },
    {
      "case_idx": 17,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.94,
      "jev_correct": true
    },
    {
      "case_idx": 24,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Berlin to Madrid on Oct 12'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.87,
      "jev_correct": true
    },
    {
      "case_idx": 25,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.94,
      "jev_correct": true
    },
    {
      "case_idx": 26,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.93,
      "jev_correct": true
    },
    {
      "case_idx": 27,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Dec 22'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.25,
      "jev_correct": true
    },
    {
      "case_idx": 31,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.94,
      "jev_correct": true
    },
    {
      "case_idx": 36,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.94,
      "jev_correct": true
    },
    {
      "case_idx": 38,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.32,
      "jev_correct": true
    },
    {
      "case_idx": 39,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.95,
      "jev_correct": true
    },
    {
      "case_idx": 42,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.45,
      "jev_correct": true
    },
    {
      "case_idx": 44,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Tokyo to Kyoto on Feb 28'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.91,
      "jev_correct": true
    },
    {
      "case_idx": 45,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Berlin to Madrid on Jan 15'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.88,
      "jev_correct": true
    },
    {
      "case_idx": 49,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.94,
      "jev_correct": true
    },
    {
      "case_idx": 50,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.48,
      "jev_correct": true
    },
    {
      "case_idx": 51,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Jan 15'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.25,
      "jev_correct": true
    },
    {
      "case_idx": 54,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Oct 12'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.89,
      "jev_correct": true
    },
    {
      "case_idx": 55,
      "url": "https://en.wikipedia.org/wiki/Main_Page",
      "title": "Wikipedia, the free encyclopedia",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Lookup G\u00f6del's incompleteness theorems on Wikipedia'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.37,
      "jev_correct": true
    },
    {
      "case_idx": 56,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Feb 28'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.202,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.89,
      "jev_correct": true
    },
    {
      "case_idx": 57,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.93,
      "jev_correct": true
    },
    {
      "case_idx": 58,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.95,
      "jev_correct": true
    },
    {
      "case_idx": 60,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.95,
      "jev_correct": true
    },
    {
      "case_idx": 64,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from Sydney to Melbourne on Nov 5'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.93,
      "jev_correct": true
    },
    {
      "case_idx": 67,
      "url": "https://www.google.com/travel/flights/search?q=zurich+to+london",
      "title": "Flights from Zurich to London from $129",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find flight ticket price from Zurich to London'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.95,
      "jev_correct": true
    },
    {
      "case_idx": 68,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "click_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is CLICK for goal 'Find one-way flights from New York to Boston on Oct 12'.",
      "gold": "1",
      "laya_pred": "5",
      "laya_conf": 0.201,
      "laya_correct": false,
      "jev_pred": "1",
      "jev_conf": 0.91,
      "jev_correct": true
    }
  ],
  "jev_failures": [
    {
      "case_idx": 2,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.84,
      "jev_correct": false
    },
    {
      "case_idx": 6,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Dec 22' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.83,
      "jev_correct": false
    },
    {
      "case_idx": 10,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.78,
      "jev_correct": false
    },
    {
      "case_idx": 11,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.84,
      "jev_correct": false
    },
    {
      "case_idx": 12,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Berlin to Madrid on Feb 28'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.24,
      "jev_correct": false
    },
    {
      "case_idx": 14,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.8,
      "jev_correct": false
    },
    {
      "case_idx": 16,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.32,
      "jev_correct": false
    },
    {
      "case_idx": 23,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Berlin to Madrid on Feb 28'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.31,
      "jev_correct": false
    },
    {
      "case_idx": 24,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.85,
      "jev_correct": false
    },
    {
      "case_idx": 32,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Berlin to Madrid on Dec 22'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.31,
      "jev_correct": false
    },
    {
      "case_idx": 34,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Feb 28'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.24,
      "jev_correct": false
    },
    {
      "case_idx": 43,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Sydney to Melbourne on Feb 28'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.32,
      "jev_correct": false
    },
    {
      "case_idx": 44,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Tokyo to Kyoto on Feb 28' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.76,
      "jev_correct": false
    },
    {
      "case_idx": 45,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Berlin to Madrid on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.82,
      "jev_correct": false
    },
    {
      "case_idx": 47,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Sydney to Melbourne on Jan 15'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.335,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.35,
      "jev_correct": false
    },
    {
      "case_idx": 54,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.76,
      "jev_correct": false
    },
    {
      "case_idx": 56,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Feb 28' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.73,
      "jev_correct": false
    },
    {
      "case_idx": 57,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Jan 15' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.8,
      "jev_correct": false
    },
    {
      "case_idx": 61,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Feb 28'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.3,
      "jev_correct": false
    },
    {
      "case_idx": 62,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Form Filled",
      "qid": "type_text_target",
      "qtype": "choice",
      "instructions": "Choose element index if operation is TYPE_TEXT for goal 'Find one-way flights from Tokyo to Kyoto on Jan 15'.",
      "gold": "1",
      "laya_pred": "3",
      "laya_conf": 0.334,
      "laya_correct": false,
      "jev_pred": "3",
      "jev_conf": 0.25,
      "jev_correct": false
    },
    {
      "case_idx": 64,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from Sydney to Melbourne on Nov 5' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.873,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.8,
      "jev_correct": false
    },
    {
      "case_idx": 68,
      "url": "https://www.google.com/travel/flights?hl=en",
      "title": "Google Flights - Search Flights",
      "qid": "operation",
      "qtype": "choice",
      "instructions": "Advance goal 'Find one-way flights from New York to Boston on Oct 12' using one immediate operation.",
      "gold": "TYPE_TEXT",
      "laya_pred": "TYPE_TEXT",
      "laya_conf": 0.874,
      "laya_correct": true,
      "jev_pred": "CLICK",
      "jev_conf": 0.78,
      "jev_correct": false
    }
  ]
}