evaluation/benchmarks_data/language_probes.json

{
  "devanagari_stress_words": [
    "दृष्टिकोण",
    "संयुक्ताक्षर",
    "अंतर्राष्ट्रीय",
    "हस्ताक्षर",
    "ऋषियों",
    "श्रद्धांजलि",
    "प्रौद्योगिकी",
    "आत्मनिर्भरता",
    "न्यायाधीश",
    "वैज्ञानिक",
    "सर्वोच्च",
    "कार्यप्रणाली",
    "विश्वविद्यालय",
    "संविधान",
    "प्रक्रिया",
    "अध्यात्म",
    "उद्देश्य",
    "सहानुभूति",
    "प्रोत्साहन",
    "समीक्षा",
    "अभिभाषण",
    "प्रत्याशी",
    "अनुसंधान",
    "अविभाज्य",
    "पर्यावरण"
  ],
  "hindi_grammar_agreement": [
    {
      "prompt": "लड़का मैदान में तेजी से दौड़ ",
      "expected_top": "रहा",
      "distractor": "रही",
      "type": "gender_masculine"
    },
    {
      "prompt": "लड़की मैदान में तेजी से दौड़ ",
      "expected_top": "रही",
      "distractor": "रहा",
      "type": "gender_feminine"
    },
    {
      "prompt": "सभी बच्चे खुशी से नाच ",
      "expected_top": "रहे",
      "distractor": "रहा",
      "type": "plural_verb"
    },
    {
      "prompt": "गाय हरी घास खा ",
      "expected_top": "रही",
      "distractor": "रहा",
      "type": "gender_feminine"
    },
    {
      "prompt": "घोड़ा घास खा ",
      "expected_top": "रहा",
      "distractor": "रही",
      "type": "gender_masculine"
    }
  ],
  "english_syntax_agreement": [
    {
      "prompt": "The dog in the park always ",
      "expected_top": "barks",
      "distractor": "bark"
    },
    {
      "prompt": "The dogs in the park always ",
      "expected_top": "bark",
      "distractor": "barks"
    },
    {
      "prompt": "She has already finished her ",
      "expected_top": "work",
      "distractor": "working"
    },
    {
      "prompt": "They have been living here for ten ",
      "expected_top": "years",
      "distractor": "year"
    }
  ]
}