Next-Trace / evaluation.json
Lamapi's picture
Publish Space with valid short description
c07bca1 verified
Raw History Blame Contribute Delete
62.4 kB
{ "project": "Next Trace", "profile": "full", "seed": 73, "device": { "python": "3.13.16", "platform": "Linux-6.6.159+-x86_64-with-glibc2.39", "cpu_count": 12, "numpy": "2.4.6", "torch": "2.14.1+cu130", "cuda": true, "gpu": "NVIDIA L4", "colab": true }, "data": { "sources": { "squad_train": { "url": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json", "file": "train-v2.0.json", "bytes": 42123633, "sha256": "68dcfbb971bd3e96d5b46c7177b16c1a4e7d4bdef19fb204502738552dede002", "verified_against_pinned_sha256": false }, "squad_dev": { "url": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json", "file": "dev-v2.0.json", "bytes": 4370528, "sha256": "80a5225e94905956a6446d296ca1093975c4d3b3260f1d6c8f68bc2ab77182d8", "verified_against_pinned_sha256": false }, "xquad_tr": { "url": "https://raw.githubusercontent.com/google-deepmind/xquad/master/xquad.tr.json", "file": "xquad.tr.json", "bytes": 729506, "sha256": "92179a564774b7696100d144c1e10870d0a966b6fccbdd254a65b9d2ab1971cc", "verified_against_pinned_sha256": false }, "xquad_en": { "url": "https://raw.githubusercontent.com/google-deepmind/xquad/master/xquad.en.json", "file": "xquad.en.json", "bytes": 609383, "sha256": "e4c57d1c9143aaa1c5d265ba5987a65f4e69528d2a98f29d6e75019b10344f29", "verified_against_pinned_sha256": false } }, "license": "CC BY-SA 4.0 (SQuAD 2.0, XQuAD)", "versions": { "squad": "2.0", "xquad": "GitHub master at download time (sha256 recorded)" }, "splits": { "squad_train_A": 126503, "squad_train_B": 16851, "squad_train_V": 15937, "SQuAD2-dev (EN, adversarial unanswerables)": { "questions": 11873, "unanswerable": 5945 }, "XQuAD-tr (TR, zero-shot)": { "questions": 2327, "unanswerable": 1137 }, "XQuAD-en (EN)": { "questions": 2319, "unanswerable": 1129 }, "rule": "SQuAD train split by article title (A: sentence model, B: answerability head, V: thresholds); each split also gets ~35% extra unanswerables made by asking a question against another paragraph of the same article; XQuAD and SQuAD dev only for testing" } }, "thresholds": { "bm25": 2.7921873897475704, "answerable": 0.6, "second_sentence": 0.5 }, "validation": { "bm25": { "questions": 15937, "trace_accuracy": 0.5530526447888561, "evidence_top1_on_answerable": 0.7978169818504887, "answered_rate_on_answerable": 0.597537758598807, "correct_abstention_on_unanswerable": 0.5953090096798213, "cited_plausible_distractor_on_unanswerable": 0.29573095060809135, "answerable": 7879, "unanswerable": 8058 }, "learned": { "questions": 15937, "trace_accuracy": 0.5940892263286691, "evidence_top1_on_answerable": 0.8188856453864704, "answered_rate_on_answerable": 0.7587257266150527, "correct_abstention_on_unanswerable": 0.5409530900967983, "cited_plausible_distractor_on_unanswerable": 0.339662447257384, "answerable": 7879, "unanswerable": 8058 } }, "results": { "SQuAD2-dev (EN, adversarial unanswerables)": { "bm25": { "questions": 11873, "trace_accuracy": 0.49237766360650215, "evidence_top1_on_answerable": 0.8247300944669366, "answered_rate_on_answerable": 0.6454116059379217, "correct_abstention_on_unanswerable": 0.4146341463414634, "cited_plausible_distractor_on_unanswerable": 0.4462573591253154, "answerable": 5928, "unanswerable": 5945 }, "learned": { "questions": 11873, "trace_accuracy": 0.5104017518739998, "evidence_top1_on_answerable": 0.8424426450742241, "answered_rate_on_answerable": 0.800944669365722, "correct_abstention_on_unanswerable": 0.3184188393608074, "cited_plausible_distractor_on_unanswerable": 0.5164003364171573, "answerable": 5928, "unanswerable": 5945 } }, "XQuAD-tr (TR, zero-shot)": { "bm25": { "questions": 2327, "trace_accuracy": 0.7189514396218307, "evidence_top1_on_answerable": 0.7697478991596639, "answered_rate_on_answerable": 0.611764705882353, "correct_abstention_on_unanswerable": 0.9340369393139841, "cited_plausible_distractor_on_unanswerable": 0.0, "answerable": 1190, "unanswerable": 1137 }, "learned": { "questions": 2327, "trace_accuracy": 0.7877094972067039, "evidence_top1_on_answerable": 0.8025210084033614, "answered_rate_on_answerable": 0.7210084033613445, "correct_abstention_on_unanswerable": 0.9718557607739666, "cited_plausible_distractor_on_unanswerable": 0.0, "answerable": 1190, "unanswerable": 1137 } }, "XQuAD-en (EN)": { "bm25": { "questions": 2319, "trace_accuracy": 0.7317809400603709, "evidence_top1_on_answerable": 0.8033613445378152, "answered_rate_on_answerable": 0.6058823529411764, "correct_abstention_on_unanswerable": 0.9530558015943312, "cited_plausible_distractor_on_unanswerable": 0.0, "answerable": 1190, "unanswerable": 1129 }, "learned": { "questions": 2319, "trace_accuracy": 0.8201811125485123, "evidence_top1_on_answerable": 0.8243697478991596, "answered_rate_on_answerable": 0.7899159663865546, "correct_abstention_on_unanswerable": 0.9654561558901683, "cited_plausible_distractor_on_unanswerable": 0.0, "answerable": 1190, "unanswerable": 1129 } } }, "failure_examples": { "SQuAD2-dev (EN, adversarial unanswerables)": { "bm25": [ { "question": "In what country is Normandy located?", "answerable": true, "abstained": true, "cited": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France.", "gold_answer": "France" }, { "question": "When were the Normans in Normandy?", "answerable": true, "abstained": true, "cited": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France.", "gold_answer": "10th and 11th centuries" }, { "question": "From which countries did the Norse originate?", "answerable": true, "abstained": true, "cited": "They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West F", "gold_answer": "Denmark, Iceland and Norway" }, { "question": "Who was the Norse leader?", "answerable": true, "abstained": true, "cited": "They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West F", "gold_answer": "Rollo" }, { "question": "Who gave their name to Normandy in the 1000's and 1100's", "answerable": false, "abstained": false, "cited": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France.", "gold_answer": null }, { "question": "Who did King Charles III swear fealty to?", "answerable": false, "abstained": false, "cited": "They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West F", "gold_answer": null }, { "question": "Who ruled the duchy of Normandy", "answerable": true, "abstained": true, "cited": "The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in", "gold_answer": "Richard I" }, { "question": "What religion were the Normans", "answerable": true, "abstained": true, "cited": "They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language.", "gold_answer": "Catholic" }, { "question": "What type of major impact did the Norman dynasty have on modern Europe?", "answerable": false, "abstained": false, "cited": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East.", "gold_answer": null }, { "question": "Who was famed for their Christian spirit?", "answerable": false, "abstained": false, "cited": "The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated.", "gold_answer": null }, { "question": "Who assimilted the Roman language?", "answerable": false, "abstained": false, "cited": "They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language.", "gold_answer": null }, { "question": "What principality did William the conquerer found?", "answerable": false, "abstained": false, "cited": "Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to", "gold_answer": null }, { "question": "What is the original meaning of the word Norman?", "answerable": true, "abstained": true, "cited": "The English name \"Normans\" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann \"Northman\" or directly from ", "gold_answer": "Viking" }, { "question": "When was the Latin version of the word Norman first recorded?", "answerable": true, "abstained": true, "cited": "The English name \"Normans\" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann \"Northman\" or directly from ", "gold_answer": "9th century" }, { "question": "When was the Duchy of Normandy founded?", "answerable": true, "abstained": true, "cited": "The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine.", "gold_answer": "911" }, { "question": "What river originally bounded the Duchy", "answerable": true, "abstained": true, "cited": "The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine.", "gold_answer": "Seine" }, { "question": "when did Nors encampments ivolve into destructive incursions?", "answerable": false, "abstained": false, "cited": "In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal prop", "gold_answer": null }, { "question": "Who established a treaty with King Charles the third of France?", "answerable": false, "abstained": false, "cited": "The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situate", "gold_answer": null }, { "question": "What did the French promises to protect Rollo and his men from?", "answerable": false, "abstained": false, "cited": "The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions.", "gold_answer": null }, { "question": "Who upon arriving gave the original viking settlers a common identity?", "answerable": true, "abstained": true, "cited": "Earlier Viking settlers had begun arriving in the 880s, but were divided between colonies in the east (Roumois and Pays de Caux) around the low Seine valley and in the west in the Cotentin Peninsula, ", "gold_answer": "Rollo" }, { "question": "What was the Norman religion?", "answerable": true, "abstained": true, "cited": "The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, ", "gold_answer": "Catholicism" }, { "question": "What part of France were the Normans located?", "answerable": true, "abstained": true, "cited": "The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, ", "gold_answer": "north" }, { "question": "What was one of the Norman's major exports?", "answerable": true, "abstained": true, "cited": "Most Norman knights remained poor and land-hungry, and by 1066 Normandy had been exporting fighting horsemen for more than a generation.", "gold_answer": "fighting horsemen" }, { "question": "Who's arristocracy eventually served as avid Crusaders?", "answerable": false, "abstained": false, "cited": "Many Normans of Italy, France and England eventually served as avid Crusaders under the Italo-Norman prince Bohemund I and the Anglo-Norman king Richard the Lion-Heart.", "gold_answer": null }, { "question": "Who entered Italy soon after the Byzantine Empire?", "answerable": false, "abstained": false, "cited": "Soon after the Normans began to enter Italy, they entered the Byzantine Empire and then Armenia, fighting against the Pechenegs, the Bulgars, and especially the Seljuk Turks.", "gold_answer": null } ], "learned": [ { "question": "In what country is Normandy located?", "answerable": true, "abstained": true, "cited": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France.", "gold_answer": "France" }, { "question": "From which countries did the Norse originate?", "answerable": true, "abstained": true, "cited": "They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West F", "gold_answer": "Denmark, Iceland and Norway" }, { "question": "What is France a region of?", "answerable": false, "abstained": false, "cited": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France.", "gold_answer": null }, { "question": "Who did King Charles III swear fealty to?", "answerable": false, "abstained": false, "cited": "They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West F", "gold_answer": null }, { "question": "When did the Frankish identity emerge?", "answerable": false, "abstained": false, "cited": "The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.", "gold_answer": null }, { "question": "Who ruled the duchy of Normandy", "answerable": true, "abstained": true, "cited": "The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in", "gold_answer": "Richard I" }, { "question": "What religion were the Normans", "answerable": true, "abstained": true, "cited": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East.", "gold_answer": "Catholic" }, { "question": "What type of major impact did the Norman dynasty have on modern Europe?", "answerable": false, "abstained": false, "cited": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East.", "gold_answer": null }, { "question": "Who was famed for their Christian spirit?", "answerable": false, "abstained": false, "cited": "The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated.", "gold_answer": null }, { "question": "Who assimilted the Roman language?", "answerable": false, "abstained": false, "cited": "They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language.", "gold_answer": null }, { "question": "What principality did William the conquerer found?", "answerable": false, "abstained": false, "cited": "Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to", "gold_answer": null }, { "question": "What is the original meaning of the word Norman?", "answerable": true, "abstained": true, "cited": "The English name \"Normans\" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann \"Northman\" or directly from ", "gold_answer": "Viking" }, { "question": "What name comes from the English words Normans/Normanz?", "answerable": false, "abstained": false, "cited": "The English name \"Normans\" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann \"Northman\" or directly from ", "gold_answer": null }, { "question": "When was the French version of the word Norman first recorded?", "answerable": false, "abstained": false, "cited": "The English name \"Normans\" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann \"Northman\" or directly from ", "gold_answer": null }, { "question": "What river originally bounded the Duchy", "answerable": true, "abstained": true, "cited": "The area corresponded to the northern part of present-day Upper Normandy down to the river Seine, but the Duchy would eventually extend west beyond the Seine.", "gold_answer": "Seine" }, { "question": "when did Nors encampments ivolve into destructive incursions?", "answerable": false, "abstained": false, "cited": "In the course of the 10th century, the initially destructive incursions of Norse war bands into the rivers of France evolved into more permanent encampments that included local women and personal prop", "gold_answer": null }, { "question": "What treaty was established in the 9th century?", "answerable": false, "abstained": false, "cited": "The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situate", "gold_answer": null }, { "question": "Who established a treaty with King Charles the third of France?", "answerable": false, "abstained": false, "cited": "The Duchy of Normandy, which began in 911 as a fiefdom, was established by the treaty of Saint-Clair-sur-Epte between King Charles III of West Francia and the famed Viking ruler Rollo, and was situate", "gold_answer": null }, { "question": "What did the French promises to protect Rollo and his men from?", "answerable": false, "abstained": false, "cited": "The treaty offered Rollo and his men the French lands between the river Epte and the Atlantic coast in exchange for their protection against further Viking incursions.", "gold_answer": null }, { "question": "Who upon arriving gave the original viking settlers a common identity?", "answerable": true, "abstained": true, "cited": "Earlier Viking settlers had begun arriving in the 880s, but were divided between colonies in the east (Roumois and Pays de Caux) around the low Seine valley and in the west in the Cotentin Peninsula, ", "gold_answer": "Rollo" }, { "question": "What part of France were the Normans located?", "answerable": true, "abstained": true, "cited": "The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, ", "gold_answer": "north" }, { "question": "What was replace with the Norse religion?", "answerable": false, "abstained": false, "cited": "The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, ", "gold_answer": null }, { "question": "What did maternal Old Norse traditions merge with?", "answerable": false, "abstained": false, "cited": "The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, ", "gold_answer": null }, { "question": "What language replaced the Gallo-Romance language?", "answerable": false, "abstained": false, "cited": "The descendants of Rollo's Vikings and their Frankish wives would replace the Norse religion and Old Norse language with Catholicism (Christianity) and the Gallo-Romance language of the local people, ", "gold_answer": null }, { "question": "What was one of the Norman's major exports?", "answerable": true, "abstained": true, "cited": "Most Norman knights remained poor and land-hungry, and by 1066 Normandy had been exporting fighting horsemen for more than a generation.", "gold_answer": "fighting horsemen" } ] }, "XQuAD-tr (TR, zero-shot)": { "bm25": [ { "question": "Luke Kuechly kaç tane top çalma kaydetmiştir?", "answerable": true, "abstained": false, "cited": "Arkalarında, Panthers'lerin üç adet ikinci hat savunma oyuncusundan ikisi de Pro Bowl'da oynamak için seçildi: Thomas Davis ve Luke Kuechly.", "gold_answer": "118" }, { "question": "Josh Norman kaç tane top çalmıştır?", "answerable": true, "abstained": false, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "dört" }, { "question": "Bu sezon takımdaki en fazla sack etmeyi kim kaydetmiştir?", "answerable": true, "abstained": false, "cited": "Davis, dördünde mecburi oyuncuyu düşürme ve dört topu kapma olmak üzere 5½ sack etmeyi bir araya getirirken, Kuechly top çalmalarda (118) iki mecburi oyuncuyu düşürme ve kendi başına dört pasını kesme", "gold_answer": "Kawann Short" }, { "question": "Panthers savunması 2015 yılında kaç top çalma ile kayda geçmiştir?", "answerable": true, "abstained": true, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "24" }, { "question": "Panthers'lere sack etmede kim önderlik etmektedir?", "answerable": true, "abstained": true, "cited": "Panthers hattında ayrıca, sadece 9 başlangıçta 5 sack eden uç çizgi savunmacısı Kony Ealy ile birlikte 136 kez ile NFL'nin aktif kariyer sack lideri ve 5 kez profesyonel bir top atıcısı olan Jared All", "gold_answer": "Kawann Short" }, { "question": "Pro Bowl için kaç tane Panthers savunma oyuncusu seçilmiştir?", "answerable": true, "abstained": false, "cited": "Arkalarında, Panthers'lerin üç adet ikinci hat savunma oyuncusundan ikisi de Pro Bowl'da oynamak için seçildi: Thomas Davis ve Luke Kuechly.", "gold_answer": "dört" }, { "question": "Thomas Davis kaç mecburi oyuncuyu düşürme almıştır?", "answerable": true, "abstained": false, "cited": "Davis, dördünde mecburi oyuncuyu düşürme ve dört topu kapma olmak üzere 5½ sack etmeyi bir araya getirirken, Kuechly top çalmalarda (118) iki mecburi oyuncuyu düşürme ve kendi başına dört pasını kesme", "gold_answer": "dört" }, { "question": "Panthers'lerin savunması 2015 sezonu top kapmalarının kaç tanesini almıştır?", "answerable": true, "abstained": false, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "24" }, { "question": "Josh Norman 2015 yılında kale çizgisini geçişle sonlanan kaç topu çalma yapmıştır?", "answerable": true, "abstained": false, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "iki" }, { "question": "Super Bowl XLIX'i kim kazanmıştır?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "New England Patriots" }, { "question": "Super Bowl XLIX'i kim kazanmıştır?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "New England Patriots" }, { "question": "Broncos oyunu kazandıran pası kestiği zaman oyunun bitimine kaç saniye kalmıştı?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "17" }, { "question": "Broncos'un rövanş oyunları süresince kim hiç atmamıştır?", "answerable": true, "abstained": true, "cited": "Manning''in sezon boyunca topu kapmalarla ilgili sorunlarına rağmen, iki rövanş maçında hiç atmadı.", "gold_answer": "Manning" }, { "question": "2015 yılında klasmanlarını kazanmak için Broncos'u kim yenmiştir?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "Pittsburgh Steelers" }, { "question": "Peyton Manning, Super Bowl %0'de oynadığı zaman kaç yaşındaydı?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "39" }, { "question": "Bir Super Bowl'da oynayan en yaşlı kilit oyuncu olma rekorunu daha önce kim elinde bulunduruyordu?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "John Elway" }, { "question": "John Elway'in şu anki Broncos üyeliğindeki rolü nedir?", "answerable": true, "abstained": true, "cited": "Geçmiş rekor, Broncos'u 38 yaşında Super Bowl XXXIII'te galibiyete götüren ve şu anda Denver'ın Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü olan John Elway'in ellerindeydi.", "gold_answer": "Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü" }, { "question": "Super Bowl XXXIII'ün kazananı hangi takımdı?", "answerable": true, "abstained": true, "cited": "Geçmiş rekor, Broncos'u 38 yaşında Super Bowl XXXIII'te galibiyete götüren ve şu anda Denver'ın Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü olan John Elway'in ellerindeydi.", "gold_answer": "Broncos" }, { "question": "Broncos'un Genel Müdürü kimdir?", "answerable": true, "abstained": true, "cited": "Geçmiş rekor, Broncos'u 38 yaşında Super Bowl XXXIII'te galibiyete götüren ve şu anda Denver'ın Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü olan John Elway'in ellerindeydi.", "gold_answer": "John Elway" }, { "question": "Herhangi bir Super Bowl oyununda en yaşlı kilit oyuncu rekoruna kim sahiptir?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "Peyton Manning" }, { "question": "Manning Super Bowl'a ulaşmak için onların takımındayken kaç takımda oynamıştır?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "iki" }, { "question": "Bir Super Bowl'da oynayan en yaşlı kilit oyuncu kimdir?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "Peyton Manning" }, { "question": "Manning'den önce bir Super Bowl'da oynayan en yaşlı kilit oyuncu kimdir?", "answerable": true, "abstained": false, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "John Elway" }, { "question": "Manning Super Bowl 50'de oynadığı zaman kaç yaşındaydı?", "answerable": true, "abstained": true, "cited": "Ayrıca, 39 yaşında bir Super Bowl'da oynayan en eski oyun kurucu oldu.", "gold_answer": "39" }, { "question": "Lady Gaga kaç Grammy kazanmıştır?", "answerable": true, "abstained": true, "cited": "Akademi Ödülü kazananı Marlee Matlin Amerikan İşaret Dili(ASL) çevirisini yaparken altı kez Grammy kazanan ve Akademi Ödülü adayı Lady Gaga ulusal marşı söylemiştir. ", "gold_answer": "altı" } ], "learned": [ { "question": "Luke Kuechly kaç tane top çalma kaydetmiştir?", "answerable": true, "abstained": true, "cited": "Davis, dördünde mecburi oyuncuyu düşürme ve dört topu kapma olmak üzere 5½ sack etmeyi bir araya getirirken, Kuechly top çalmalarda (118) iki mecburi oyuncuyu düşürme ve kendi başına dört pasını kesme", "gold_answer": "118" }, { "question": "Josh Norman kaç tane top çalmıştır?", "answerable": true, "abstained": false, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "dört" }, { "question": "Bu sezon takımdaki en fazla sack etmeyi kim kaydetmiştir?", "answerable": true, "abstained": false, "cited": "Davis, dördünde mecburi oyuncuyu düşürme ve dört topu kapma olmak üzere 5½ sack etmeyi bir araya getirirken, Kuechly top çalmalarda (118) iki mecburi oyuncuyu düşürme ve kendi başına dört pasını kesme", "gold_answer": "Kawann Short" }, { "question": "Panthers savunması 2015 yılında kaç top çalma ile kayda geçmiştir?", "answerable": true, "abstained": true, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "24" }, { "question": "Panthers'lere sack etmede kim önderlik etmektedir?", "answerable": true, "abstained": true, "cited": "Panthers hattında ayrıca, sadece 9 başlangıçta 5 sack eden uç çizgi savunmacısı Kony Ealy ile birlikte 136 kez ile NFL'nin aktif kariyer sack lideri ve 5 kez profesyonel bir top atıcısı olan Jared All", "gold_answer": "Kawann Short" }, { "question": "Pro Bowl için kaç tane Panthers savunma oyuncusu seçilmiştir?", "answerable": true, "abstained": false, "cited": "Arkalarında, Panthers'lerin üç adet ikinci hat savunma oyuncusundan ikisi de Pro Bowl'da oynamak için seçildi: Thomas Davis ve Luke Kuechly.", "gold_answer": "dört" }, { "question": "Thomas Davis kaç mecburi oyuncuyu düşürme almıştır?", "answerable": true, "abstained": false, "cited": "Davis, dördünde mecburi oyuncuyu düşürme ve dört topu kapma olmak üzere 5½ sack etmeyi bir araya getirirken, Kuechly top çalmalarda (118) iki mecburi oyuncuyu düşürme ve kendi başına dört pasını kesme", "gold_answer": "dört" }, { "question": "Panthers'lerin savunması 2015 sezonu top kapmalarının kaç tanesini almıştır?", "answerable": true, "abstained": true, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "24" }, { "question": "2015 yılında Panthers'lerin ofansif orta çizgi oyuncusu lideri kimdi?", "answerable": true, "abstained": true, "cited": "Panthers hattında ayrıca, sadece 9 başlangıçta 5 sack eden uç çizgi savunmacısı Kony Ealy ile birlikte 136 kez ile NFL'nin aktif kariyer sack lideri ve 5 kez profesyonel bir top atıcısı olan Jared All", "gold_answer": "Luke Kuechly." }, { "question": "Josh Norman 2015 yılında kale çizgisini geçişle sonlanan kaç topu çalma yapmıştır?", "answerable": true, "abstained": true, "cited": "Carolina’nın ikincisi olarak gösterilen Pro Bowl güvenliği için kariyerinin en yükseği olarak 7 top kapma ile takımı sürükleyen Kurt Coleman’a ve 88 top çalma sayısıyla ve Pro Bowl köşe savunmacısı se", "gold_answer": "iki" }, { "question": "Super Bowl XLIX'i kim kazanmıştır?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "New England Patriots" }, { "question": "AFC Şampiyonluk Oyunu final skoru neydi?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "20–18" }, { "question": "Super Bowl XLIX'i kim kazanmıştır?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "New England Patriots" }, { "question": "Broncos'un rövanş oyunları süresince kim hiç atmamıştır?", "answerable": true, "abstained": true, "cited": "Manning''in sezon boyunca topu kapmalarla ilgili sorunlarına rağmen, iki rövanş maçında hiç atmadı.", "gold_answer": "Manning" }, { "question": "2015 yılında klasmanlarını kazanmak için Broncos'u kim yenmiştir?", "answerable": true, "abstained": true, "cited": "Daha sonra, New England'ın 2 puanlık dönüşüm girişimini saat üzerinde 17 saniye kala bir pası keserek, AFC Şampiyonluk Oyununda Super Bowl'u savunan XLIX şampiyonu New England Patriots'u 20–18 yendi.", "gold_answer": "Pittsburgh Steelers" }, { "question": "Peyton Manning, Super Bowl %0'de oynadığı zaman kaç yaşındaydı?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "39" }, { "question": "Bir Super Bowl'da oynayan en yaşlı kilit oyuncu olma rekorunu daha önce kim elinde bulunduruyordu?", "answerable": true, "abstained": false, "cited": "Ayrıca, 39 yaşında bir Super Bowl'da oynayan en eski oyun kurucu oldu.", "gold_answer": "John Elway" }, { "question": "John Elway'in şu anki Broncos üyeliğindeki rolü nedir?", "answerable": true, "abstained": true, "cited": "Geçmiş rekor, Broncos'u 38 yaşında Super Bowl XXXIII'te galibiyete götüren ve şu anda Denver'ın Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü olan John Elway'in ellerindeydi.", "gold_answer": "Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü" }, { "question": "Broncos'un Genel Müdürü kimdir?", "answerable": true, "abstained": true, "cited": "Geçmiş rekor, Broncos'u 38 yaşında Super Bowl XXXIII'te galibiyete götüren ve şu anda Denver'ın Futbol Operasyonları Başkan Yardımcısı ve Genel Müdür'ü olan John Elway'in ellerindeydi.", "gold_answer": "John Elway" }, { "question": "Herhangi bir Super Bowl oyununda en yaşlı kilit oyuncu rekoruna kim sahiptir?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "Peyton Manning" }, { "question": "Manning Super Bowl'a ulaşmak için onların takımındayken kaç takımda oynamıştır?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "iki" }, { "question": "Peyton Manning Super Bowl'a kaç farklı takımı götürmüştür?", "answerable": true, "abstained": true, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "iki" }, { "question": "Bir Super Bowl'da oynayan en yaşlı kilit oyuncu kimdir?", "answerable": true, "abstained": false, "cited": "Ayrıca, 39 yaşında bir Super Bowl'da oynayan en eski oyun kurucu oldu.", "gold_answer": "Peyton Manning" }, { "question": "Manning'den önce bir Super Bowl'da oynayan en yaşlı kilit oyuncu kimdir?", "answerable": true, "abstained": false, "cited": "Peyton Manning,iki farklı takımı birden fazla Super Bowls'a gitmesinde önderlik eden ilk kilit oyuncu oldu.", "gold_answer": "John Elway" }, { "question": "Manning Super Bowl 50'de oynadığı zaman kaç yaşındaydı?", "answerable": true, "abstained": true, "cited": "Ayrıca, 39 yaşında bir Super Bowl'da oynayan en eski oyun kurucu oldu.", "gold_answer": "39" } ] }, "XQuAD-en (EN)": { "bm25": [ { "question": "How many tackles did Luke Kuechly register?", "answerable": true, "abstained": false, "cited": "Behind them, two of the Panthers three starting linebackers were also selected to play in the Pro Bowl: Thomas Davis and Luke Kuechly.", "gold_answer": "118" }, { "question": "How many balls did Josh Norman intercept?", "answerable": true, "abstained": false, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "four" }, { "question": "Who registered the most sacks on the team this season?", "answerable": true, "abstained": true, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "Kawann Short" }, { "question": "How many interceptions are the Panthers defense credited with in 2015?", "answerable": true, "abstained": true, "cited": "The Panthers defense gave up just 308 points, ranking sixth in the league, while also leading the NFL in interceptions with 24 and boasting four Pro Bowl selections.", "gold_answer": "24" }, { "question": "Who led the Panthers in sacks?", "answerable": true, "abstained": true, "cited": "Pro Bowl defensive tackle Kawann Short led the team in sacks with 11, while also forcing three fumbles and recovering two.", "gold_answer": "Kawann Short" }, { "question": "How many forced fumbles did Thomas Davis have?", "answerable": true, "abstained": false, "cited": "Davis compiled 5½ sacks, four forced fumbles, and four interceptions, while Kuechly led the team in tackles (118) forced two fumbles, and intercepted four passes of his own.", "gold_answer": "four" }, { "question": "Which player had the most interceptions for the season?", "answerable": true, "abstained": true, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "Kurt Coleman" }, { "question": "How many 2015 season interceptions did the Panthers' defense get?", "answerable": true, "abstained": true, "cited": "The Panthers defense gave up just 308 points, ranking sixth in the league, while also leading the NFL in interceptions with 24 and boasting four Pro Bowl selections.", "gold_answer": "24" }, { "question": "Who had five sacks in nine games as a Carolina Panthers starter?", "answerable": true, "abstained": true, "cited": "Behind them, two of the Panthers three starting linebackers were also selected to play in the Pro Bowl: Thomas Davis and Luke Kuechly.", "gold_answer": "Kony Ealy" }, { "question": "Who was the Panthers' tackle leader for 2015?", "answerable": true, "abstained": true, "cited": "The Panthers line also featured veteran defensive end Jared Allen, a 5-time pro bowler who was the NFL's active career sack leader with 136, along with defensive end Kony Ealy, who had 5 sacks in just", "gold_answer": "Luke Kuechly." }, { "question": "How many interceptions did Josh Norman score touchdowns with in 2015?", "answerable": true, "abstained": false, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "two." }, { "question": "Who won Super Bowl XLIX?", "answerable": true, "abstained": true, "cited": "They then beat the defending Super Bowl XLIX champion New England Patriots in the AFC Championship Game, 20–18, by intercepting a pass on New England's 2-point conversion attempt with 17 seconds left ", "gold_answer": "New England Patriots" }, { "question": "What was the final score of the AFC Championship Game?", "answerable": true, "abstained": true, "cited": "They then beat the defending Super Bowl XLIX champion New England Patriots in the AFC Championship Game, 20–18, by intercepting a pass on New England's 2-point conversion attempt with 17 seconds left ", "gold_answer": "20–18" }, { "question": "Who won Super Bowl XLIX?", "answerable": true, "abstained": true, "cited": "They then beat the defending Super Bowl XLIX champion New England Patriots in the AFC Championship Game, 20–18, by intercepting a pass on New England's 2-point conversion attempt with 17 seconds left ", "gold_answer": "New England Patriots" }, { "question": "Who did the Broncos beat in the divisional game?", "answerable": true, "abstained": true, "cited": "The Broncos defeated the Pittsburgh Steelers in the divisional round, 23–16, by scoring 11 points in the final three minutes of the game.", "gold_answer": "Pittsburgh Steelers" }, { "question": "Who did the Broncos defeat in the AFC Championship game?", "answerable": true, "abstained": true, "cited": "The Broncos defeated the Pittsburgh Steelers in the divisional round, 23–16, by scoring 11 points in the final three minutes of the game.", "gold_answer": "New England Patriots" }, { "question": "Who did the Broncos beat to win their division in 2015?", "answerable": true, "abstained": true, "cited": "The Broncos defeated the Pittsburgh Steelers in the divisional round, 23–16, by scoring 11 points in the final three minutes of the game.", "gold_answer": "Pittsburgh Steelers" }, { "question": "How old was Peyton Manning when he played in Super Bowl 50?", "answerable": true, "abstained": true, "cited": "Peyton Manning became the first quarterback ever to lead two different teams to multiple Super Bowls.", "gold_answer": "39" }, { "question": "Who previously held the record for being the oldest quarterback to play in a Super Bowl?", "answerable": true, "abstained": false, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "John Elway" }, { "question": "What team was the winner of Super Bowl XXXIII?", "answerable": true, "abstained": true, "cited": "The past record was held by John Elway, who led the Broncos to victory in Super Bowl XXXIII at age 38 and is currently Denver's Executive Vice President of Football Operations and General Manager.", "gold_answer": "Broncos" }, { "question": "Who is the General Manager for the Broncos?", "answerable": true, "abstained": true, "cited": "The past record was held by John Elway, who led the Broncos to victory in Super Bowl XXXIII at age 38 and is currently Denver's Executive Vice President of Football Operations and General Manager.", "gold_answer": "John Elway" }, { "question": "How old was Elway during his Super Bowl XXXIII win?", "answerable": true, "abstained": true, "cited": "The past record was held by John Elway, who led the Broncos to victory in Super Bowl XXXIII at age 38 and is currently Denver's Executive Vice President of Football Operations and General Manager.", "gold_answer": "38" }, { "question": "Who has the record of being the oldest quarter back in any Super Bowl game?", "answerable": true, "abstained": true, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "Peyton Manning" }, { "question": "How many teams has Manning played for that reached the Super Bowl, while he was on their team?", "answerable": true, "abstained": true, "cited": "Peyton Manning became the first quarterback ever to lead two different teams to multiple Super Bowls.", "gold_answer": "two" }, { "question": "Who is the oldest quarterback to play in a Super Bowl?", "answerable": true, "abstained": false, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "Peyton Manning" } ], "learned": [ { "question": "How many tackles did Luke Kuechly register?", "answerable": true, "abstained": true, "cited": "Behind them, two of the Panthers three starting linebackers were also selected to play in the Pro Bowl: Thomas Davis and Luke Kuechly.", "gold_answer": "118" }, { "question": "How many balls did Josh Norman intercept?", "answerable": true, "abstained": false, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "four" }, { "question": "Who registered the most sacks on the team this season?", "answerable": true, "abstained": true, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "Kawann Short" }, { "question": "How many interceptions are the Panthers defense credited with in 2015?", "answerable": true, "abstained": true, "cited": "The Panthers defense gave up just 308 points, ranking sixth in the league, while also leading the NFL in interceptions with 24 and boasting four Pro Bowl selections.", "gold_answer": "24" }, { "question": "Who led the Panthers in sacks?", "answerable": true, "abstained": true, "cited": "Pro Bowl defensive tackle Kawann Short led the team in sacks with 11, while also forcing three fumbles and recovering two.", "gold_answer": "Kawann Short" }, { "question": "How many forced fumbles did Thomas Davis have?", "answerable": true, "abstained": false, "cited": "Davis compiled 5½ sacks, four forced fumbles, and four interceptions, while Kuechly led the team in tackles (118) forced two fumbles, and intercepted four passes of his own.", "gold_answer": "four" }, { "question": "How many 2015 season interceptions did the Panthers' defense get?", "answerable": true, "abstained": true, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "24" }, { "question": "Who was the Panthers' tackle leader for 2015?", "answerable": true, "abstained": true, "cited": "The Panthers line also featured veteran defensive end Jared Allen, a 5-time pro bowler who was the NFL's active career sack leader with 136, along with defensive end Kony Ealy, who had 5 sacks in just", "gold_answer": "Luke Kuechly." }, { "question": "How many interceptions did Josh Norman score touchdowns with in 2015?", "answerable": true, "abstained": true, "cited": "Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed", "gold_answer": "two." }, { "question": "Who won Super Bowl XLIX?", "answerable": true, "abstained": true, "cited": "They then beat the defending Super Bowl XLIX champion New England Patriots in the AFC Championship Game, 20–18, by intercepting a pass on New England's 2-point conversion attempt with 17 seconds left ", "gold_answer": "New England Patriots" }, { "question": "Who won Super Bowl XLIX?", "answerable": true, "abstained": true, "cited": "They then beat the defending Super Bowl XLIX champion New England Patriots in the AFC Championship Game, 20–18, by intercepting a pass on New England's 2-point conversion attempt with 17 seconds left ", "gold_answer": "New England Patriots" }, { "question": "Who did the Broncos beat to win their division in 2015?", "answerable": true, "abstained": true, "cited": "The Broncos defeated the Pittsburgh Steelers in the divisional round, 23–16, by scoring 11 points in the final three minutes of the game.", "gold_answer": "Pittsburgh Steelers" }, { "question": "How old was Peyton Manning when he played in Super Bowl 50?", "answerable": true, "abstained": true, "cited": "Peyton Manning became the first quarterback ever to lead two different teams to multiple Super Bowls.", "gold_answer": "39" }, { "question": "Who previously held the record for being the oldest quarterback to play in a Super Bowl?", "answerable": true, "abstained": false, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "John Elway" }, { "question": "Who has the record of being the oldest quarter back in any Super Bowl game?", "answerable": true, "abstained": false, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "Peyton Manning" }, { "question": "How many teams has Manning played for that reached the Super Bowl, while he was on their team?", "answerable": true, "abstained": true, "cited": "Peyton Manning became the first quarterback ever to lead two different teams to multiple Super Bowls.", "gold_answer": "two" }, { "question": "Who is the oldest quarterback to play in a Super Bowl?", "answerable": true, "abstained": false, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "Peyton Manning" }, { "question": "Prior to Manning, who was the oldest quarterback to play in a Super Bowl?", "answerable": true, "abstained": false, "cited": "He is also the oldest quarterback ever to play in a Super Bowl at age 39.", "gold_answer": "John Elway" }, { "question": "How old was Manning when he played Super Bowl 50?", "answerable": true, "abstained": true, "cited": "Peyton Manning became the first quarterback ever to lead two different teams to multiple Super Bowls.", "gold_answer": "39" }, { "question": "How many Grammys has Lady Gaga won?", "answerable": true, "abstained": true, "cited": "Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.", "gold_answer": "Six" }, { "question": "What actress did the ASL translation for the game?", "answerable": true, "abstained": true, "cited": "Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.", "gold_answer": "Marlee Matlin" }, { "question": "Who did the National Anthem at Super Bowl 50?", "answerable": true, "abstained": true, "cited": "Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.", "gold_answer": "Lady Gaga" }, { "question": "What actor did sign language for the National Anthem at Superbowl 50?", "answerable": true, "abstained": true, "cited": "Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.", "gold_answer": "Marlee Matlin" }, { "question": "Who did the Super Bowl 50 National Anthem?", "answerable": true, "abstained": true, "cited": "Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.", "gold_answer": "Lady Gaga" }, { "question": "How many Grammy Award's does the text say Lady Gaga has won?", "answerable": true, "abstained": true, "cited": "Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.", "gold_answer": "Six" } ] } }, "hand_written_examples": [ { "title": "TR · kütüphane (yanıtlanabilir)", "abstain": false, "answerable": 0.6524745300529133, "evidence": [ "Kütüphanede bugün yaklaşık 40 bin kitap bulunuyor." ] }, { "title": "TR · kütüphane (belgede yok, yanıltıcı isim)", "abstain": true, "answerable": 0.27141084178818015, "evidence": [] }, { "title": "TR · şirket (yanıltıcı yıl)", "abstain": true, "answerable": 0.28409286741619816, "evidence": [] }, { "title": "TR · Ay (yanıtlanabilir)", "abstain": false, "answerable": 0.6624891481709587, "evidence": [ "Ay'a ilk insanlı iniş 1969 yılında gerçekleşti." ] }, { "title": "EN · bridge (answerable)", "abstain": false, "answerable": 0.756120137796548, "evidence": [ "Its main span is 503 metres long." ] }, { "title": "EN · bridge (not in the document)", "abstain": true, "answerable": 0.5633153393855327, "evidence": [] }, { "title": "XQuAD-tr örneği", "abstain": false, "answerable": 0.7965940107656674, "evidence": [ "Panthers savunması ligdeki derecesi altıncı sırada olarak sadece 308 sayı bıraktı, aynı zamanda NFL'de 24 topu kapma ile ligin başını çekmesi ve dört Pro Bowl seçmeleri ile övünç duydu." ] }, { "title": "XQuAD-tr · başka paragrafa sorulmuş soru", "abstain": true, "answerable": 0.521832748704309, "evidence": [] }, { "title": "SQuAD 2.0 dev · unanswerable", "abstain": true, "answerable": 0.18313970364344162, "evidence": [] } ], "tests": { "reload": { "returncode": 0, "seconds": 0.18, "process": "fresh python -E subprocess, cwd = temporary copy of the model folder", "passed": true, "cases": 9, "max_abs_diff": 0.0, "decision_mismatches": [] }, "node_parity": { "cases": 9, "failed": 0, "maxDiff": 1.6653345369377348e-16, "results": [ { "name": "TR · kütüphane (yanıtlanabilir)", "maxDiff": 1.6653345369377348e-16 }, { "name": "TR · kütüphane (belgede yok, yanıltıcı isim)", "maxDiff": 1.1102230246251565e-16 }, { "name": "TR · şirket (yanıltıcı yıl)", "maxDiff": 1.6653345369377348e-16 }, { "name": "TR · Ay (yanıtlanabilir)", "maxDiff": 8.673617379884035e-18 }, { "name": "EN · bridge (answerable)", "maxDiff": 3.122502256758253e-17 }, { "name": "EN · bridge (not in the document)", "maxDiff": 1.1102230246251565e-16 }, { "name": "XQuAD-tr örneği", "maxDiff": 1.0408340855860843e-17 }, { "name": "XQuAD-tr · başka paragrafa sorulmuş soru", "maxDiff": 1.1102230246251565e-16 }, { "name": "SQuAD 2.0 dev · unanswerable", "maxDiff": 5.551115123125783e-17 } ], "ms": 15, "runtime": "node v22.23.4", "status": "passed" }, "browser": { "status": "skipped", "reason": "playwright not installed" } }, "notes": [ "Turkish is zero-shot: the model never saw Turkish during training or threshold selection.", "XQuAD unanswerables are constructed (question paired with another paragraph of the same article, answer string absent)." ] }
{
"project": "Next Trace",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"sources": {
"squad_train": {
"url": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json",
"file": "train-v2.0.json",
"bytes": 42123633,
"sha256": "68dcfbb971bd3e96d5b46c7177b16c1a4e7d4bdef19fb204502738552dede002",
"verified_against_pinned_sha256": false
},
"squad_dev": {
"url": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json",
"file": "dev-v2.0.json",
"bytes": 4370528,
"sha256": "80a5225e94905956a6446d296ca1093975c4d3b3260f1d6c8f68bc2ab77182d8",
"verified_against_pinned_sha256": false
},
"xquad_tr": {
"url": "https://raw.githubusercontent.com/google-deepmind/xquad/master/xquad.tr.json",
"file": "xquad.tr.json",
"bytes": 729506,
"sha256": "92179a564774b7696100d144c1e10870d0a966b6fccbdd254a65b9d2ab1971cc",
"verified_against_pinned_sha256": false
},
"xquad_en": {
"url": "https://raw.githubusercontent.com/google-deepmind/xquad/master/xquad.en.json",
"file": "xquad.en.json",
"bytes": 609383,
"sha256": "e4c57d1c9143aaa1c5d265ba5987a65f4e69528d2a98f29d6e75019b10344f29",
"verified_against_pinned_sha256": false
}
},
"license": "CC BY-SA 4.0 (SQuAD 2.0, XQuAD)",
"versions": {
"squad": "2.0",
"xquad": "GitHub master at download time (sha256 recorded)"
},
"splits": {
"squad_train_A": 126503,
"squad_train_B": 16851,
"squad_train_V": 15937,
"SQuAD2-dev (EN, adversarial unanswerables)": {
"questions": 11873,
"unanswerable": 5945
},
"XQuAD-tr (TR, zero-shot)": {
"questions": 2327,
"unanswerable": 1137
},
"XQuAD-en (EN)": {
"questions": 2319,
"unanswerable": 1129
},
"rule": "SQuAD train split by article title (A: sentence model, B: answerability head, V: thresholds); each split also gets ~35% extra unanswerables made by asking a question against another paragraph of the same article; XQuAD and SQuAD dev only for testing"
}
},
"thresholds": {
"bm25": 2.7921873897475704,
"answerable": 0.6,
"second_sentence": 0.5
},
"validation": {
"bm25": {
"questions": 15937,
"trace_accuracy": 0.5530526447888561,
"evidence_top1_on_answerable": 0.7978169818504887,
"answered_rate_on_answerable": 0.597537758598807,
"correct_abstention_on_unanswerable": 0.5953090096798213,
"cited_plausible_distractor_on_unanswerable": 0.29573095060809135,
"answerable": 7879,
"unanswerable": 8058
},
"learned": {
"questions": 15937,
"trace_accuracy": 0.5940892263286691,
"evidence_top1_on_answerable": 0.8188856453864704,
"answered_rate_on_answerable": 0.7587257266150527,
"correct_abstention_on_unanswerable": 0.5409530900967983,
"cited_plausible_distractor_on_unanswerable": 0.339662447257384,
"answerable": 7879,
"unanswerable": 8058
}
},
"results": {
"SQuAD2-dev (EN, adversarial unanswerables)": {
"bm25": {
"questions": 11873,