Spaces:
Sleeping
Sleeping
File size: 2,104 Bytes
dd990bf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | {
"dataset": "hover",
"questions": 300,
"candidate_count_total": 3000,
"candidate_pool": "10 HotpotQA distractor passages, only HoVer claims with all support titles present",
"gold_support_count_distribution": {
"2": 152,
"3": 144,
"4": 4
},
"gold_available_fraction": 100.0,
"gold_available_count": 300,
"graph_link_fraction": 94.33,
"oracle_complete_coverage_pct": {
"2": 50.67,
"3": 98.67,
"5": 100.0,
"8": 100.0
},
"complete_coverage_pct": {
"dense_cosine": {
"2": 26.33,
"3": 47.33,
"5": 75.67,
"8": 92.0
},
"crossencoder": {
"2": 33.0,
"3": 57.0,
"5": 82.0,
"8": 92.33
},
"dense_graph": {
"2": 32.33,
"3": 73.33,
"5": 89.33,
"8": 97.33
},
"crossencoder_graph": {
"2": 38.33,
"3": 72.67,
"5": 92.33,
"8": 98.67
}
},
"crossencoder_graph_minus_crossencoder_at3_pp": 15.67,
"ci95_pp": [
9.99,
22.0
],
"paired_wins_losses_ties_at3": {
"wins": 70,
"losses": 23,
"ties": 207
},
"timing_seconds": {
"embedding_seconds": 45.51700891600012,
"prep_seconds": 45.92742176499996,
"candidate_pool_size": 10,
"eligibility": "HoVer dev claims with HotpotQA dev passages available and all annotated support titles present; conditional subset",
"crossencoder_scoring_seconds": 84.113,
"crossencoder_scoring_ms_per_question": 280.38,
"crossencoder_forward_seconds": 83.542,
"graph_selection_total_s": 0.11,
"graph_selection_ms_per_question": 0.37
},
"models": {
"dense": "user uploaded all-MiniLM-L6-v2",
"crossencoder": "user uploaded cross-encoder/ms-marco-MiniLM-L6-v2"
},
"protocol": "MiniLM embeddings at max 128 WordPiece for dense. Cross-encoder truncates paired query+passage at 256 WordPiece. Both score exact same candidate pools. Fixed evidence graph .4 minmax relevance + .5 title in already-selected passage + .2 query-title score. No TF-IDF anywhere in candidate construction or scoring. Gold used solely for pool audit and evaluation."
} |