| { |
| "queries": 970, |
| "models": { |
| "cuda": { |
| "1": { |
| "scored_queries": 965, |
| "excluded_queries": 5, |
| "hit": 0.7233160621761658, |
| "precision": 0.7233160621761658, |
| "macro_precision": 0.7233160621761658, |
| "ndcg": 0.6458425857389588, |
| "known_positive_recall": 0.021209009540392988, |
| "recall_queries": 920, |
| "useful": 698, |
| "retained": 965, |
| "excluded_positions": 0, |
| "mean_useful": 0.7233160621761658, |
| "mean_retained": 1.0 |
| }, |
| "3": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8536082474226804, |
| "precision": 0.7157640565712314, |
| "macro_precision": 0.7166666666666667, |
| "ndcg": 0.6374467700901159, |
| "known_positive_recall": 0.0589083117861917, |
| "recall_queries": 925, |
| "useful": 2075, |
| "retained": 2899, |
| "excluded_positions": 11, |
| "mean_useful": 2.1391752577319587, |
| "mean_retained": 2.988659793814433 |
| }, |
| "5": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8845360824742268, |
| "precision": 0.7028311634635255, |
| "macro_precision": 0.7031786941580757, |
| "ndcg": 0.632593817782834, |
| "known_positive_recall": 0.09351524011264825, |
| "recall_queries": 925, |
| "useful": 3401, |
| "retained": 4839, |
| "excluded_positions": 11, |
| "mean_useful": 3.5061855670103093, |
| "mean_retained": 4.988659793814433 |
| }, |
| "10": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8989690721649485, |
| "precision": 0.67180070291503, |
| "macro_precision": 0.6722631320569465, |
| "ndcg": 0.6315462046841925, |
| "known_positive_recall": 0.17039667688880922, |
| "recall_queries": 925, |
| "useful": 6499, |
| "retained": 9674, |
| "excluded_positions": 26, |
| "mean_useful": 6.7, |
| "mean_retained": 9.97319587628866 |
| }, |
| "20": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.9103092783505154, |
| "precision": 0.61794500723589, |
| "macro_precision": 0.6184956360634603, |
| "ndcg": 0.6416731640651591, |
| "known_positive_recall": 0.29717756138114526, |
| "recall_queries": 925, |
| "useful": 11956, |
| "retained": 19348, |
| "excluded_positions": 52, |
| "mean_useful": 12.32577319587629, |
| "mean_retained": 19.94639175257732 |
| }, |
| "50": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.9309278350515464, |
| "precision": 0.4043216890119198, |
| "macro_precision": 0.4044939351610387, |
| "ndcg": 0.6051209510262134, |
| "known_positive_recall": 0.4598077844640693, |
| "recall_queries": 925, |
| "useful": 19572, |
| "retained": 48407, |
| "excluded_positions": 93, |
| "mean_useful": 20.177319587628865, |
| "mean_retained": 49.904123711340205 |
| }, |
| "selected": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8618556701030928, |
| "precision": 0.8033770583310076, |
| "macro_precision": 0.7040325511660611, |
| "ndcg": 0.6359990281117442, |
| "known_positive_recall": 0.128026594631235, |
| "recall_queries": 925, |
| "useful": 5757, |
| "retained": 7166, |
| "excluded_positions": 17, |
| "mean_useful": 5.935051546391753, |
| "mean_retained": 7.387628865979382 |
| } |
| }, |
| "vulkan": { |
| "1": { |
| "scored_queries": 965, |
| "excluded_queries": 5, |
| "hit": 0.7243523316062176, |
| "precision": 0.7243523316062176, |
| "macro_precision": 0.7243523316062176, |
| "ndcg": 0.6461386627189736, |
| "known_positive_recall": 0.021228078953055077, |
| "recall_queries": 920, |
| "useful": 699, |
| "retained": 965, |
| "excluded_positions": 0, |
| "mean_useful": 0.7243523316062176, |
| "mean_retained": 1.0 |
| }, |
| "3": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8525773195876288, |
| "precision": 0.7157640565712314, |
| "macro_precision": 0.7166666666666667, |
| "ndcg": 0.6373247400469291, |
| "known_positive_recall": 0.05893952330659241, |
| "recall_queries": 925, |
| "useful": 2075, |
| "retained": 2899, |
| "excluded_positions": 11, |
| "mean_useful": 2.1391752577319587, |
| "mean_retained": 2.988659793814433 |
| }, |
| "5": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8845360824742268, |
| "precision": 0.7023563455973543, |
| "macro_precision": 0.702766323024055, |
| "ndcg": 0.6321994825191731, |
| "known_positive_recall": 0.09344587015679974, |
| "recall_queries": 925, |
| "useful": 3398, |
| "retained": 4838, |
| "excluded_positions": 12, |
| "mean_useful": 3.5030927835051546, |
| "mean_retained": 4.9876288659793815 |
| }, |
| "10": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8989690721649485, |
| "precision": 0.6723514211886304, |
| "macro_precision": 0.6727589592538046, |
| "ndcg": 0.631891212140629, |
| "known_positive_recall": 0.17053838380216288, |
| "recall_queries": 925, |
| "useful": 6505, |
| "retained": 9675, |
| "excluded_positions": 25, |
| "mean_useful": 6.706185567010309, |
| "mean_retained": 9.974226804123711 |
| }, |
| "20": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.9103092783505154, |
| "precision": 0.6178416373785404, |
| "macro_precision": 0.6183925432799551, |
| "ndcg": 0.6416203428445276, |
| "known_positive_recall": 0.2971249330393808, |
| "recall_queries": 925, |
| "useful": 11954, |
| "retained": 19348, |
| "excluded_positions": 52, |
| "mean_useful": 12.323711340206186, |
| "mean_retained": 19.94639175257732 |
| }, |
| "50": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.9309278350515464, |
| "precision": 0.4043216890119198, |
| "macro_precision": 0.4044939351610387, |
| "ndcg": 0.6050905860403181, |
| "known_positive_recall": 0.4598077844640693, |
| "recall_queries": 925, |
| "useful": 19572, |
| "retained": 48407, |
| "excluded_positions": 93, |
| "mean_useful": 20.177319587628865, |
| "mean_retained": 49.904123711340205 |
| }, |
| "selected": { |
| "scored_queries": 970, |
| "excluded_queries": 0, |
| "hit": 0.8608247422680413, |
| "precision": 0.8027855153203343, |
| "macro_precision": 0.7038533836490732, |
| "ndcg": 0.6358997417721292, |
| "known_positive_recall": 0.12825382711378852, |
| "recall_queries": 925, |
| "useful": 5764, |
| "retained": 7180, |
| "excluded_positions": 17, |
| "mean_useful": 5.942268041237114, |
| "mean_retained": 7.402061855670103 |
| } |
| } |
| }, |
| "public": {}, |
| "answerable": { |
| "queries": 925, |
| "models": { |
| "cuda": { |
| "1": { |
| "scored_queries": 920, |
| "excluded_queries": 5, |
| "hit": 0.758695652173913, |
| "precision": 0.758695652173913, |
| "macro_precision": 0.758695652173913, |
| "ndcg": 0.6513457556935818, |
| "known_positive_recall": 0.021209009540392988, |
| "recall_queries": 920, |
| "useful": 698, |
| "retained": 920, |
| "excluded_positions": 0, |
| "mean_useful": 0.758695652173913, |
| "mean_retained": 1.0 |
| }, |
| "3": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.8951351351351351, |
| "precision": 0.7507235890014472, |
| "macro_precision": 0.7515315315315315, |
| "ndcg": 0.6422807686434969, |
| "known_positive_recall": 0.0589083117861917, |
| "recall_queries": 925, |
| "useful": 2075, |
| "retained": 2764, |
| "excluded_positions": 11, |
| "mean_useful": 2.2432432432432434, |
| "mean_retained": 2.9881081081081082 |
| }, |
| "5": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9275675675675675, |
| "precision": 0.7371044646727352, |
| "macro_precision": 0.7373873873873874, |
| "ndcg": 0.6375956845322198, |
| "known_positive_recall": 0.09351524011264825, |
| "recall_queries": 925, |
| "useful": 3401, |
| "retained": 4614, |
| "excluded_positions": 11, |
| "mean_useful": 3.676756756756757, |
| "mean_retained": 4.988108108108108 |
| }, |
| "10": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9427027027027027, |
| "precision": 0.7045750216825672, |
| "macro_precision": 0.704967824967825, |
| "ndcg": 0.6377512015628823, |
| "known_positive_recall": 0.17039667688880922, |
| "recall_queries": 925, |
| "useful": 6499, |
| "retained": 9224, |
| "excluded_positions": 26, |
| "mean_useful": 7.025945945945946, |
| "mean_retained": 9.971891891891891 |
| }, |
| "20": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9545945945945946, |
| "precision": 0.6480919340849957, |
| "macro_precision": 0.648584612953034, |
| "ndcg": 0.6491335337062211, |
| "known_positive_recall": 0.29717756138114526, |
| "recall_queries": 925, |
| "useful": 11956, |
| "retained": 18448, |
| "excluded_positions": 52, |
| "mean_useful": 12.925405405405405, |
| "mean_retained": 19.943783783783783 |
| }, |
| "50": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9762162162162162, |
| "precision": 0.424031024546656, |
| "macro_precision": 0.42417201849319736, |
| "ndcg": 0.6129768665312807, |
| "known_positive_recall": 0.4598077844640693, |
| "recall_queries": 925, |
| "useful": 19572, |
| "retained": 46157, |
| "excluded_positions": 93, |
| "mean_useful": 21.158918918918918, |
| "mean_retained": 49.89945945945946 |
| }, |
| "selected": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9037837837837838, |
| "precision": 0.8188024463092021, |
| "macro_precision": 0.7382827833849506, |
| "ndcg": 0.6407625959742856, |
| "known_positive_recall": 0.128026594631235, |
| "recall_queries": 925, |
| "useful": 5757, |
| "retained": 7031, |
| "excluded_positions": 17, |
| "mean_useful": 6.223783783783784, |
| "mean_retained": 7.601081081081081 |
| } |
| }, |
| "vulkan": { |
| "1": { |
| "scored_queries": 920, |
| "excluded_queries": 5, |
| "hit": 0.7597826086956522, |
| "precision": 0.7597826086956522, |
| "macro_precision": 0.7597826086956522, |
| "ndcg": 0.6516563146997929, |
| "known_positive_recall": 0.021228078953055077, |
| "recall_queries": 920, |
| "useful": 699, |
| "retained": 920, |
| "excluded_positions": 0, |
| "mean_useful": 0.7597826086956522, |
| "mean_retained": 1.0 |
| }, |
| "3": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.894054054054054, |
| "precision": 0.7507235890014472, |
| "macro_precision": 0.7515315315315315, |
| "ndcg": 0.6421528020036147, |
| "known_positive_recall": 0.05893952330659241, |
| "recall_queries": 925, |
| "useful": 2075, |
| "retained": 2764, |
| "excluded_positions": 11, |
| "mean_useful": 2.2432432432432434, |
| "mean_retained": 2.9881081081081082 |
| }, |
| "5": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9275675675675675, |
| "precision": 0.7366139171905485, |
| "macro_precision": 0.736954954954955, |
| "ndcg": 0.6371821653908674, |
| "known_positive_recall": 0.09344587015679974, |
| "recall_queries": 925, |
| "useful": 3398, |
| "retained": 4613, |
| "excluded_positions": 12, |
| "mean_useful": 3.6735135135135133, |
| "mean_retained": 4.987027027027027 |
| }, |
| "10": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9427027027027027, |
| "precision": 0.7051490514905149, |
| "macro_precision": 0.7054877734877735, |
| "ndcg": 0.6381124055299272, |
| "known_positive_recall": 0.17053838380216288, |
| "recall_queries": 925, |
| "useful": 6505, |
| "retained": 9225, |
| "excluded_positions": 25, |
| "mean_useful": 7.032432432432432, |
| "mean_retained": 9.972972972972974 |
| }, |
| "20": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9545945945945946, |
| "precision": 0.6479835212489159, |
| "macro_precision": 0.6484765048449259, |
| "ndcg": 0.6490652861835569, |
| "known_positive_recall": 0.2971249330393808, |
| "recall_queries": 925, |
| "useful": 11954, |
| "retained": 18448, |
| "excluded_positions": 52, |
| "mean_useful": 12.923243243243244, |
| "mean_retained": 19.943783783783783 |
| }, |
| "50": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9762162162162162, |
| "precision": 0.424031024546656, |
| "macro_precision": 0.42417201849319736, |
| "ndcg": 0.6129402668287812, |
| "known_positive_recall": 0.4598077844640693, |
| "recall_queries": 925, |
| "useful": 19572, |
| "retained": 46157, |
| "excluded_positions": 93, |
| "mean_useful": 21.158918918918918, |
| "mean_retained": 49.89945945945946 |
| }, |
| "selected": { |
| "scored_queries": 925, |
| "excluded_queries": 0, |
| "hit": 0.9027027027027027, |
| "precision": 0.8181689141234918, |
| "macro_precision": 0.7380948996103794, |
| "ndcg": 0.640658479488419, |
| "known_positive_recall": 0.12825382711378852, |
| "recall_queries": 925, |
| "useful": 5764, |
| "retained": 7045, |
| "excluded_positions": 17, |
| "mean_useful": 6.231351351351352, |
| "mean_retained": 7.616216216216216 |
| } |
| } |
| } |
| } |
| } |
|
|
