{
"checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal/retriever.pt",
"use_retrieval_rope": false,
"time_embedding_size": 128,
"memory_temporal_positions_shuffled": false,
"temporal_wording_counts": {
"train:after": 1,
"train:before": 98,
"train:comparison": 903,
"train:earliest": 783,
"train:latest": 748,
"train:session": 1431,
"train:unmatched": 390,
"validation:before": 70,
"validation:comparison": 414,
"validation:earliest": 354,
"validation:latest": 358,
"validation:session": 789,
"validation:unmatched": 161
},
"task_distribution": {
"all_complete_samples": {
"sample_count": 6168,
"rag_query_count": 10985,
"control_query_count": 111,
"by_task": {
"change_comparison": {
"rag_query_count": 1480,
"control_query_count": 0
},
"current_only_control": {
"rag_query_count": 0,
"control_query_count": 110
},
"direct_recall": {
"rag_query_count": 2814,
"control_query_count": 0
},
"historical_state": {
"rag_query_count": 1508,
"control_query_count": 0
},
"latest_state": {
"rag_query_count": 1470,
"control_query_count": 0
},
"multi_hop_composition": {
"rag_query_count": 1671,
"control_query_count": 0
},
"recent_context_control": {
"rag_query_count": 0,
"control_query_count": 1
},
"temporal_localization": {
"rag_query_count": 2042,
"control_query_count": 0
}
}
},
"train_split": {
"sample_count": 5014,
"rag_query_count": 7000,
"control_query_count": 0,
"by_task": {
"change_comparison": {
"rag_query_count": 1010,
"control_query_count": 0
},
"direct_recall": {
"rag_query_count": 1323,
"control_query_count": 0
},
"historical_state": {
"rag_query_count": 1008,
"control_query_count": 0
},
"latest_state": {
"rag_query_count": 1013,
"control_query_count": 0
},
"multi_hop_composition": {
"rag_query_count": 1323,
"control_query_count": 0
},
"temporal_localization": {
"rag_query_count": 1323,
"control_query_count": 0
}
}
},
"validation_split": {
"sample_count": 1154,
"rag_query_count": 3985,
"control_query_count": 111,
"by_task": {
"change_comparison": {
"rag_query_count": 470,
"control_query_count": 0
},
"current_only_control": {
"rag_query_count": 0,
"control_query_count": 110
},
"direct_recall": {
"rag_query_count": 1491,
"control_query_count": 0
},
"historical_state": {
"rag_query_count": 500,
"control_query_count": 0
},
"latest_state": {
"rag_query_count": 457,
"control_query_count": 0
},
"multi_hop_composition": {
"rag_query_count": 348,
"control_query_count": 0
},
"recent_context_control": {
"rag_query_count": 0,
"control_query_count": 1
},
"temporal_localization": {
"rag_query_count": 719,
"control_query_count": 0
}
}
}
},
"validation": {
"query_count": 3985,
"recall_at_k": {
"1": 0.6943565607070923,
"3": 0.9150701761245728,
"5": 0.9561480283737183
},
"all_support_em_at_k": {
"3": 0.8687578439712524,
"5": 0.9327477812767029
},
"by_speaker_query_type": {
"fact_to_person": {
"query_count": 786,
"recall_at_k": {
"1": 0.825658917427063,
"3": 0.9750257730484009,
"5": 0.9804449677467346
},
"all_support_em_at_k": {
"3": 0.9631043076515198,
"5": 0.9681933522224426
}
},
"person_to_fact": {
"query_count": 969,
"recall_at_k": {
"1": 0.7372819781303406,
"3": 0.9379526972770691,
"5": 0.9745736718177795
},
"all_support_em_at_k": {
"3": 0.9050567746162415,
"5": 0.9659442901611328
}
},
"self_to_fact": {
"query_count": 509,
"recall_at_k": {
"1": 0.2673543095588684,
"3": 0.6982645392417908,
"5": 0.898657500743866
},
"all_support_em_at_k": {
"3": 0.48919451236724854,
"5": 0.8251473307609558
}
},
"speaker_independent": {
"query_count": 1721,
"recall_at_k": {
"1": 0.7365096807479858,
"3": 0.9389259815216064,
"5": 0.9516802430152893
},
"all_support_em_at_k": {
"3": 0.9174898266792297,
"5": 0.9296920299530029
}
}
},
"by_task_and_speaker_query_type": {
"change_comparison": {
"fact_to_person": {
"query_count": 74,
"recall_at_k": {
"1": 0.5,
"3": 1.0,
"5": 1.0
},
"all_support_em_at_k": {
"3": 1.0,
"5": 1.0
}
},
"person_to_fact": {
"query_count": 88,
"recall_at_k": {
"1": 0.5,
"3": 0.9545454382896423,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.9090908765792847,
"5": 1.0
}
},
"self_to_fact": {
"query_count": 99,
"recall_at_k": {
"1": 0.1111111119389534,
"3": 0.49494948983192444,
"5": 0.8282828330993652
},
"all_support_em_at_k": {
"3": 0.09090909361839294,
"5": 0.6565656661987305
}
},
"speaker_independent": {
"query_count": 209,
"recall_at_k": {
"1": 0.49441784620285034,
"3": 0.9896332025527954,
"5": 0.9932217597961426
},
"all_support_em_at_k": {
"3": 0.9856459498405457,
"5": 0.9904305934906006
}
}
},
"direct_recall": {
"fact_to_person": {
"query_count": 524,
"recall_at_k": {
"1": 0.9266858100891113,
"3": 0.9726463556289673,
"5": 0.980120837688446
},
"all_support_em_at_k": {
"3": 0.9599236249923706,
"5": 0.9675572514533997
}
},
"person_to_fact": {
"query_count": 306,
"recall_at_k": {
"1": 0.8153594732284546,
"3": 0.9229303598403931,
"5": 0.9507080912590027
},
"all_support_em_at_k": {
"3": 0.9117646813392639,
"5": 0.9346405267715454
}
},
"self_to_fact": {
"query_count": 57,
"recall_at_k": {
"1": 0.4312865138053894,
"3": 0.5979532599449158,
"5": 0.6769006252288818
},
"all_support_em_at_k": {
"3": 0.42105263471603394,
"5": 0.5438596606254578
}
},
"speaker_independent": {
"query_count": 604,
"recall_at_k": {
"1": 0.8199503421783447,
"3": 0.9064569473266602,
"5": 0.9199779629707336
},
"all_support_em_at_k": {
"3": 0.8824503421783447,
"5": 0.8923841118812561
}
}
},
"historical_state": {
"fact_to_person": {
"query_count": 91,
"recall_at_k": {
"1": 0.8681318759918213,
"3": 1.0,
"5": 1.0
},
"all_support_em_at_k": {
"3": 1.0,
"5": 1.0
}
},
"person_to_fact": {
"query_count": 125,
"recall_at_k": {
"1": 0.8320000171661377,
"3": 1.0,
"5": 1.0
},
"all_support_em_at_k": {
"3": 1.0,
"5": 1.0
}
},
"self_to_fact": {
"query_count": 82,
"recall_at_k": {
"1": 0.353658527135849,
"3": 0.9512194991111755,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.9512194991111755,
"5": 1.0
}
},
"speaker_independent": {
"query_count": 202,
"recall_at_k": {
"1": 0.896039605140686,
"3": 0.9975247383117676,
"5": 0.9975247383117676
},
"all_support_em_at_k": {
"3": 0.9950494766235352,
"5": 0.9950494766235352
}
}
},
"latest_state": {
"fact_to_person": {
"query_count": 31,
"recall_at_k": {
"1": 0.5483871102333069,
"3": 1.0,
"5": 1.0
},
"all_support_em_at_k": {
"3": 1.0,
"5": 1.0
}
},
"person_to_fact": {
"query_count": 92,
"recall_at_k": {
"1": 0.46195653080940247,
"3": 0.989130437374115,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.989130437374115,
"5": 1.0
}
},
"self_to_fact": {
"query_count": 82,
"recall_at_k": {
"1": 0.18292683362960815,
"3": 0.8536585569381714,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.8536585569381714,
"5": 1.0
}
},
"speaker_independent": {
"query_count": 252,
"recall_at_k": {
"1": 0.4523809552192688,
"3": 0.9821428656578064,
"5": 0.9980158805847168
},
"all_support_em_at_k": {
"3": 0.9801587462425232,
"5": 0.9960317611694336
}
}
},
"multi_hop_composition": {
"fact_to_person": {
"query_count": 66,
"recall_at_k": {
"1": 0.46037158370018005,
"3": 0.9197511076927185,
"5": 0.9249458909034729
},
"all_support_em_at_k": {
"3": 0.8787878751754761,
"5": 0.8787878751754761
}
},
"person_to_fact": {
"query_count": 84,
"recall_at_k": {
"1": 0.2699546217918396,
"3": 0.6632086038589478,
"5": 0.9249432682991028
},
"all_support_em_at_k": {
"3": 0.380952388048172,
"5": 0.8928571343421936
}
},
"self_to_fact": {
"query_count": 168,
"recall_at_k": {
"1": 0.2738095223903656,
"3": 0.6547619104385376,
"5": 0.9375
},
"all_support_em_at_k": {
"3": 0.3392857015132904,
"5": 0.8809523582458496
}
},
"speaker_independent": {
"query_count": 30,
"recall_at_k": {
"1": 0.2761111259460449,
"3": 0.5186111330986023,
"5": 0.5752777457237244
},
"all_support_em_at_k": {
"3": 0.20000000298023224,
"5": 0.2666666805744171
}
}
},
"temporal_localization": {
"person_to_fact": {
"query_count": 274,
"recall_at_k": {
"1": 0.9187955856323242,
"3": 0.9881386756896973,
"5": 0.9881386756896973
},
"all_support_em_at_k": {
"3": 0.985401451587677,
"5": 0.985401451587677
}
},
"self_to_fact": {
"query_count": 21,
"recall_at_k": {
"1": 0.5,
"3": 0.682539701461792,
"5": 0.7301587462425232
},
"all_support_em_at_k": {
"3": 0.5238094925880432,
"5": 0.5714285969734192
}
},
"speaker_independent": {
"query_count": 424,
"recall_at_k": {
"1": 0.8624213337898254,
"3": 0.9363208413124084,
"5": 0.9536163806915283
},
"all_support_em_at_k": {
"3": 0.9103773832321167,
"5": 0.9292452931404114
}
}
}
},
"by_task": {
"change_comparison": {
"query_count": 470,
"recall_at_k": {
"1": 0.4156028628349304,
"3": 0.8804963827133179,
"5": 0.9608155488967896
},
"all_support_em_at_k": {
"3": 0.785106360912323,
"5": 0.9234042763710022
}
},
"direct_recall": {
"query_count": 1491,
"recall_at_k": {
"1": 0.8416610360145569,
"3": 0.9213054776191711,
"5": 0.9381287693977356
},
"all_support_em_at_k": {
"3": 0.8980550169944763,
"5": 0.9141515493392944
}
},
"historical_state": {
"query_count": 500,
"recall_at_k": {
"1": 0.7860000133514404,
"3": 0.9909999966621399,
"5": 0.9990000128746033
},
"all_support_em_at_k": {
"3": 0.9900000095367432,
"5": 0.9980000257492065
}
},
"latest_state": {
"query_count": 457,
"recall_at_k": {
"1": 0.4124726355075836,
"3": 0.9617067575454712,
"5": 0.9989058971405029
},
"all_support_em_at_k": {
"3": 0.9606127142906189,
"5": 0.9978117942810059
}
},
"multi_hop_composition": {
"query_count": 348,
"recall_at_k": {
"1": 0.30845990777015686,
"3": 0.695320188999176,
"5": 0.9008620381355286
},
"all_support_em_at_k": {
"3": 0.4396551847457886,
"5": 0.8304597735404968
}
},
"temporal_localization": {
"query_count": 719,
"recall_at_k": {
"1": 0.8733193874359131,
"3": 0.9486554861068726,
"5": 0.9602457284927368
},
"all_support_em_at_k": {
"3": 0.92767733335495,
"5": 0.9401947259902954
}
}
},
"semantic_only": {
"query_count": 3985,
"recall_at_k": {
"1": 0.6349337697029114,
"3": 0.9045097827911377,
"5": 0.9561062455177307
},
"all_support_em_at_k": {
"3": 0.8587201833724976,
"5": 0.9324968457221985
},
"by_speaker_query_type": {
"fact_to_person": {
"query_count": 786,
"recall_at_k": {
"1": 0.7658624649047852,
"3": 0.9737535119056702,
"5": 0.982141375541687
},
"all_support_em_at_k": {
"3": 0.9618320465087891,
"5": 0.9694656729698181
}
},
"person_to_fact": {
"query_count": 969,
"recall_at_k": {
"1": 0.6729888916015625,
"3": 0.9353727102279663,
"5": 0.974057674407959
},
"all_support_em_at_k": {
"3": 0.8998968005180359,
"5": 0.9649122953414917
}
},
"self_to_fact": {
"query_count": 509,
"recall_at_k": {
"1": 0.24476097524166107,
"3": 0.6501309275627136,
"5": 0.8966928124427795
},
"all_support_em_at_k": {
"3": 0.45186641812324524,
"5": 0.8251473307609558
}
},
"speaker_independent": {
"query_count": 1721,
"recall_at_k": {
"1": 0.6691070795059204,
"3": 0.9307428002357483,
"5": 0.9516802430152893
},
"all_support_em_at_k": {
"3": 0.9087739586830139,
"5": 0.9291110038757324
}
}
},
"by_task_and_speaker_query_type": {
"change_comparison": {
"fact_to_person": {
"query_count": 74,
"recall_at_k": {
"1": 0.5,
"3": 1.0,
"5": 1.0
},
"all_support_em_at_k": {
"3": 1.0,
"5": 1.0
}
},
"person_to_fact": {
"query_count": 88,
"recall_at_k": {
"1": 0.5,
"3": 0.9545454382896423,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.9090908765792847,
"5": 1.0
}
},
"self_to_fact": {
"query_count": 99,
"recall_at_k": {
"1": 0.1111111119389534,
"3": 0.47979798913002014,
"5": 0.8383838534355164
},
"all_support_em_at_k": {
"3": 0.10101009905338287,
"5": 0.6767676472663879
}
},
"speaker_independent": {
"query_count": 209,
"recall_at_k": {
"1": 0.48963314294815063,
"3": 0.9908294081687927,
"5": 0.9932217597961426
},
"all_support_em_at_k": {
"3": 0.9856459498405457,
"5": 0.9904305934906006
}
}
},
"direct_recall": {
"fact_to_person": {
"query_count": 524,
"recall_at_k": {
"1": 0.9285942316055298,
"3": 0.9726463556289673,
"5": 0.9820291996002197
},
"all_support_em_at_k": {
"3": 0.9599236249923706,
"5": 0.9694656133651733
}
},
"person_to_fact": {
"query_count": 306,
"recall_at_k": {
"1": 0.7990195751190186,
"3": 0.9327342510223389,
"5": 0.9507080912590027
},
"all_support_em_at_k": {
"3": 0.9215686321258545,
"5": 0.9346405267715454
}
},
"self_to_fact": {
"query_count": 57,
"recall_at_k": {
"1": 0.4312865138053894,
"3": 0.6067252159118652,
"5": 0.694444477558136
},
"all_support_em_at_k": {
"3": 0.4385964870452881,
"5": 0.5614035129547119
}
},
"speaker_independent": {
"query_count": 604,
"recall_at_k": {
"1": 0.8083609342575073,
"3": 0.9050771594047546,
"5": 0.9199779629707336
},
"all_support_em_at_k": {
"3": 0.8807947039604187,
"5": 0.8923841118812561
}
}
},
"historical_state": {
"fact_to_person": {
"query_count": 91,
"recall_at_k": {
"1": 0.3956044018268585,
"3": 0.9890109896659851,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.9890109896659851,
"5": 1.0
}
},
"person_to_fact": {
"query_count": 125,
"recall_at_k": {
"1": 0.47200000286102295,
"3": 0.9919999837875366,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.9919999837875366,
"5": 1.0
}
},
"self_to_fact": {
"query_count": 82,
"recall_at_k": {
"1": 0.2195121943950653,
"3": 0.7804877758026123,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.7804877758026123,
"5": 1.0
}
},
"speaker_independent": {
"query_count": 202,
"recall_at_k": {
"1": 0.5990098714828491,
"3": 0.9876237511634827,
"5": 0.9975247383117676
},
"all_support_em_at_k": {
"3": 0.9851484894752502,
"5": 0.9950494766235352
}
}
},
"latest_state": {
"fact_to_person": {
"query_count": 31,
"recall_at_k": {
"1": 0.3870967626571655,
"3": 1.0,
"5": 1.0
},
"all_support_em_at_k": {
"3": 1.0,
"5": 1.0
}
},
"person_to_fact": {
"query_count": 92,
"recall_at_k": {
"1": 0.34239131212234497,
"3": 0.97826087474823,
"5": 1.0
},
"all_support_em_at_k": {
"3": 0.97826087474823,
"5": 1.0
}
},
"self_to_fact": {
"query_count": 82,
"recall_at_k": {
"1": 0.13414634764194489,
"3": 0.6951219439506531,
"5": 0.9634146094322205
},
"all_support_em_at_k": {
"3": 0.6951219439506531,
"5": 0.9634146094322205
}
},
"speaker_independent": {
"query_count": 252,
"recall_at_k": {
"1": 0.2698412835597992,
"3": 0.9384920597076416,
"5": 0.9980158805847168
},
"all_support_em_at_k": {
"3": 0.9365079402923584,
"5": 0.9960317611694336
}
}
},
"multi_hop_composition": {
"fact_to_person": {
"query_count": 66,
"recall_at_k": {
"1": 0.46037158370018005,
"3": 0.9197511076927185,
"5": 0.9299963712692261
},
"all_support_em_at_k": {
"3": 0.8787878751754761,
"5": 0.8787878751754761
}
},
"person_to_fact": {
"query_count": 84,
"recall_at_k": {
"1": 0.25447848439216614,
"3": 0.6393990516662598,
"5": 0.9189909100532532
},
"all_support_em_at_k": {
"3": 0.3333333432674408,
"5": 0.8809523582458496
}
},
"self_to_fact": {
"query_count": 168,
"recall_at_k": {
"1": 0.2946428656578064,
"3": 0.6696428656578064,
"5": 0.9375
},
"all_support_em_at_k": {
"3": 0.369047611951828,
"5": 0.8809523582458496
}
},
"speaker_independent": {
"query_count": 30,
"recall_at_k": {
"1": 0.2761111259460449,
"3": 0.5019444227218628,
"5": 0.5586110949516296
},
"all_support_em_at_k": {
"3": 0.1666666716337204,
"5": 0.23333333432674408
}
}
},
"temporal_localization": {
"person_to_fact": {
"query_count": 274,
"recall_at_k": {
"1": 0.9187955856323242,
"3": 0.9826642274856567,
"5": 0.9881386756896973
},
"all_support_em_at_k": {
"3": 0.9781022071838379,
"5": 0.985401451587677
}
},
"self_to_fact": {
"query_count": 21,
"recall_at_k": {
"1": 0.5,
"3": 0.7301587462425232,
"5": 0.7301587462425232
},
"all_support_em_at_k": {
"3": 0.5714285969734192,
"5": 0.5714285969734192
}
},
"speaker_independent": {
"query_count": 424,
"recall_at_k": {
"1": 0.8577044010162354,
"3": 0.9363208413124084,
"5": 0.9547955989837646
},
"all_support_em_at_k": {
"3": 0.9103773832321167,
"5": 0.9292452931404114
}
}
}
},
"by_task": {
"change_comparison": {
"query_count": 470,
"recall_at_k": {
"1": 0.4134751856327057,
"3": 0.8778368234634399,
"5": 0.9629431962966919
},
"all_support_em_at_k": {
"3": 0.7872340679168701,
"5": 0.9276595711708069
}
},
"direct_recall": {
"query_count": 1491,
"recall_at_k": {
"1": 0.8342834711074829,
"3": 0.9230940937995911,
"5": 0.9394700527191162
},
"all_support_em_at_k": {
"3": 0.9000670909881592,
"5": 0.9154929518699646
}
},
"historical_state": {
"query_count": 500,
"recall_at_k": {
"1": 0.46799999475479126,
"3": 0.9549999833106995,
"5": 0.9990000128746033
},
"all_support_em_at_k": {
"3": 0.9539999961853027,
"5": 0.9980000257492065
}
},
"latest_state": {
"query_count": 457,
"recall_at_k": {
"1": 0.26805251836776733,
"3": 0.9070022106170654,
"5": 0.9923413395881653
},
"all_support_em_at_k": {
"3": 0.9059081077575684,
"5": 0.9912472367286682
}
},
"multi_hop_composition": {
"query_count": 348,
"recall_at_k": {
"1": 0.31478172540664673,
"3": 0.6953202486038208,
"5": 0.8989464044570923
},
"all_support_em_at_k": {
"3": 0.4396551847457886,
"5": 0.8247126340866089
}
},
"temporal_localization": {
"query_count": 719,
"recall_at_k": {
"1": 0.8705376982688904,
"3": 0.9479600787162781,
"5": 0.9609411358833313
},
"all_support_em_at_k": {
"3": 0.926286518573761,
"5": 0.9401947259902954
}
}
}
},
"skipped_control_query_count": 0,
"score_analysis": {
"overall": {
"rag_query_count": 3985,
"control_query_count": 111,
"gold_score": {
"count": 5141,
"mean": 0.7154593990645001,
"std": 0.17573212749321732,
"min": -0.12303857505321503,
"p05": 0.3636300265789032,
"p25": 0.6405549645423889,
"median": 0.7480038404464722,
"p75": 0.8282420039176941,
"p95": 0.9467622637748718,
"max": 1.1032496690750122
},
"best_gold_score": {
"count": 3985,
"mean": 0.7544670392907549,
"std": 0.14403546974973816,
"min": 0.02280385047197342,
"p05": 0.48611941933631897,
"p25": 0.6786033511161804,
"median": 0.7758575081825256,
"p75": 0.845791757106781,
"p95": 0.9638454675674438,
"max": 1.1032496690750122
},
"rag_max_candidate_score": {
"count": 3985,
"mean": 0.7627401347104997,
"std": 0.13235941487299727,
"min": 0.21979452669620514,
"p05": 0.5201244235038758,
"p25": 0.6830075979232788,
"median": 0.7791073322296143,
"p75": 0.8487366437911987,
"p95": 0.9649398803710937,
"max": 1.1042900085449219
},
"control_max_candidate_score": {
"count": 111,
"mean": 0.3775643181928375,
"std": 0.14569058322524672,
"min": 0.018502261489629745,
"p05": 0.14916608482599258,
"p25": 0.2863771915435791,
"median": 0.3824880123138428,
"p75": 0.4608069360256195,
"p95": 0.6159006059169769,
"max": 0.8022838234901428
},
"max_non_gold_score": {
"count": 3985,
"mean": 0.6269493476958616,
"std": 0.179593426300524,
"min": 0.08614442497491837,
"p05": 0.34568926095962527,
"p25": 0.49576282501220703,
"median": 0.61491858959198,
"p75": 0.7543556690216064,
"p95": 0.9488252282142638,
"max": 1.1042900085449219
},
"query_temporal_norm": {
"count": 4096,
"mean": 0.3116771188815619,
"std": 0.12893400789015785,
"min": 0.12013464421033859,
"p05": 0.1599961407482624,
"p25": 0.203378327190876,
"median": 0.2639659196138382,
"p75": 0.4172542244195938,
"p95": 0.5408022552728653,
"max": 0.5694292783737183
},
"best_gold_margin": {
"count": 3985,
"mean": 0.12751769159489323,
"std": 0.14761390159917398,
"min": -0.41839342564344406,
"p05": -0.050421607494354245,
"p25": 0.0031561851501464844,
"median": 0.0983622670173645,
"p75": 0.2348206639289856,
"p95": 0.3919700920581816,
"max": 0.7321364879608154
},
"weakest_gold_margin": {
"count": 3985,
"mean": 0.10261004417278738,
"std": 0.16404433255357592,
"min": -0.6536026895046234,
"p05": -0.13799275159835814,
"p25": -0.0018881559371948242,
"median": 0.08711892366409302,
"p75": 0.2224733829498291,
"p95": 0.3672234594821928,
"max": 0.6410713195800781
},
"top_score_best_balanced_threshold": {
"threshold": 0.5664574205875397,
"balanced_accuracy": 0.911291215933625,
"true_positive_rate": 0.91267252195734,
"false_positive_rate": 0.09009009009009009
},
"gold_score_best_balanced_threshold": {
"threshold": 0.5665663480758667,
"balanced_accuracy": 0.9048922196977405,
"true_positive_rate": 0.8998745294855709,
"false_positive_rate": 0.09009009009009009
}
},
"by_task": {
"change_comparison": {
"rag_query_count": 470,
"control_query_count": 0,
"gold_score": {
"count": 942,
"mean": 0.7326939189905447,
"std": 0.10824093478227866,
"min": 0.07003629952669144,
"p05": 0.5355867475271225,
"p25": 0.6708965450525284,
"median": 0.7509642839431763,
"p75": 0.8066658228635788,
"p95": 0.8816887825727461,
"max": 0.9440944790840149
},
"best_gold_score": {
"count": 470,
"mean": 0.7679149062075513,
"std": 0.09213641523962647,
"min": 0.3496818542480469,
"p05": 0.6097596526145935,
"p25": 0.7119392454624176,
"median": 0.7841155230998993,
"p75": 0.8302914351224899,
"p95": 0.8930411905050277,
"max": 0.9440944790840149
},
"rag_max_candidate_score": {
"count": 470,
"mean": 0.7722497434058088,
"std": 0.09157903790860208,
"min": 0.3496818542480469,
"p05": 0.6142909288406372,
"p25": 0.7141461968421936,
"median": 0.7864619791507721,
"p75": 0.8346060365438461,
"p95": 0.8946467816829681,
"max": 0.9440944790840149
},
"control_max_candidate_score": null,
"max_non_gold_score": {
"count": 470,
"mean": 0.547240946980867,
"std": 0.19383881040767714,
"min": 0.08614442497491837,
"p05": 0.2577131509780884,
"p25": 0.3841584548354149,
"median": 0.5354843437671661,
"p75": 0.7122959047555923,
"p95": 0.843116596341133,
"max": 0.9219087958335876
},
"query_temporal_norm": {
"count": 470,
"mean": 0.40276982140667894,
"std": 0.03980443605509354,
"min": 0.17388875782489777,
"p05": 0.328983274102211,
"p25": 0.38821743428707123,
"median": 0.40812453627586365,
"p75": 0.424331434071064,
"p95": 0.4521392062306403,
"max": 0.5628159642219543
},
"best_gold_margin": {
"count": 470,
"mean": 0.22067395922668437,
"std": 0.1824797004933215,
"min": -0.09542953968048096,
"p05": -0.03293478488922119,
"p25": 0.06616732478141785,
"median": 0.2181701958179474,
"p75": 0.3659444600343704,
"p95": 0.51416195705533,
"max": 0.7321364879608154
},
"weakest_gold_margin": {
"count": 470,
"mean": 0.15360000450560388,
"std": 0.1851890519021113,
"min": -0.4024420902132988,
"p05": -0.10997549891471863,
"p25": 0.003862246870994568,
"median": 0.14909625053405762,
"p75": 0.3048042953014374,
"p95": 0.45087446719408025,
"max": 0.6410713195800781
},
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"current_only_control": {
"rag_query_count": 0,
"control_query_count": 110,
"gold_score": null,
"best_gold_score": null,
"rag_max_candidate_score": null,
"control_max_candidate_score": {
"count": 110,
"mean": 0.3769120380790396,
"std": 0.14618987745617867,
"min": 0.018502261489629745,
"p05": 0.14905614629387856,
"p25": 0.2858238220214844,
"median": 0.3782581090927124,
"p75": 0.46130822598934174,
"p95": 0.6162560909986495,
"max": 0.8022838234901428
},
"max_non_gold_score": null,
"query_temporal_norm": {
"count": 110,
"mean": 0.26989374445243314,
"std": 0.0766767531184094,
"min": 0.13687971234321594,
"p05": 0.16108160167932511,
"p25": 0.215445127338171,
"median": 0.2581220120191574,
"p75": 0.3229278400540352,
"p95": 0.4131780833005905,
"max": 0.4925289750099182
},
"best_gold_margin": null,
"weakest_gold_margin": null,
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"direct_recall": {
"rag_query_count": 1491,
"control_query_count": 0,
"gold_score": {
"count": 1669,
"mean": 0.6795029017735003,
"std": 0.1983251415999637,
"min": -0.12303857505321503,
"p05": 0.2784222841262817,
"p25": 0.5722606182098389,
"median": 0.7289072275161743,
"p75": 0.8250808119773865,
"p95": 0.9214394807815551,
"max": 1.063254952430725
},
"best_gold_score": {
"count": 1491,
"mean": 0.7199413462833699,
"std": 0.15911103886386538,
"min": 0.02280385047197342,
"p05": 0.4366766959428787,
"p25": 0.6310526430606842,
"median": 0.7523093819618225,
"p75": 0.8343758881092072,
"p95": 0.9265364408493042,
"max": 1.063254952430725
},
"rag_max_candidate_score": {
"count": 1491,
"mean": 0.7315587912786734,
"std": 0.1390575200032724,
"min": 0.21979452669620514,
"p05": 0.4833293706178665,
"p25": 0.6411759853363037,
"median": 0.7542001605033875,
"p75": 0.8348446488380432,
"p95": 0.9265364408493042,
"max": 1.063254952430725
},
"control_max_candidate_score": null,
"max_non_gold_score": {
"count": 1491,
"mean": 0.5671377542393068,
"std": 0.13058292145303427,
"min": 0.08767721056938171,
"p05": 0.35685819387435913,
"p25": 0.48237597942352295,
"median": 0.5671817064285278,
"p75": 0.6538146734237671,
"p95": 0.7913401126861572,
"max": 0.9612478017807007
},
"query_temporal_norm": {
"count": 1491,
"mean": 0.2383564575847006,
"std": 0.08033840921935426,
"min": 0.12013464421033859,
"p05": 0.1468081772327423,
"p25": 0.18328498303890228,
"median": 0.21576981246471405,
"p75": 0.2750905752182007,
"p95": 0.39991748332977295,
"max": 0.5644786357879639
},
"best_gold_margin": {
"count": 1491,
"mean": 0.1528035920440631,
"std": 0.14736356526555025,
"min": -0.41839342564344406,
"p05": -0.08735679090023041,
"p25": 0.058132946491241455,
"median": 0.15042370557785034,
"p75": 0.2618226259946823,
"p95": 0.3783687353134155,
"max": 0.6177643090486526
},
"weakest_gold_margin": {
"count": 1491,
"mean": 0.13596027125220303,
"std": 0.17475061101661918,
"min": -0.6536026895046234,
"p05": -0.1832170933485031,
"p25": 0.04592122137546539,
"median": 0.14661788940429688,
"p75": 0.26139336824417114,
"p95": 0.3783687353134155,
"max": 0.6177643090486526
},
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"historical_state": {
"rag_query_count": 500,
"control_query_count": 0,
"gold_score": {
"count": 502,
"mean": 0.7761994309396858,
"std": 0.10471581264550778,
"min": 0.16526609659194946,
"p05": 0.5885683983564377,
"p25": 0.7164510041475296,
"median": 0.7957707047462463,
"p75": 0.8472662419080734,
"p95": 0.9122402310371399,
"max": 0.9831515550613403
},
"best_gold_score": {
"count": 500,
"mean": 0.777970435500145,
"std": 0.10054288277351174,
"min": 0.40203285217285156,
"p05": 0.5916313827037811,
"p25": 0.7174811512231827,
"median": 0.7958914339542389,
"p75": 0.8473147451877594,
"p95": 0.9122692972421645,
"max": 0.9831515550613403
},
"rag_max_candidate_score": {
"count": 500,
"mean": 0.7819046204090119,
"std": 0.09902156513217696,
"min": 0.40203285217285156,
"p05": 0.6030461072921753,
"p25": 0.7233852297067642,
"median": 0.7978901863098145,
"p75": 0.8500061184167862,
"p95": 0.912781223654747,
"max": 0.9831515550613403
},
"control_max_candidate_score": null,
"max_non_gold_score": {
"count": 500,
"mean": 0.6704996504485607,
"std": 0.14438779622177209,
"min": 0.17858107388019562,
"p05": 0.4236251160502434,
"p25": 0.5728582739830017,
"median": 0.6769407391548157,
"p75": 0.7875670939683914,
"p95": 0.8805044949054718,
"max": 0.9633103013038635
},
"query_temporal_norm": {
"count": 500,
"mean": 0.4810723942518234,
"std": 0.0908062463914881,
"min": 0.16505520045757294,
"p05": 0.2751061350107193,
"p25": 0.44165537506341934,
"median": 0.5179827213287354,
"p75": 0.54570671916008,
"p95": 0.5624057739973068,
"max": 0.5694292783737183
},
"best_gold_margin": {
"count": 500,
"mean": 0.10747078505158425,
"std": 0.11929718685672921,
"min": -0.17465877532958984,
"p05": -0.029424968361854553,
"p25": 0.001565694808959961,
"median": 0.08554309606552124,
"p75": 0.17575012892484665,
"p95": 0.3152986243367195,
"max": 0.5076251327991486
},
"weakest_gold_margin": {
"count": 500,
"mean": 0.1064475021660328,
"std": 0.12140862200311647,
"min": -0.3940350413322449,
"p05": -0.03014567494392395,
"p25": 0.0015095025300979614,
"median": 0.08444157242774963,
"p75": 0.17575012892484665,
"p95": 0.3152986243367195,
"max": 0.5076251327991486
},
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"latest_state": {
"rag_query_count": 457,
"control_query_count": 0,
"gold_score": {
"count": 460,
"mean": 0.9076748492925063,
"std": 0.11461868176356446,
"min": 0.24560272693634033,
"p05": 0.6936595916748047,
"p25": 0.8535026013851166,
"median": 0.9299749135971069,
"p75": 0.9843438863754272,
"p95": 1.0409013211727143,
"max": 1.1032496690750122
},
"best_gold_score": {
"count": 457,
"mean": 0.9106951447884416,
"std": 0.1083083226667221,
"min": 0.24560272693634033,
"p05": 0.7078387379646301,
"p25": 0.8549481630325317,
"median": 0.9306370615959167,
"p75": 0.9854380488395691,
"p95": 1.0409518003463745,
"max": 1.1032496690750122
},
"rag_max_candidate_score": {
"count": 457,
"mean": 0.9170270060684279,
"std": 0.10489952671179081,
"min": 0.34878870844841003,
"p05": 0.718699598312378,
"p25": 0.8615692853927612,
"median": 0.9357645511627197,
"p75": 0.9880915880203247,
"p95": 1.0481631994247436,
"max": 1.1042900085449219
},
"control_max_candidate_score": null,
"max_non_gold_score": {
"count": 457,
"mean": 0.9147816273002708,
"std": 0.10795329594168684,
"min": 0.3380759358406067,
"p05": 0.718699598312378,
"p25": 0.8614855408668518,
"median": 0.935660183429718,
"p75": 0.9872651696205139,
"p95": 1.0466254949569702,
"max": 1.1042900085449219
},
"query_temporal_norm": {
"count": 457,
"mean": 0.48360304671382487,
"std": 0.0791067935911107,
"min": 0.1427229642868042,
"p05": 0.30160648822784425,
"p25": 0.46912088990211487,
"median": 0.513687789440155,
"p75": 0.5321373343467712,
"p95": 0.545603621006012,
"max": 0.5621570348739624
},
"best_gold_margin": {
"count": 457,
"mean": -0.004086482511829234,
"std": 0.01985870933325241,
"min": -0.14556145668029785,
"p05": -0.04004968404769897,
"p25": -0.00482553243637085,
"median": -0.0011156201362609863,
"p75": 0.001955687999725342,
"p95": 0.008001494407653806,
"max": 0.18877702951431274
},
"weakest_gold_margin": {
"count": 457,
"mean": -0.004729862351386239,
"std": 0.016954887432432158,
"min": -0.14556145668029785,
"p05": -0.04004968404769897,
"p25": -0.004851281642913818,
"median": -0.0011196136474609375,
"p75": 0.0019491314888000488,
"p95": 0.007873499393463134,
"max": 0.09029287099838257
},
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"multi_hop_composition": {
"rag_query_count": 348,
"control_query_count": 0,
"gold_score": {
"count": 773,
"mean": 0.6477225914014381,
"std": 0.1696077516567076,
"min": -0.10431600362062454,
"p05": 0.28633753657341005,
"p25": 0.5937712788581848,
"median": 0.6980624198913574,
"p75": 0.7609883546829224,
"p95": 0.8197827219963074,
"max": 0.8692178726196289
},
"best_gold_score": {
"count": 348,
"mean": 0.7196337887267957,
"std": 0.09670981163335679,
"min": 0.35165759921073914,
"p05": 0.5244349539279938,
"p25": 0.6749561429023743,
"median": 0.7393551766872406,
"p75": 0.7847351729869843,
"p95": 0.8334286898374558,
"max": 0.8692178726196289
},
"rag_max_candidate_score": {
"count": 348,
"mean": 0.7297802659115572,
"std": 0.0901932536288958,
"min": 0.3799307644367218,
"p05": 0.5428706228733062,
"p25": 0.684001088142395,
"median": 0.751622200012207,
"p75": 0.7945983111858368,
"p95": 0.8368684709072113,
"max": 0.8703596591949463
},
"control_max_candidate_score": null,
"max_non_gold_score": {
"count": 348,
"mean": 0.6496828974535068,
"std": 0.1525620029439476,
"min": 0.11702252179384232,
"p05": 0.3462520882487297,
"p25": 0.5447135716676712,
"median": 0.7023131251335144,
"p75": 0.7659444659948349,
"p95": 0.8243616819381714,
"max": 0.8703596591949463
},
"query_temporal_norm": {
"count": 348,
"mean": 0.2268703466859357,
"std": 0.031543356431370365,
"min": 0.12830781936645508,
"p05": 0.18497471064329146,
"p25": 0.20330190658569336,
"median": 0.22173745930194855,
"p75": 0.24691196158528328,
"p95": 0.2863227754831314,
"max": 0.3312945067882538
},
"best_gold_margin": {
"count": 348,
"mean": 0.06995089127328889,
"std": 0.13036276721273612,
"min": -0.1642531156539917,
"p05": -0.05332456827163696,
"p25": -0.011286526918411255,
"median": 0.0241546630859375,
"p75": 0.09257993847131729,
"p95": 0.3807756379246711,
"max": 0.4885251224040985
},
"weakest_gold_margin": {
"count": 348,
"mean": -0.015732982335076934,
"std": 0.15913426710387032,
"min": -0.5917035602033138,
"p05": -0.2659562837332487,
"p25": -0.07573521137237549,
"median": -0.030487626791000366,
"p75": 0.006903454661369324,
"p95": 0.3199440449476241,
"max": 0.454546719789505
},
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"recent_context_control": {
"rag_query_count": 0,
"control_query_count": 1,
"gold_score": null,
"best_gold_score": null,
"rag_max_candidate_score": null,
"control_max_candidate_score": {
"count": 1,
"mean": 0.4493151307106018,
"std": 0.0,
"min": 0.4493151307106018,
"p05": 0.4493151307106018,
"p25": 0.4493151307106018,
"median": 0.4493151307106018,
"p75": 0.4493151307106018,
"p95": 0.4493151307106018,
"max": 0.4493151307106018
},
"max_non_gold_score": null,
"query_temporal_norm": {
"count": 1,
"mean": 0.1407139152288437,
"std": 0.0,
"min": 0.1407139152288437,
"p05": 0.1407139152288437,
"p25": 0.1407139152288437,
"median": 0.1407139152288437,
"p75": 0.1407139152288437,
"p95": 0.1407139152288437,
"max": 0.1407139152288437
},
"best_gold_margin": null,
"weakest_gold_margin": null,
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
},
"temporal_localization": {
"rag_query_count": 719,
"control_query_count": 0,
"gold_score": {
"count": 795,
"mean": 0.6868133933106203,
"std": 0.16939127613428318,
"min": -0.10412678867578506,
"p05": 0.3516594350337982,
"p25": 0.6090777516365051,
"median": 0.7244535684585571,
"p75": 0.8079478740692139,
"p95": 0.8911272704601287,
"max": 1.0223890542984009
},
"best_gold_score": {
"count": 719,
"mean": 0.7184883740643798,
"std": 0.13249211277583256,
"min": 0.12042880803346634,
"p05": 0.4689896821975708,
"p25": 0.6429046988487244,
"median": 0.7374533414840698,
"p75": 0.8162335455417633,
"p95": 0.8952352583408356,
"max": 1.0223890542984009
},
"rag_max_candidate_score": {
"count": 719,
"mean": 0.7257450836009873,
"std": 0.12004393947778003,
"min": 0.3028644323348999,
"p05": 0.4953923970460892,
"p25": 0.6520695090293884,
"median": 0.7405902147293091,
"p75": 0.8165587782859802,
"p95": 0.8952352583408356,
"max": 1.0223890542984009
},
"control_max_candidate_score": null,
"max_non_gold_score": {
"count": 719,
"mean": 0.5788495642585118,
"std": 0.13251074801721197,
"min": 0.24814382195472717,
"p05": 0.36748895347118377,
"p25": 0.4737658351659775,
"median": 0.5815503597259521,
"p75": 0.6805538833141327,
"p95": 0.803080928325653,
"max": 0.9469888210296631
},
"query_temporal_norm": {
"count": 719,
"mean": 0.22477814839098484,
"std": 0.05035853463905679,
"min": 0.13936284184455872,
"p05": 0.16335002928972245,
"p25": 0.18785642832517624,
"median": 0.2102562040090561,
"p75": 0.2554599493741989,
"p95": 0.3239457279443741,
"max": 0.49927112460136414
},
"best_gold_margin": {
"count": 719,
"mean": 0.13963880980586807,
"std": 0.12473303118320828,
"min": -0.3510739803314209,
"p05": -0.04535098373889923,
"p25": 0.06478071212768555,
"median": 0.11660575866699219,
"p75": 0.23043745756149292,
"p95": 0.35062283277511597,
"max": 0.5024953782558441
},
"weakest_gold_margin": {
"count": 719,
"mean": 0.12295567919063008,
"std": 0.14893883801184157,
"min": -0.5288597047328949,
"p05": -0.1383767902851105,
"p25": 0.054179370403289795,
"median": 0.11057806015014648,
"p75": 0.22584694623947144,
"p95": 0.348681515455246,
"max": 0.5024953782558441
},
"top_score_best_balanced_threshold": null,
"gold_score_best_balanced_threshold": null
}
}
}
}
}
{
"checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal/retriever.pt",
"use_retrieval_rope": false,
"time_embedding_size": 128,
"memory_temporal_positions_shuffled": false,
"temporal_wording_counts": {
"train:after": 1,
"train:before": 98,
"train:comparison": 903,
"train:earliest": 783,
"train:latest": 748,
"train:session": 1431,
"train:unmatched": 390,
"validation:before": 70,
"validation:comparison": 414,
"validation:earliest": 354,
"validation:latest": 358,
"validation:session": 789,
"validation:unmatched": 161
},
"task_distribution": {
"all_complete_samples": {
"sample_count": 6168,
"rag_query_count": 10985,
"control_query_count": 111,
"by_task": {
"change_comparison": {
"rag_query_count": 1480,
"control_query_count": 0
},
"current_only_control": {
"rag_query_count": 0,
"control_query_count": 110
},
"direct_recall": {
"rag_query_count": 2814,
"control_query_count": 0
},
"historical_state": {
"rag_query_count": 1508,
"control_query_count": 0
},
"latest_state": {
"rag_query_count": 1470,
"control_query_count": 0
},
"multi_hop_composition": {
"rag_query_count": 1671,
"control_query_count": 0
},
"recent_context_control": {
"rag_query_count": 0,
"control_query_count": 1
},
"temporal_localization": {
"rag_query_count": 2042,
"control_query_count": 0
}
}
},
"train_split": {
"sample_count": 5014,
"rag_query_count": 7000,
"control_query_count": 0,
"by_task": {
"change_comparison": {
"rag_query_count": 1010,
"control_query_count": 0
},
"direct_recall": {
"rag_query_count": 1323,
"control_query_count": 0
},
"historical_state": {
"rag_query_count": 1008,
"control_query_count": 0
},
"latest_state": {
"rag_query_count": 1013,
"control_query_count": 0
},
"multi_hop_composition": {
"rag_query_count": 1323,
"control_query_count": 0
},
"temporal_localization": {
"rag_query_count": 1323,
"control_query_count": 0
}
}
},
"validation_split": {
"sample_count": 1154,
"rag_query_count": 3985,
"control_query_count": 111,
"by_task": {
"change_comparison": {
"rag_query_count": 470,
"control_query_count": 0
},