| { |
| "model": "devtaji/recICL", |
| "checkpoint_sha256": "661fd3415a0b8f2d727898006e6314dcd273e15f59da3383668f7bc1a3997a3a", |
| "split": "test", |
| "protocol": { |
| "split": "user-level 70/10/20, seed 42; at most 10,000 users sampled into the test split, then users with a single interaction dropped", |
| "target": "each test user's last item; input = up to 14 preceding items", |
| "context": "8 sequences from train-split users (retrieval as in 'inference')", |
| "batch_size": 1, |
| "ranking": "full item catalog, no candidate sampling, repeated items allowed", |
| "metrics": [ |
| "HR@10", |
| "MRR@10" |
| ], |
| "hardware": "NVIDIA L4, fp32, TF32 off" |
| }, |
| "baselines": { |
| "embknn_balanced": "score = dot(mean of the history item embeddings, item embedding); its numbers are close to the EmbKNN column of the RecPFN paper's Table 2 (within about 10% on Arts, Games, Movies, Pantry, Software)", |
| "embknn_last": "score = dot(last history item embedding, item embedding); the EmbKNN that the RecPFN paper's text describes" |
| }, |
| "datasets_source": "Amazon Reviews 2018 (McAuley et al.) categories, rebuilt from the raw ratings + metadata: Appliances, Arts_Crafts_and_Sewing, Video_Games, Movies_and_TV, Prime_Pantry, Industrial_and_Scientific, Software", |
| "paired_differences_note": "model minus baseline, paired bootstrap over test users (2,000 resamples), 95% CI; McNemar exact p on HR@10 hits", |
| "datasets": { |
| "Appliances": { |
| "n_test_users_evaluated": 145, |
| "n_test_users_in_split": 145, |
| "flag": "evaluation data differs from the RecPFN paper's (undisclosed filtering)", |
| "recicl_v0": { |
| "hr@3": 0.06206896551724138, |
| "hr@5": 0.06896551724137931, |
| "hr@10": 0.10344827586206896, |
| "mrr@3": 0.04827586206896552, |
| "mrr@5": 0.04965517243434643, |
| "mrr@10": 0.05434865910431434, |
| "loss": 9.28750396761401 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.027586206896551724, |
| "hr@5": 0.041379310344827586, |
| "hr@10": 0.06206896551724138, |
| "mrr@3": 0.013793103448275862, |
| "mrr@5": 0.016551724179037688, |
| "mrr@10": 0.01906951295918432, |
| "loss": 10.228556942117626 |
| }, |
| "embknn_last": { |
| "hr@3": 0.09655172413793103, |
| "hr@5": 0.1103448275862069, |
| "hr@10": 0.1310344827586207, |
| "mrr@3": 0.05862068986070567, |
| "mrr@5": 0.062068965722774636, |
| "mrr@10": 0.06475095810561345, |
| "loss": 10.193267019863786 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": -0.027586206896551738, |
| "d_hr@10_ci": [ |
| -0.06206896551724139, |
| 0.0 |
| ], |
| "d_mrr@10": -0.010402298850574712, |
| "d_mrr@10_ci": [ |
| -0.023873084291187747, |
| 0.0023036398467432624 |
| ], |
| "mcnemar_p": 0.21875000000000014 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.041379310344827586, |
| "d_hr@10_ci": [ |
| 0.0, |
| 0.08275862068965519 |
| ], |
| "d_mrr@10": 0.035279146141215106, |
| "d_mrr@10_ci": [ |
| 0.007594074986316366, |
| 0.06752230432402839 |
| ], |
| "mcnemar_p": 0.109375 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:12:17Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.05517241379310345, |
| "mrr@10": 0.020000000092489967, |
| "pct_change_hr@10": -46.666666666666664, |
| "pct_change_hr@10_ci95": [ |
| -75.0, |
| -14.261904761904896 |
| ], |
| "pct_change_mrr@10": -63.200563873888996, |
| "pct_change_mrr@10_ci95": [ |
| -91.32381136817679, |
| -33.380598313527344 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.041379310344827586, |
| "mrr@10": 0.03103448275862069, |
| "pct_change_hr@10": -60.0, |
| "pct_change_hr@10_ci95": [ |
| -85.71428571428572, |
| -33.333333333333336 |
| ], |
| "pct_change_mrr@10": -42.897426964933, |
| "pct_change_mrr@10_ci95": [ |
| -81.16710629577956, |
| 6.011155432140703 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| }, |
| "Arts": { |
| "n_test_users_evaluated": 2997, |
| "n_test_users_in_split": 10000, |
| "flag": null, |
| "recicl_v0": { |
| "hr@3": 0.1871871871871872, |
| "hr@5": 0.20186853520186854, |
| "hr@10": 0.22055388722055388, |
| "mrr@3": 0.15170726306366053, |
| "mrr@5": 0.15502724973786303, |
| "mrr@10": 0.15758139124245177, |
| "loss": 10.023291922466811 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.16049382716049382, |
| "hr@5": 0.1828495161828495, |
| "hr@10": 0.20153486820153488, |
| "mrr@3": 0.131854076474199, |
| "mrr@5": 0.1367589814198745, |
| "mrr@10": 0.1393652915019849, |
| "loss": 12.450908683481558 |
| }, |
| "embknn_last": { |
| "hr@3": 0.17450784117450785, |
| "hr@5": 0.19953286619953287, |
| "hr@10": 0.21788455121788455, |
| "mrr@3": 0.14547881232447174, |
| "mrr@5": 0.15100100122374777, |
| "mrr@10": 0.153521643620274, |
| "loss": 12.435229098116672 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": 0.0026693360026693314, |
| "d_hr@10_ci": [ |
| -0.003336670003336678, |
| 0.00934267600934266 |
| ], |
| "d_mrr@10": 0.004059747578266104, |
| "d_mrr@10_ci": [ |
| 0.0016380765686321512, |
| 0.006624263152040991 |
| ], |
| "mcnemar_p": 0.46570744577306067 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.019019019019019007, |
| "d_hr@10_ci": [ |
| 0.01167834501167836, |
| 0.026359693026359682 |
| ], |
| "d_mrr@10": 0.018216099697581145, |
| "d_mrr@10_ci": [ |
| 0.014113558002446853, |
| 0.022250213176138966 |
| ], |
| "mcnemar_p": 4.3609050114828154e-07 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:15:00Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.18618618618618618, |
| "mrr@10": 0.12968577070408438, |
| "pct_change_hr@10": -15.582450832072613, |
| "pct_change_hr@10_ci95": [ |
| -19.082868467410457, |
| -12.157674772036467 |
| ], |
| "pct_change_mrr@10": -17.702357060325546, |
| "pct_change_mrr@10_ci95": [ |
| -20.833086015350958, |
| -14.734061825197962 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.08541875208541876, |
| "mrr@10": 0.048784234107017994, |
| "pct_change_hr@10": -61.27080181543116, |
| "pct_change_hr@10_ci95": [ |
| -64.90455212922173, |
| -57.557142362952874 |
| ], |
| "pct_change_mrr@10": -69.0418813272441, |
| "pct_change_mrr@10_ci95": [ |
| -72.615949611093, |
| -65.57546376581101 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| }, |
| "Games": { |
| "n_test_users_evaluated": 10000, |
| "n_test_users_in_split": 10000, |
| "flag": null, |
| "recicl_v0": { |
| "hr@3": 0.0513, |
| "hr@5": 0.0588, |
| "hr@10": 0.0722, |
| "mrr@3": 0.04260000007748604, |
| "mrr@5": 0.044295000088214874, |
| "mrr@10": 0.046121071556210515, |
| "loss": 10.213145974993706 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.02, |
| "hr@5": 0.0303, |
| "hr@10": 0.0479, |
| "mrr@3": 0.011700000071525574, |
| "mrr@5": 0.014055000084638596, |
| "mrr@10": 0.01636996045857668, |
| "loss": 11.093444060993194 |
| }, |
| "embknn_last": { |
| "hr@3": 0.0506, |
| "hr@5": 0.0585, |
| "hr@10": 0.0736, |
| "mrr@3": 0.03945000005960465, |
| "mrr@5": 0.04121000007241964, |
| "mrr@10": 0.04316595249548554, |
| "loss": 11.0686561044693 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": -0.0013999999999999985, |
| "d_hr@10_ci": [ |
| -0.004699999999999996, |
| 0.0020000000000000018 |
| ], |
| "d_mrr@10": 0.002955119047619019, |
| "d_mrr@10_ci": [ |
| 0.00160891765873009, |
| 0.004420426587301565 |
| ], |
| "mcnemar_p": 0.43727338509353675 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.024300000000000002, |
| "d_hr@10_ci": [ |
| 0.019600000000000006, |
| 0.0288 |
| ], |
| "d_mrr@10": 0.029751111111111096, |
| "d_mrr@10_ci": [ |
| 0.026522191468253967, |
| 0.03297455158730153 |
| ], |
| "mcnemar_p": 1.5319468898317913e-25 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:16:13Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.0716, |
| "mrr@10": 0.03594726204872131, |
| "pct_change_hr@10": -0.8310249307479256, |
| "pct_change_hr@10_ci95": [ |
| -5.364047361136176, |
| 3.841910853289687 |
| ], |
| "pct_change_mrr@10": -22.058918329965017, |
| "pct_change_mrr@10_ci95": [ |
| -25.54905745074236, |
| -18.56859102137774 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.0119, |
| "mrr@10": 0.004851468288898468, |
| "pct_change_hr@10": -83.5180055401662, |
| "pct_change_hr@10_ci95": [ |
| -86.22530832926526, |
| -80.73741871966297 |
| ], |
| "pct_change_mrr@10": -89.4810156720108, |
| "pct_change_mrr@10_ci95": [ |
| -91.75046365557786, |
| -87.02863863527669 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| }, |
| "Movies": { |
| "n_test_users_evaluated": 10000, |
| "n_test_users_in_split": 10000, |
| "flag": null, |
| "recicl_v0": { |
| "hr@3": 0.0951, |
| "hr@5": 0.1096, |
| "hr@10": 0.1237, |
| "mrr@3": 0.06308333349823951, |
| "mrr@5": 0.06639833351671695, |
| "mrr@10": 0.0683068653061986, |
| "loss": 10.933070637273788 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.0335, |
| "hr@5": 0.0491, |
| "hr@10": 0.0695, |
| "mrr@3": 0.018566666802763938, |
| "mrr@5": 0.02212666682302952, |
| "mrr@10": 0.02482007957994938, |
| "loss": 12.046554122447967 |
| }, |
| "embknn_last": { |
| "hr@3": 0.0778, |
| "hr@5": 0.1049, |
| "hr@10": 0.1221, |
| "mrr@3": 0.05258333346545696, |
| "mrr@5": 0.058868333494663236, |
| "mrr@10": 0.061181428787857296, |
| "loss": 12.016274190807342 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": 0.0016000000000000042, |
| "d_hr@10_ci": [ |
| -0.0011999999999999932, |
| 0.00449999999999999 |
| ], |
| "d_mrr@10": 0.0071254365079365944, |
| "d_mrr@10_ci": [ |
| 0.005697232142857268, |
| 0.008514329365079398 |
| ], |
| "mcnemar_p": 0.3029036319209554 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.0542, |
| "d_hr@10_ci": [ |
| 0.04870000000000001, |
| 0.05929999999999999 |
| ], |
| "d_mrr@10": 0.04348678571428581, |
| "d_mrr@10_ci": [ |
| 0.0402409722222223, |
| 0.046700817460317574 |
| ], |
| "mcnemar_p": 1.4361709413897885e-91 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:17:56Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.102, |
| "mrr@10": 0.04952440501749516, |
| "pct_change_hr@10": -17.542441390460805, |
| "pct_change_hr@10_ci95": [ |
| -20.496807702572312, |
| -14.590643244107216 |
| ], |
| "pct_change_mrr@10": -27.497177925685023, |
| "pct_change_mrr@10_ci95": [ |
| -30.157991463516414, |
| -24.89154990329349 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.0179, |
| "mrr@10": 0.010642261940985918, |
| "pct_change_hr@10": -85.52950687146321, |
| "pct_change_hr@10_ci95": [ |
| -87.47939468293607, |
| -83.4124518447913 |
| ], |
| "pct_change_mrr@10": -84.419923395284, |
| "pct_change_mrr@10_ci95": [ |
| -86.68075954394435, |
| -81.80099004889513 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| }, |
| "Pantry": { |
| "n_test_users_evaluated": 3106, |
| "n_test_users_in_split": 10000, |
| "flag": null, |
| "recicl_v0": { |
| "hr@3": 0.2063747585318738, |
| "hr@5": 0.2169993560849968, |
| "hr@10": 0.2347070186735351, |
| "mrr@3": 0.190759819809345, |
| "mrr@5": 0.19320669683405298, |
| "mrr@10": 0.19541504426456927, |
| "loss": 7.7768060520089834 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.19639407598197037, |
| "hr@5": 0.20927237604636187, |
| "hr@10": 0.23148744365743723, |
| "mrr@3": 0.1807791372882268, |
| "mrr@5": 0.18370895057014702, |
| "mrr@10": 0.18661793887720748, |
| "loss": 9.150342638233134 |
| }, |
| "embknn_last": { |
| "hr@3": 0.20573084352865423, |
| "hr@5": 0.2189311010946555, |
| "hr@10": 0.23889246619446233, |
| "mrr@3": 0.1899012664941074, |
| "mrr@5": 0.19294376489967205, |
| "mrr@10": 0.19550230500875407, |
| "loss": 9.141914023173983 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": -0.004185447520927227, |
| "d_hr@10_ci": [ |
| -0.008692852543464247, |
| 0.0003219575016097931 |
| ], |
| "d_mrr@10": -8.726070380921103e-05, |
| "d_mrr@10_ci": [ |
| -0.0011988125900714338, |
| 0.0010856164208545979 |
| ], |
| "mcnemar_p": 0.07894069368148691 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.0032195750160978753, |
| "d_hr@10_ci": [ |
| -0.0028976175144880822, |
| 0.009014810045074068 |
| ], |
| "d_mrr@10": 0.008797105448747455, |
| "d_mrr@10_ci": [ |
| 0.0058886282566257536, |
| 0.011998394684123979 |
| ], |
| "mcnemar_p": 0.34814129031920826 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:13:21Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.19671603348358016, |
| "mrr@10": 0.15395381714229958, |
| "pct_change_hr@10": -16.18655692729768, |
| "pct_change_hr@10_ci95": [ |
| -19.264387170902758, |
| -13.167998565647263 |
| ], |
| "pct_change_mrr@10": -21.21700879187991, |
| "pct_change_mrr@10_ci95": [ |
| -24.17786279642231, |
| -18.26020458277787 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.15421764327108822, |
| "mrr@10": 0.1047861795327929, |
| "pct_change_hr@10": -34.293552812071326, |
| "pct_change_hr@10_ci95": [ |
| -37.772055176467944, |
| -30.716229295376298 |
| ], |
| "pct_change_mrr@10": -46.37762925206281, |
| "pct_change_mrr@10_ci95": [ |
| -49.80445898826904, |
| -43.037569407074 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| }, |
| "Scientific": { |
| "n_test_users_evaluated": 5619, |
| "n_test_users_in_split": 5619, |
| "flag": "evaluation data differs from the RecPFN paper's (undisclosed filtering)", |
| "recicl_v0": { |
| "hr@3": 0.043602064424274785, |
| "hr@5": 0.05089873642996975, |
| "hr@10": 0.06157679302366969, |
| "mrr@3": 0.030758735298325443, |
| "mrr@5": 0.032431630840354976, |
| "mrr@10": 0.033883832448115235, |
| "loss": 11.270965716363188 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.023313756896244882, |
| "hr@5": 0.029364655632674853, |
| "hr@10": 0.041110517885744796, |
| "mrr@3": 0.01319926444651245, |
| "mrr@5": 0.014614106951011927, |
| "mrr@10": 0.016231917315561336, |
| "loss": 11.957719512629879 |
| }, |
| "embknn_last": { |
| "hr@3": 0.05196654208933974, |
| "hr@5": 0.06371240434240968, |
| "hr@10": 0.07492436376579463, |
| "mrr@3": 0.03336892694547775, |
| "mrr@5": 0.036029542729849914, |
| "mrr@10": 0.037563454074025004, |
| "loss": 11.922406332712196 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": -0.013347570742124941, |
| "d_hr@10_ci": [ |
| -0.017262858159814917, |
| -0.009432283324434958 |
| ], |
| "d_mrr@10": -0.003679621578713932, |
| "d_mrr@10_ci": [ |
| -0.0055008781854083595, |
| -0.0017209008833408014 |
| ], |
| "mcnemar_p": 1.8819747521132367e-11 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.020466275137924893, |
| "d_hr@10_ci": [ |
| 0.015127246841074922, |
| 0.025627335824879874 |
| ], |
| "d_mrr@10": 0.017651915129224236, |
| "d_mrr@10_ci": [ |
| 0.01421929070020367, |
| 0.02114898572586773 |
| ], |
| "mcnemar_p": 8.883400844804864e-14 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:14:55Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.041110517885744796, |
| "mrr@10": 0.01950525012033693, |
| "pct_change_hr@10": -33.23699421965317, |
| "pct_change_hr@10_ci95": [ |
| -39.64649321266968, |
| -26.436781609195403 |
| ], |
| "pct_change_mrr@10": -42.43493515615617, |
| "pct_change_mrr@10_ci95": [ |
| -49.3741780404384, |
| -35.22245199758977 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.019754404698344902, |
| "mrr@10": 0.007489329049820195, |
| "pct_change_hr@10": -67.91907514450867, |
| "pct_change_hr@10_ci95": [ |
| -73.18456788046039, |
| -62.63396894784444 |
| ], |
| "pct_change_mrr@10": -77.89704260494068, |
| "pct_change_mrr@10_ci95": [ |
| -82.35918468816068, |
| -73.04899679853824 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| }, |
| "Software": { |
| "n_test_users_evaluated": 718, |
| "n_test_users_in_split": 718, |
| "flag": null, |
| "recicl_v0": { |
| "hr@3": 0.10167130919220056, |
| "hr@5": 0.12256267409470752, |
| "hr@10": 0.1573816155988858, |
| "mrr@3": 0.08983286916379477, |
| "mrr@5": 0.0945682452374182, |
| "mrr@10": 0.09920911284417827, |
| "loss": 8.473287451898155 |
| }, |
| "embknn_balanced": { |
| "hr@3": 0.07242339832869081, |
| "hr@5": 0.08774373259052924, |
| "hr@10": 0.1309192200557103, |
| "mrr@3": 0.029944290316204505, |
| "mrr@5": 0.03328690872966081, |
| "mrr@10": 0.0390475092969399, |
| "loss": 9.862932594373698 |
| }, |
| "embknn_last": { |
| "hr@3": 0.10306406685236769, |
| "hr@5": 0.13231197771587744, |
| "hr@10": 0.1713091922005571, |
| "mrr@3": 0.08890436407085249, |
| "mrr@5": 0.09551996299815377, |
| "mrr@10": 0.10036034862352614, |
| "loss": 9.816932444452908 |
| }, |
| "paired_differences_95ci": { |
| "vs_embknn_last": { |
| "d_hr@10": -0.013927576601671293, |
| "d_hr@10_ci": [ |
| -0.03203342618384403, |
| 0.005571030640668523 |
| ], |
| "d_mrr@10": -0.0011512357960826308, |
| "d_mrr@10_ci": [ |
| -0.009422102290312597, |
| 0.00782562618826542 |
| ], |
| "mcnemar_p": 0.20263875106454401 |
| }, |
| "vs_embknn_balanced": { |
| "d_hr@10": 0.026462395543175504, |
| "d_hr@10_ci": [ |
| 0.0069637883008356605, |
| 0.04596100278551532 |
| ], |
| "d_mrr@10": 0.06016160410310832, |
| "d_mrr@10_ci": [ |
| 0.04537651434761468, |
| 0.07661199263828095 |
| ], |
| "mcnemar_p": 0.01445383353382763 |
| } |
| }, |
| "eval_gpu": "NVIDIA L4", |
| "evaluated_at": "2026-09-29T11:12:30Z", |
| "context_ablations": { |
| "no_context": { |
| "hr@10": 0.15459610027855153, |
| "mrr@10": 0.09153733946237723, |
| "pct_change_hr@10": -1.7699115044247926, |
| "pct_change_hr@10_ci95": [ |
| -12.381163084702909, |
| 9.890381895332398 |
| ], |
| "pct_change_mrr@10": -7.732932148935379, |
| "pct_change_mrr@10_ci95": [ |
| -16.936256020977453, |
| 1.2985526363363442 |
| ], |
| "mean_context_sequences": 0.0 |
| }, |
| "random_context": { |
| "hr@10": 0.04456824512534819, |
| "mrr@10": 0.033185767398986314, |
| "pct_change_hr@10": -71.68141592920354, |
| "pct_change_hr@10_ci95": [ |
| -80.17665045682287, |
| -62.59908536585367 |
| ], |
| "pct_change_mrr@10": -66.54967830313211, |
| "pct_change_mrr@10_ci95": [ |
| -76.83137579538865, |
| -54.930787982373936 |
| ], |
| "mean_context_sequences": 8.0 |
| } |
| } |
| } |
| }, |
| "macro_mean_7": { |
| "recicl_v0": { |
| "hr@10": 0.1390810843398162, |
| "mrr@10": 0.09355228239514829 |
| }, |
| "embknn_balanced": { |
| "hr@10": 0.11207443075966694, |
| "mrr@10": 0.06307460142705772 |
| }, |
| "embknn_last": { |
| "hr@10": 0.1471064365910456, |
| "mrr@10": 0.09372087010221936 |
| } |
| }, |
| "context_ablations_note": "same checkpoint, TEST split and protocol, batch 1; 'no_context' = 0 context sequences, 'random_context' = 8 randomly chosen train-split sequences instead of the retrieved ones. pct_change = % change vs the reported numbers of this model; ci95 = paired bootstrap over test users (2,000 resamples)", |
| "source_files": "our evaluation outputs for this checkpoint: per-dataset TEST results and metadata; paired differences and n_test from our aggregate tables; context ablations from our ablation run" |
| } |
|
|
