{
  "model": "devtaji/recICL",
  "checkpoint_sha256": "661fd3415a0b8f2d727898006e6314dcd273e15f59da3383668f7bc1a3997a3a",
  "split": "test",
  "protocol": {
    "split": "user-level 70/10/20, seed 42; at most 10,000 users sampled into the test split, then users with a single interaction dropped",
    "target": "each test user's last item; input = up to 14 preceding items",
    "context": "8 sequences from train-split users (retrieval as in 'inference')",
    "batch_size": 1,
    "ranking": "full item catalog, no candidate sampling, repeated items allowed",
    "metrics": [
      "HR@10",
      "MRR@10"
    ],
    "hardware": "NVIDIA L4, fp32, TF32 off"
  },
  "baselines": {
    "embknn_balanced": "score = dot(mean of the history item embeddings, item embedding); its numbers are close to the EmbKNN column of the RecPFN paper's Table 2 (within about 10% on Arts, Games, Movies, Pantry, Software)",
    "embknn_last": "score = dot(last history item embedding, item embedding); the EmbKNN that the RecPFN paper's text describes"
  },
  "datasets_source": "Amazon Reviews 2018 (McAuley et al.) categories, rebuilt from the raw ratings + metadata: Appliances, Arts_Crafts_and_Sewing, Video_Games, Movies_and_TV, Prime_Pantry, Industrial_and_Scientific, Software",
  "paired_differences_note": "model minus baseline, paired bootstrap over test users (2,000 resamples), 95% CI; McNemar exact p on HR@10 hits",
  "datasets": {
    "Appliances": {
      "n_test_users_evaluated": 145,
      "n_test_users_in_split": 145,
      "flag": "evaluation data differs from the RecPFN paper's (undisclosed filtering)",
      "recicl_v0": {
        "hr@3": 0.06206896551724138,
        "hr@5": 0.06896551724137931,
        "hr@10": 0.10344827586206896,
        "mrr@3": 0.04827586206896552,
        "mrr@5": 0.04965517243434643,
        "mrr@10": 0.05434865910431434,
        "loss": 9.28750396761401
      },
      "embknn_balanced": {
        "hr@3": 0.027586206896551724,
        "hr@5": 0.041379310344827586,
        "hr@10": 0.06206896551724138,
        "mrr@3": 0.013793103448275862,
        "mrr@5": 0.016551724179037688,
        "mrr@10": 0.01906951295918432,
        "loss": 10.228556942117626
      },
      "embknn_last": {
        "hr@3": 0.09655172413793103,
        "hr@5": 0.1103448275862069,
        "hr@10": 0.1310344827586207,
        "mrr@3": 0.05862068986070567,
        "mrr@5": 0.062068965722774636,
        "mrr@10": 0.06475095810561345,
        "loss": 10.193267019863786
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": -0.027586206896551738,
          "d_hr@10_ci": [
            -0.06206896551724139,
            0.0
          ],
          "d_mrr@10": -0.010402298850574712,
          "d_mrr@10_ci": [
            -0.023873084291187747,
            0.0023036398467432624
          ],
          "mcnemar_p": 0.21875000000000014
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.041379310344827586,
          "d_hr@10_ci": [
            0.0,
            0.08275862068965519
          ],
          "d_mrr@10": 0.035279146141215106,
          "d_mrr@10_ci": [
            0.007594074986316366,
            0.06752230432402839
          ],
          "mcnemar_p": 0.109375
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:12:17Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.05517241379310345,
          "mrr@10": 0.020000000092489967,
          "pct_change_hr@10": -46.666666666666664,
          "pct_change_hr@10_ci95": [
            -75.0,
            -14.261904761904896
          ],
          "pct_change_mrr@10": -63.200563873888996,
          "pct_change_mrr@10_ci95": [
            -91.32381136817679,
            -33.380598313527344
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.041379310344827586,
          "mrr@10": 0.03103448275862069,
          "pct_change_hr@10": -60.0,
          "pct_change_hr@10_ci95": [
            -85.71428571428572,
            -33.333333333333336
          ],
          "pct_change_mrr@10": -42.897426964933,
          "pct_change_mrr@10_ci95": [
            -81.16710629577956,
            6.011155432140703
          ],
          "mean_context_sequences": 8.0
        }
      }
    },
    "Arts": {
      "n_test_users_evaluated": 2997,
      "n_test_users_in_split": 10000,
      "flag": null,
      "recicl_v0": {
        "hr@3": 0.1871871871871872,
        "hr@5": 0.20186853520186854,
        "hr@10": 0.22055388722055388,
        "mrr@3": 0.15170726306366053,
        "mrr@5": 0.15502724973786303,
        "mrr@10": 0.15758139124245177,
        "loss": 10.023291922466811
      },
      "embknn_balanced": {
        "hr@3": 0.16049382716049382,
        "hr@5": 0.1828495161828495,
        "hr@10": 0.20153486820153488,
        "mrr@3": 0.131854076474199,
        "mrr@5": 0.1367589814198745,
        "mrr@10": 0.1393652915019849,
        "loss": 12.450908683481558
      },
      "embknn_last": {
        "hr@3": 0.17450784117450785,
        "hr@5": 0.19953286619953287,
        "hr@10": 0.21788455121788455,
        "mrr@3": 0.14547881232447174,
        "mrr@5": 0.15100100122374777,
        "mrr@10": 0.153521643620274,
        "loss": 12.435229098116672
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": 0.0026693360026693314,
          "d_hr@10_ci": [
            -0.003336670003336678,
            0.00934267600934266
          ],
          "d_mrr@10": 0.004059747578266104,
          "d_mrr@10_ci": [
            0.0016380765686321512,
            0.006624263152040991
          ],
          "mcnemar_p": 0.46570744577306067
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.019019019019019007,
          "d_hr@10_ci": [
            0.01167834501167836,
            0.026359693026359682
          ],
          "d_mrr@10": 0.018216099697581145,
          "d_mrr@10_ci": [
            0.014113558002446853,
            0.022250213176138966
          ],
          "mcnemar_p": 4.3609050114828154e-07
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:15:00Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.18618618618618618,
          "mrr@10": 0.12968577070408438,
          "pct_change_hr@10": -15.582450832072613,
          "pct_change_hr@10_ci95": [
            -19.082868467410457,
            -12.157674772036467
          ],
          "pct_change_mrr@10": -17.702357060325546,
          "pct_change_mrr@10_ci95": [
            -20.833086015350958,
            -14.734061825197962
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.08541875208541876,
          "mrr@10": 0.048784234107017994,
          "pct_change_hr@10": -61.27080181543116,
          "pct_change_hr@10_ci95": [
            -64.90455212922173,
            -57.557142362952874
          ],
          "pct_change_mrr@10": -69.0418813272441,
          "pct_change_mrr@10_ci95": [
            -72.615949611093,
            -65.57546376581101
          ],
          "mean_context_sequences": 8.0
        }
      }
    },
    "Games": {
      "n_test_users_evaluated": 10000,
      "n_test_users_in_split": 10000,
      "flag": null,
      "recicl_v0": {
        "hr@3": 0.0513,
        "hr@5": 0.0588,
        "hr@10": 0.0722,
        "mrr@3": 0.04260000007748604,
        "mrr@5": 0.044295000088214874,
        "mrr@10": 0.046121071556210515,
        "loss": 10.213145974993706
      },
      "embknn_balanced": {
        "hr@3": 0.02,
        "hr@5": 0.0303,
        "hr@10": 0.0479,
        "mrr@3": 0.011700000071525574,
        "mrr@5": 0.014055000084638596,
        "mrr@10": 0.01636996045857668,
        "loss": 11.093444060993194
      },
      "embknn_last": {
        "hr@3": 0.0506,
        "hr@5": 0.0585,
        "hr@10": 0.0736,
        "mrr@3": 0.03945000005960465,
        "mrr@5": 0.04121000007241964,
        "mrr@10": 0.04316595249548554,
        "loss": 11.0686561044693
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": -0.0013999999999999985,
          "d_hr@10_ci": [
            -0.004699999999999996,
            0.0020000000000000018
          ],
          "d_mrr@10": 0.002955119047619019,
          "d_mrr@10_ci": [
            0.00160891765873009,
            0.004420426587301565
          ],
          "mcnemar_p": 0.43727338509353675
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.024300000000000002,
          "d_hr@10_ci": [
            0.019600000000000006,
            0.0288
          ],
          "d_mrr@10": 0.029751111111111096,
          "d_mrr@10_ci": [
            0.026522191468253967,
            0.03297455158730153
          ],
          "mcnemar_p": 1.5319468898317913e-25
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:16:13Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.0716,
          "mrr@10": 0.03594726204872131,
          "pct_change_hr@10": -0.8310249307479256,
          "pct_change_hr@10_ci95": [
            -5.364047361136176,
            3.841910853289687
          ],
          "pct_change_mrr@10": -22.058918329965017,
          "pct_change_mrr@10_ci95": [
            -25.54905745074236,
            -18.56859102137774
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.0119,
          "mrr@10": 0.004851468288898468,
          "pct_change_hr@10": -83.5180055401662,
          "pct_change_hr@10_ci95": [
            -86.22530832926526,
            -80.73741871966297
          ],
          "pct_change_mrr@10": -89.4810156720108,
          "pct_change_mrr@10_ci95": [
            -91.75046365557786,
            -87.02863863527669
          ],
          "mean_context_sequences": 8.0
        }
      }
    },
    "Movies": {
      "n_test_users_evaluated": 10000,
      "n_test_users_in_split": 10000,
      "flag": null,
      "recicl_v0": {
        "hr@3": 0.0951,
        "hr@5": 0.1096,
        "hr@10": 0.1237,
        "mrr@3": 0.06308333349823951,
        "mrr@5": 0.06639833351671695,
        "mrr@10": 0.0683068653061986,
        "loss": 10.933070637273788
      },
      "embknn_balanced": {
        "hr@3": 0.0335,
        "hr@5": 0.0491,
        "hr@10": 0.0695,
        "mrr@3": 0.018566666802763938,
        "mrr@5": 0.02212666682302952,
        "mrr@10": 0.02482007957994938,
        "loss": 12.046554122447967
      },
      "embknn_last": {
        "hr@3": 0.0778,
        "hr@5": 0.1049,
        "hr@10": 0.1221,
        "mrr@3": 0.05258333346545696,
        "mrr@5": 0.058868333494663236,
        "mrr@10": 0.061181428787857296,
        "loss": 12.016274190807342
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": 0.0016000000000000042,
          "d_hr@10_ci": [
            -0.0011999999999999932,
            0.00449999999999999
          ],
          "d_mrr@10": 0.0071254365079365944,
          "d_mrr@10_ci": [
            0.005697232142857268,
            0.008514329365079398
          ],
          "mcnemar_p": 0.3029036319209554
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.0542,
          "d_hr@10_ci": [
            0.04870000000000001,
            0.05929999999999999
          ],
          "d_mrr@10": 0.04348678571428581,
          "d_mrr@10_ci": [
            0.0402409722222223,
            0.046700817460317574
          ],
          "mcnemar_p": 1.4361709413897885e-91
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:17:56Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.102,
          "mrr@10": 0.04952440501749516,
          "pct_change_hr@10": -17.542441390460805,
          "pct_change_hr@10_ci95": [
            -20.496807702572312,
            -14.590643244107216
          ],
          "pct_change_mrr@10": -27.497177925685023,
          "pct_change_mrr@10_ci95": [
            -30.157991463516414,
            -24.89154990329349
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.0179,
          "mrr@10": 0.010642261940985918,
          "pct_change_hr@10": -85.52950687146321,
          "pct_change_hr@10_ci95": [
            -87.47939468293607,
            -83.4124518447913
          ],
          "pct_change_mrr@10": -84.419923395284,
          "pct_change_mrr@10_ci95": [
            -86.68075954394435,
            -81.80099004889513
          ],
          "mean_context_sequences": 8.0
        }
      }
    },
    "Pantry": {
      "n_test_users_evaluated": 3106,
      "n_test_users_in_split": 10000,
      "flag": null,
      "recicl_v0": {
        "hr@3": 0.2063747585318738,
        "hr@5": 0.2169993560849968,
        "hr@10": 0.2347070186735351,
        "mrr@3": 0.190759819809345,
        "mrr@5": 0.19320669683405298,
        "mrr@10": 0.19541504426456927,
        "loss": 7.7768060520089834
      },
      "embknn_balanced": {
        "hr@3": 0.19639407598197037,
        "hr@5": 0.20927237604636187,
        "hr@10": 0.23148744365743723,
        "mrr@3": 0.1807791372882268,
        "mrr@5": 0.18370895057014702,
        "mrr@10": 0.18661793887720748,
        "loss": 9.150342638233134
      },
      "embknn_last": {
        "hr@3": 0.20573084352865423,
        "hr@5": 0.2189311010946555,
        "hr@10": 0.23889246619446233,
        "mrr@3": 0.1899012664941074,
        "mrr@5": 0.19294376489967205,
        "mrr@10": 0.19550230500875407,
        "loss": 9.141914023173983
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": -0.004185447520927227,
          "d_hr@10_ci": [
            -0.008692852543464247,
            0.0003219575016097931
          ],
          "d_mrr@10": -8.726070380921103e-05,
          "d_mrr@10_ci": [
            -0.0011988125900714338,
            0.0010856164208545979
          ],
          "mcnemar_p": 0.07894069368148691
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.0032195750160978753,
          "d_hr@10_ci": [
            -0.0028976175144880822,
            0.009014810045074068
          ],
          "d_mrr@10": 0.008797105448747455,
          "d_mrr@10_ci": [
            0.0058886282566257536,
            0.011998394684123979
          ],
          "mcnemar_p": 0.34814129031920826
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:13:21Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.19671603348358016,
          "mrr@10": 0.15395381714229958,
          "pct_change_hr@10": -16.18655692729768,
          "pct_change_hr@10_ci95": [
            -19.264387170902758,
            -13.167998565647263
          ],
          "pct_change_mrr@10": -21.21700879187991,
          "pct_change_mrr@10_ci95": [
            -24.17786279642231,
            -18.26020458277787
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.15421764327108822,
          "mrr@10": 0.1047861795327929,
          "pct_change_hr@10": -34.293552812071326,
          "pct_change_hr@10_ci95": [
            -37.772055176467944,
            -30.716229295376298
          ],
          "pct_change_mrr@10": -46.37762925206281,
          "pct_change_mrr@10_ci95": [
            -49.80445898826904,
            -43.037569407074
          ],
          "mean_context_sequences": 8.0
        }
      }
    },
    "Scientific": {
      "n_test_users_evaluated": 5619,
      "n_test_users_in_split": 5619,
      "flag": "evaluation data differs from the RecPFN paper's (undisclosed filtering)",
      "recicl_v0": {
        "hr@3": 0.043602064424274785,
        "hr@5": 0.05089873642996975,
        "hr@10": 0.06157679302366969,
        "mrr@3": 0.030758735298325443,
        "mrr@5": 0.032431630840354976,
        "mrr@10": 0.033883832448115235,
        "loss": 11.270965716363188
      },
      "embknn_balanced": {
        "hr@3": 0.023313756896244882,
        "hr@5": 0.029364655632674853,
        "hr@10": 0.041110517885744796,
        "mrr@3": 0.01319926444651245,
        "mrr@5": 0.014614106951011927,
        "mrr@10": 0.016231917315561336,
        "loss": 11.957719512629879
      },
      "embknn_last": {
        "hr@3": 0.05196654208933974,
        "hr@5": 0.06371240434240968,
        "hr@10": 0.07492436376579463,
        "mrr@3": 0.03336892694547775,
        "mrr@5": 0.036029542729849914,
        "mrr@10": 0.037563454074025004,
        "loss": 11.922406332712196
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": -0.013347570742124941,
          "d_hr@10_ci": [
            -0.017262858159814917,
            -0.009432283324434958
          ],
          "d_mrr@10": -0.003679621578713932,
          "d_mrr@10_ci": [
            -0.0055008781854083595,
            -0.0017209008833408014
          ],
          "mcnemar_p": 1.8819747521132367e-11
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.020466275137924893,
          "d_hr@10_ci": [
            0.015127246841074922,
            0.025627335824879874
          ],
          "d_mrr@10": 0.017651915129224236,
          "d_mrr@10_ci": [
            0.01421929070020367,
            0.02114898572586773
          ],
          "mcnemar_p": 8.883400844804864e-14
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:14:55Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.041110517885744796,
          "mrr@10": 0.01950525012033693,
          "pct_change_hr@10": -33.23699421965317,
          "pct_change_hr@10_ci95": [
            -39.64649321266968,
            -26.436781609195403
          ],
          "pct_change_mrr@10": -42.43493515615617,
          "pct_change_mrr@10_ci95": [
            -49.3741780404384,
            -35.22245199758977
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.019754404698344902,
          "mrr@10": 0.007489329049820195,
          "pct_change_hr@10": -67.91907514450867,
          "pct_change_hr@10_ci95": [
            -73.18456788046039,
            -62.63396894784444
          ],
          "pct_change_mrr@10": -77.89704260494068,
          "pct_change_mrr@10_ci95": [
            -82.35918468816068,
            -73.04899679853824
          ],
          "mean_context_sequences": 8.0
        }
      }
    },
    "Software": {
      "n_test_users_evaluated": 718,
      "n_test_users_in_split": 718,
      "flag": null,
      "recicl_v0": {
        "hr@3": 0.10167130919220056,
        "hr@5": 0.12256267409470752,
        "hr@10": 0.1573816155988858,
        "mrr@3": 0.08983286916379477,
        "mrr@5": 0.0945682452374182,
        "mrr@10": 0.09920911284417827,
        "loss": 8.473287451898155
      },
      "embknn_balanced": {
        "hr@3": 0.07242339832869081,
        "hr@5": 0.08774373259052924,
        "hr@10": 0.1309192200557103,
        "mrr@3": 0.029944290316204505,
        "mrr@5": 0.03328690872966081,
        "mrr@10": 0.0390475092969399,
        "loss": 9.862932594373698
      },
      "embknn_last": {
        "hr@3": 0.10306406685236769,
        "hr@5": 0.13231197771587744,
        "hr@10": 0.1713091922005571,
        "mrr@3": 0.08890436407085249,
        "mrr@5": 0.09551996299815377,
        "mrr@10": 0.10036034862352614,
        "loss": 9.816932444452908
      },
      "paired_differences_95ci": {
        "vs_embknn_last": {
          "d_hr@10": -0.013927576601671293,
          "d_hr@10_ci": [
            -0.03203342618384403,
            0.005571030640668523
          ],
          "d_mrr@10": -0.0011512357960826308,
          "d_mrr@10_ci": [
            -0.009422102290312597,
            0.00782562618826542
          ],
          "mcnemar_p": 0.20263875106454401
        },
        "vs_embknn_balanced": {
          "d_hr@10": 0.026462395543175504,
          "d_hr@10_ci": [
            0.0069637883008356605,
            0.04596100278551532
          ],
          "d_mrr@10": 0.06016160410310832,
          "d_mrr@10_ci": [
            0.04537651434761468,
            0.07661199263828095
          ],
          "mcnemar_p": 0.01445383353382763
        }
      },
      "eval_gpu": "NVIDIA L4",
      "evaluated_at": "2026-09-29T11:12:30Z",
      "context_ablations": {
        "no_context": {
          "hr@10": 0.15459610027855153,
          "mrr@10": 0.09153733946237723,
          "pct_change_hr@10": -1.7699115044247926,
          "pct_change_hr@10_ci95": [
            -12.381163084702909,
            9.890381895332398
          ],
          "pct_change_mrr@10": -7.732932148935379,
          "pct_change_mrr@10_ci95": [
            -16.936256020977453,
            1.2985526363363442
          ],
          "mean_context_sequences": 0.0
        },
        "random_context": {
          "hr@10": 0.04456824512534819,
          "mrr@10": 0.033185767398986314,
          "pct_change_hr@10": -71.68141592920354,
          "pct_change_hr@10_ci95": [
            -80.17665045682287,
            -62.59908536585367
          ],
          "pct_change_mrr@10": -66.54967830313211,
          "pct_change_mrr@10_ci95": [
            -76.83137579538865,
            -54.930787982373936
          ],
          "mean_context_sequences": 8.0
        }
      }
    }
  },
  "macro_mean_7": {
    "recicl_v0": {
      "hr@10": 0.1390810843398162,
      "mrr@10": 0.09355228239514829
    },
    "embknn_balanced": {
      "hr@10": 0.11207443075966694,
      "mrr@10": 0.06307460142705772
    },
    "embknn_last": {
      "hr@10": 0.1471064365910456,
      "mrr@10": 0.09372087010221936
    }
  },
  "context_ablations_note": "same checkpoint, TEST split and protocol, batch 1; 'no_context' = 0 context sequences, 'random_context' = 8 randomly chosen train-split sequences instead of the retrieved ones. pct_change = % change vs the reported numbers of this model; ci95 = paired bootstrap over test users (2,000 resamples)",
  "source_files": "our evaluation outputs for this checkpoint: per-dataset TEST results and metadata; paired differences and n_test from our aggregate tables; context ablations from our ablation run"
}