{
  "task": "LEMBSummScreenFDRetrieval",
  "split": "validation",
  "subset": "default",
  "main_metric": "ndcg_at_10",
  "scores": {
    "original": 0.76988,
    "vela": 0.8323,
    "r6": 0.88493,
    "r9": 0.87688,
    "r13": 0.88041,
    "r20": 0.88471,
    "candidate": 0.88362,
    "r3": 0.86925
  },
  "score_scale": "Official native FP32 metrics on0–1 scale; multiply by100 for display.",
  "candidate_content_sha256": "3bf3ad85cdc82f8c5e67d44a59088b89de98f465a1a2b9126ff43a5f6f4b6225",
  "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c",
  "coverage": {
    "queries": 336,
    "documents": 336,
    "configured_max_length": 32768,
    "document_token_lengths": {
      "minimum": 2662,
      "maximum": 23856,
      "sum": 2887533,
      "count": 336
    },
    "truncated_documents": 0
  },
  "paired_comparison": {
    "status": "completed",
    "gate_component": {
      "passed": true,
      "delta_vs_r13": 0.0032154935251345294,
      "minimum_delta": -0.005,
      "units": "raw score delta",
      "release_decision": false
    },
    "scope": "Already exposed long-development validation. No new model inference; no sealed confirmation data.",
    "candidate_content_sha256": "3bf3ad85cdc82f8c5e67d44a59088b89de98f465a1a2b9126ff43a5f6f4b6225",
    "task": "LEMBSummScreenFDRetrieval",
    "split": "validation",
    "metric": "ndcg_at_10",
    "replicates": 2000,
    "seed": 20260915,
    "unit": "Paired query/source-document, verified336distinct sources",
    "evidence": {
      "original": {
        "official_ndcg_at_10": 0.76988,
        "recomputed_ndcg_at_10": 0.7698826358058495,
        "result_sha256": "2e3c8594a8b7f7f62fad52f7dceaaf338b1146f8d90ad731b6345d50c6ffe833",
        "prediction_sha256": "e14f41b52de7d7a9f6956283a9dee3dcdf496f42c90f824153922d04ca4ef743",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "e8c0d697b5a37c08a9e3ed177295a6e6a23fd15e2e81adf0e17f1a4dc88f7d93"
      },
      "vela": {
        "official_ndcg_at_10": 0.8323,
        "recomputed_ndcg_at_10": 0.8323025262730762,
        "result_sha256": "2a0b49ad538fa71dccd5bde5bdbd0aa3d877708f7ac6cc3ea435852c7812ddf8",
        "prediction_sha256": "9340d48309f598ab2ee5ba3b58fcfe37a530f052528ead0f754ab37bea8fa030",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "3db940cd46ac4b4d4c62af8d98baa1b116a761305f2005e2c658e9bffe728862"
      },
      "r6-intent-all22-lr1e5": {
        "official_ndcg_at_10": 0.88493,
        "recomputed_ndcg_at_10": 0.884925127232939,
        "result_sha256": "b7ea70458c34d2fd0f6210d19f95440fb33cafba551f1d404978cc8ca6200241",
        "prediction_sha256": "8cb20cd7bd7442d7c80fb29c5cf54525a111eb6e67c4b509ed90314146da79f7",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "cdd98ac1f6110610422007186dc6b19823466446aa6cfca379b45ac4d6b89f0c"
      },
      "r9-guide-mask-lr3e6": {
        "official_ndcg_at_10": 0.87688,
        "recomputed_ndcg_at_10": 0.876882778899065,
        "result_sha256": "95f44d11d6ff02fd672cc2c0fe7398ee148177ffd5291277d14d305e625499b0",
        "prediction_sha256": "7c5f26d11fcb98cb6efd07bc1f56f590b463dbda83b017867783c6cd3a29fc07",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "0c78366b6fddb877399974ba0e3671d5f49cf14a49dc38db52126504e9ce5151"
      },
      "r13-bge-kd-w01": {
        "official_ndcg_at_10": 0.88041,
        "recomputed_ndcg_at_10": 0.8804095047305854,
        "result_sha256": "34afebd3836a1d25bf0e7798a6c47d979ff4537d389a0fa46494f81ba3b3572f",
        "prediction_sha256": "9d314706274078458ea080b56c7c63a2b3e339ea20584d980bc919cfc43c2dbf",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "fd547245c7fdc2b299dd8a4e1bd7631ccee3fb299911933301cacacd6e7ad7ef"
      },
      "r20-t2ranking-zh8192": {
        "official_ndcg_at_10": 0.88471,
        "recomputed_ndcg_at_10": 0.8847132026337041,
        "result_sha256": "469de4a61c0c787dfe3bfc52805fa51b6d22e94ca712682f46a5868b4f85f088",
        "prediction_sha256": "801ab1f7fb36034b361cdc0996cfff3473ad4704292f301e0294b933566d1400",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "2df6252ed766d521014b14ee0d9d9355c6285e7dcf7a87177daa44a3acdbf9a3"
      },
      "r24-new-queries": {
        "official_ndcg_at_10": 0.88362,
        "recomputed_ndcg_at_10": 0.8836249982557198,
        "result_sha256": "fe279244d0ae16d2c759e4320e28b0b3e0f82688386d5aa429103430277e27b3",
        "prediction_sha256": "4415bf7da963741204a32979abd479066736bb47fe6dda101a6e6618abeb4b14",
        "qrels_sha256": "e8e521e2122fedc7d57678af6819ae09418cc7e346b6f029099da1a606c64558",
        "token_lengths_sha256": "c28375f904225f4bc5d62415e676f732970fd04ca540b45119abb0a0e4bd6800"
      }
    },
    "rows": [
      {
        "candidate": "r24-new-queries",
        "baseline": "original",
        "group": "all",
        "queries": 336,
        "candidate_score": 0.8836249982557198,
        "baseline_score": 0.7698826358058495,
        "delta": 0.11374236244987043,
        "paired_bootstrap_95_interval": [
          0.08580597893036003,
          0.1436500045695695
        ],
        "improved": 94,
        "equal": 225,
        "regressed": 17
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "original",
        "group": "relevant_document_at_most_8192_tokens",
        "queries": 170,
        "candidate_score": 0.9207402405116178,
        "baseline_score": 0.839537458569051,
        "delta": 0.08120278194256693,
        "paired_bootstrap_95_interval": [
          0.04316328424717932,
          0.12102952443348348
        ],
        "improved": 35,
        "equal": 126,
        "regressed": 9
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "original",
        "group": "relevant_document_above_8192_tokens",
        "queries": 166,
        "candidate_score": 0.845615412812933,
        "baseline_score": 0.6985493835784745,
        "delta": 0.14706602923445836,
        "paired_bootstrap_95_interval": [
          0.10448118709621954,
          0.19084205323592268
        ],
        "improved": 59,
        "equal": 99,
        "regressed": 8
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "vela",
        "group": "all",
        "queries": 336,
        "candidate_score": 0.8836249982557198,
        "baseline_score": 0.8323025262730762,
        "delta": 0.05132247198264383,
        "paired_bootstrap_95_interval": [
          0.02595699843539391,
          0.0778629398066223
        ],
        "improved": 59,
        "equal": 246,
        "regressed": 31
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "vela",
        "group": "relevant_document_at_most_8192_tokens",
        "queries": 170,
        "candidate_score": 0.9207402405116178,
        "baseline_score": 0.8468191092768992,
        "delta": 0.07392113123471859,
        "paired_bootstrap_95_interval": [
          0.04501357914334632,
          0.10670490435922182
        ],
        "improved": 30,
        "equal": 135,
        "regressed": 5
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "vela",
        "group": "relevant_document_above_8192_tokens",
        "queries": 166,
        "candidate_score": 0.845615412812933,
        "baseline_score": 0.817436146088438,
        "delta": 0.02817926672449499,
        "paired_bootstrap_95_interval": [
          -0.014128021736724831,
          0.07003966358355486
        ],
        "improved": 29,
        "equal": 111,
        "regressed": 26
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r6-intent-all22-lr1e5",
        "group": "all",
        "queries": 336,
        "candidate_score": 0.8836249982557198,
        "baseline_score": 0.884925127232939,
        "delta": -0.0013001289772192758,
        "paired_bootstrap_95_interval": [
          -0.008658072287171298,
          0.005251621338530261
        ],
        "improved": 18,
        "equal": 306,
        "regressed": 12
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r6-intent-all22-lr1e5",
        "group": "relevant_document_at_most_8192_tokens",
        "queries": 170,
        "candidate_score": 0.9207402405116178,
        "baseline_score": 0.9245553524790817,
        "delta": -0.0038151119674638572,
        "paired_bootstrap_95_interval": [
          -0.012504089298200982,
          0.002641760875011354
        ],
        "improved": 4,
        "equal": 162,
        "regressed": 4
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r6-intent-all22-lr1e5",
        "group": "relevant_document_above_8192_tokens",
        "queries": 166,
        "candidate_score": 0.845615412812933,
        "baseline_score": 0.8443399568001426,
        "delta": 0.0012754560127902356,
        "paired_bootstrap_95_interval": [
          -0.009602981539440099,
          0.012731905072227398
        ],
        "improved": 14,
        "equal": 144,
        "regressed": 8
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r9-guide-mask-lr3e6",
        "group": "all",
        "queries": 336,
        "candidate_score": 0.8836249982557198,
        "baseline_score": 0.876882778899065,
        "delta": 0.0067422193566549615,
        "paired_bootstrap_95_interval": [
          0.0006376803680880756,
          0.01385482075292914
        ],
        "improved": 19,
        "equal": 311,
        "regressed": 6
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r9-guide-mask-lr3e6",
        "group": "relevant_document_at_most_8192_tokens",
        "queries": 170,
        "candidate_score": 0.9207402405116178,
        "baseline_score": 0.9142852803660881,
        "delta": 0.006454960145529764,
        "paired_bootstrap_95_interval": [
          -0.0024266134694441643,
          0.01719387898700963
        ],
        "improved": 6,
        "equal": 162,
        "regressed": 2
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r9-guide-mask-lr3e6",
        "group": "relevant_document_above_8192_tokens",
        "queries": 166,
        "candidate_score": 0.845615412812933,
        "baseline_score": 0.838579012336451,
        "delta": 0.00703640047648197,
        "paired_bootstrap_95_interval": [
          -0.0023528919316009025,
          0.016606970299408556
        ],
        "improved": 13,
        "equal": 149,
        "regressed": 4
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r13-bge-kd-w01",
        "group": "all",
        "queries": 336,
        "candidate_score": 0.8836249982557198,
        "baseline_score": 0.8804095047305854,
        "delta": 0.0032154935251345294,
        "paired_bootstrap_95_interval": [
          -0.002654427117917938,
          0.009620371358263215
        ],
        "improved": 16,
        "equal": 311,
        "regressed": 9
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r13-bge-kd-w01",
        "group": "relevant_document_at_most_8192_tokens",
        "queries": 170,
        "candidate_score": 0.9207402405116178,
        "baseline_score": 0.9174067251903617,
        "delta": 0.00333351532125612,
        "paired_bootstrap_95_interval": [
          -0.004205604809041235,
          0.012017521119574765
        ],
        "improved": 5,
        "equal": 162,
        "regressed": 3
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r13-bge-kd-w01",
        "group": "relevant_document_above_8192_tokens",
        "queries": 166,
        "candidate_score": 0.845615412812933,
        "baseline_score": 0.8425207849826216,
        "delta": 0.003094627830311214,
        "paired_bootstrap_95_interval": [
          -0.006415175063604415,
          0.012801972665767698
        ],
        "improved": 11,
        "equal": 149,
        "regressed": 6
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r20-t2ranking-zh8192",
        "group": "all",
        "queries": 336,
        "candidate_score": 0.8836249982557198,
        "baseline_score": 0.8847132026337041,
        "delta": -0.0010882043779841832,
        "paired_bootstrap_95_interval": [
          -0.005630972134979018,
          0.003024722199180693
        ],
        "improved": 2,
        "equal": 326,
        "regressed": 8
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r20-t2ranking-zh8192",
        "group": "relevant_document_at_most_8192_tokens",
        "queries": 170,
        "candidate_score": 0.9207402405116178,
        "baseline_score": 0.9214058122337676,
        "delta": -0.0006655717221497551,
        "paired_bootstrap_95_interval": [
          -0.00717857607088874,
          0.00584743262658923
        ],
        "improved": 1,
        "equal": 166,
        "regressed": 3
      },
      {
        "candidate": "r24-new-queries",
        "baseline": "r20-t2ranking-zh8192",
        "group": "relevant_document_above_8192_tokens",
        "queries": 166,
        "candidate_score": 0.845615412812933,
        "baseline_score": 0.8471364337661693,
        "delta": -0.001521020953236308,
        "paired_bootstrap_95_interval": [
          -0.007675908408223043,
          0.004130748431944928
        ],
        "improved": 1,
        "equal": 160,
        "regressed": 5
      }
    ],
    "r20_reference_content_sha256": "dbbba2a09857723e44f77f6084a22cbc5e49b6b281d95eaa545abbe546edde04",
    "r20_reference_manifest_sha256": "99e03e6512f7d77defc4f43157822f3545742cd677824474f2bf2fcd3d3fafa6",
    "runner_sha256": "5c7b45cbfb3619db1eff32b6232eb87a990eae6eb850670946f7f0e5607b20d6",
    "launch_manifest_sha256": "7c2606f42cab34d637d58d3235dafc4b3290154d818ce4daddd3b078e4fe3688",
    "lengths": {
      "minimum": 2662,
      "maximum": 23856,
      "sum": 2887533,
      "count": 336
    },
    "limitations": [
      "Conditional development-query uncertainty; fixed corpus retained.",
      "Intervals are exploratory and not adjusted for task/subgroup multiplicity.",
      "A single long retrieval task does not establish all long-context or 32768-token endpoint capabilities."
    ]
  },
  "scope": "Previously exposed development validation, not fresh confirmation. Native FP32 MATH SDPA, not ONNX benchmark scores."
}