{
 "datasets": [
  "sst2",
  "ag_news",
  "emotion",
  "banking77"
 ],
 "zero": [
  {
   "name": "typesafe/jev (bare labels)",
   "acc": {
    "sst2": 0.898,
    "ag_news": 0.8828,
    "emotion": 0.586,
    "banking77": 0.7836
   },
   "lat": {
    "sst2": 381.7,
    "ag_news": 382.1,
    "emotion": 374.0,
    "banking77": 392.1
   },
   "ram": null,
   "cloud": true
  },
  {
   "name": "typesafe/jev (enriched)",
   "acc": {
    "sst2": 0.9164,
    "ag_news": 0.8864,
    "emotion": 0.59,
    "banking77": 0.7784
   },
   "lat": {
    "sst2": 381.7,
    "ag_news": 382.1,
    "emotion": 374.0,
    "banking77": 392.1
   },
   "ram": null,
   "cloud": true
  },
  {
   "name": "Cross-encoder 149M (bare)",
   "acc": {
    "sst2": 0.8548,
    "ag_news": 0.8444,
    "emotion": 0.762,
    "banking77": 0.6884
   },
   "lat": {
    "sst2": 50.0,
    "ag_news": 311.2,
    "emotion": 216.0,
    "banking77": 615.8
   },
   "ram": null
  },
  {
   "name": "Cross-encoder 149M (enriched)",
   "acc": {
    "sst2": 0.8448,
    "ag_news": 0.8464,
    "emotion": 0.7172,
    "banking77": 0.5688
   },
   "lat": {
    "sst2": 71.9,
    "ag_news": 378.7,
    "emotion": 269.5,
    "banking77": 707.4
   },
   "ram": null
  },
  {
   "name": "Bi-encoder + 8-float head (bare)",
   "acc": {
    "sst2": 0.8112,
    "ag_news": 0.6632,
    "emotion": 0.4364,
    "banking77": 0.6324
   },
   "lat": {
    "sst2": 15.685,
    "ag_news": 15.685,
    "emotion": 15.685,
    "banking77": 15.685
   },
   "ram": 490
  },
  {
   "name": "Bi-encoder + 8-float head (enriched)",
   "acc": {
    "sst2": 0.8284,
    "ag_news": 0.8,
    "emotion": 0.5084,
    "banking77": 0.6576
   },
   "lat": {
    "sst2": 15.685,
    "ag_news": 15.685,
    "emotion": 15.685,
    "banking77": 15.685
   },
   "ram": 490
  },
  {
   "name": "Qwen2.5-0.5B generative",
   "acc": {
    "sst2": 0.844,
    "ag_news": 0.6556,
    "emotion": 0.3412,
    "banking77": 0.142
   },
   "lat": {
    "sst2": 103.5,
    "ag_news": 110.1,
    "emotion": 109.9,
    "banking77": 314.2
   },
   "ram": 2382
  },
  {
   "name": "Laya 421M (open weights, MPS)",
   "acc": {
    "sst2": 0.8652,
    "ag_news": 0.9392,
    "emotion": 0.582,
    "banking77": 0.5428
   },
   "lat": {
    "sst2": 19.4,
    "ag_news": 25.8,
    "emotion": 21.8,
    "banking77": 86.3
   },
   "ram": null,
   "device": "mps",
   "note": "laya.load() auto-selects cuda>mps>cpu and chose mps; every other row here is pure CPU, batch=1. CPU re-measure in `laya_cpu`."
  }
 ],
 "few": [
  {
   "name": "static + 8-float head (2k labels)",
   "acc": {
    "sst2": 0.7832,
    "ag_news": 0.8772,
    "emotion": 0.6728,
    "banking77": 0.7936
   },
   "lat": {
    "sst2": 0.1067,
    "ag_news": 0.1067,
    "emotion": 0.1067,
    "banking77": 0.1067
   },
   "ram": 82.5
  },
  {
   "name": "minilm + 8-float head (2k labels)",
   "acc": {
    "sst2": 0.7972,
    "ag_news": 0.8872,
    "emotion": 0.6424,
    "banking77": 0.8512
   },
   "lat": {
    "sst2": 2.8222,
    "ag_news": 2.8222,
    "emotion": 2.8222,
    "banking77": 2.8222
   },
   "ram": 451.7
  },
  {
   "name": "mbert-embed + 8-float head (2k labels, proper score)",
   "acc": {
    "sst2": 0.876,
    "ag_news": 0.8976,
    "emotion": 0.6824,
    "banking77": 0.8892
   },
   "lat": {
    "sst2": 15.767900000000001,
    "ag_news": 15.767900000000001,
    "emotion": 15.767900000000001,
    "banking77": 15.767900000000001
   },
   "ram": 489.5
  }
 ],
 "failures": [
  [
   "Template ensembling (3 NLI hypotheses)",
   "0.0 pp",
   "errors across templates are correlated"
  ],
  [
   "Evolutionary weight search, honest LODO",
   "-1.5 pp",
   "negative transfer; helps only when the base signal is weak"
  ],
  [
   "Cross-encoder 149M -> 395M",
   "+2.8 / -3.2 / +0.2 pp",
   "no net gain for 3x the compute"
  ],
  [
   "Listwise rerank with Qwen2.5-1.5B (top-5)",
   "-1.0 pp",
   "weaker than the cross-encoder alone"
  ],
  [
   "Column-centering NLI scores",
   "0.0 pp macro",
   "BANKING77 +2.6, Emotion -3.1"
  ],
  [
   "Asymmetric query/document prefixes",
   "-1.8 pp",
   "classification by similarity is not retrieval"
  ],
  [
   "Distil cross-encoder into bi-encoder head",
   "-5.4 pp",
   "collapses on Emotion (-18.8 pp)"
  ],
  [
   "Raw ModernBERT-base (MLM) as sentence encoder",
   "35.0 %",
   "worse than an 8M static embedding"
  ],
  [
   "ES objective = accuracy only",
   "+1.0 pp when swapped",
   "log score is a smoother signal; calibration was already fine"
  ]
 ],
 "jev_cost_usd": 0.2841,
 "jev_tokens": 6764108,
 "extra": {
  "laya_shortlist": {
   "base": 0.5428,
   "base_lat": 86.3,
   "fixed": 0.608,
   "fixed_lat": 39.8,
   "ceiling": 0.9784,
   "device": "mps",
   "cpu": {
    "base": 0.536,
    "base_lat": 309.0,
    "fixed": 0.62,
    "fixed_lat": 169.9,
    "n": 250,
    "ceiling": 0.98
   }
  },
  "chinese": {
   "multilingual": {
    "acc": 0.39166666666666666,
    "conf": 0.7564826666666666,
    "lat": 14.3
   },
   "english": {
    "acc": 0.225,
    "conf": 0.9681070000000002,
    "lat": 42.5
   },
   "random": 0.06666666666666667
  },
  "objective": {
   "acc": {
    "per": {
     "sst2": {
      "acc": 0.858,
      "ece": 0.013683630180358876,
      "temp": 4.51,
      "nll": 0.3143
     },
     "ag_news": {
      "acc": 0.8948,
      "ece": 0.03082646280527116,
      "temp": 4.51,
      "nll": 0.3369
     },
     "emotion": {
      "acc": 0.6616,
      "ece": 0.01806434515714644,
      "temp": 4.87,
      "nll": 0.9243
     },
     "banking77": {
      "acc": 0.888,
      "ece": 0.010833000487089142,
      "temp": 3.59,
      "nll": 0.384
     }
    },
    "macro_acc": 0.8256,
    "mean_ece": 0.018351859657466406,
    "mean_nll": 0.48987499999999995,
    "weights": {
     "proto": 3.9147,
     "desc": 0.8717,
     "knn1": 2.5684,
     "knnk": 2.8151,
     "margin": 0.9435,
     "prior": 0.3723,
     "white": -0.9152,
     "tight": 0.0329
    }
   },
   "proper": {
    "per": {
     "sst2": {
      "acc": 0.876,
      "ece": 0.01733291234970094,
      "temp": 5.25,
      "nll": 0.2758
     },
     "ag_news": {
      "acc": 0.8976,
      "ece": 0.02284402102231979,
      "temp": 4.18,
      "nll": 0.3222
     },
     "emotion": {
      "acc": 0.6824,
      "ece": 0.021511850953102124,
      "temp": 4.51,
      "nll": 0.8761
     },
     "banking77": {
      "acc": 0.8892,
      "ece": 0.010253664314746886,
      "temp": 3.33,
      "nll": 0.375
     }
    },
    "macro_acc": 0.8363,
    "mean_ece": 0.017985612159967437,
    "mean_nll": 0.462275,
    "weights": {
     "proto": 1.5202,
     "desc": 0.8578,
     "knn1": 1.6066,
     "knnk": 2.7418,
     "margin": 2.3915,
     "prior": 0.4453,
     "white": 0.4608,
     "tight": -4.7245
    }
   }
  },
  "laya_cpu": {
   "device": "cpu",
   "n_per_dataset": 250,
   "acc": {
    "sst2": 0.856,
    "ag_news": 0.948,
    "emotion": 0.588,
    "banking77": 0.536
   },
   "lat": {
    "sst2": 59.5,
    "ag_news": 78.0,
    "emotion": 61.9,
    "banking77": 309.0
   },
   "note": "Like-for-like with every other CPU row. ~3x the MPS latencies."
  }
 }
}