Spaces:
Running
Running
| { | |
| "datasets": [ | |
| "sst2", | |
| "ag_news", | |
| "emotion", | |
| "banking77" | |
| ], | |
| "zero": [ | |
| { | |
| "name": "typesafe/jev (bare labels)", | |
| "acc": { | |
| "sst2": 0.898, | |
| "ag_news": 0.8828, | |
| "emotion": 0.586, | |
| "banking77": 0.7836 | |
| }, | |
| "lat": { | |
| "sst2": 381.7, | |
| "ag_news": 382.1, | |
| "emotion": 374.0, | |
| "banking77": 392.1 | |
| }, | |
| "ram": null, | |
| "cloud": true | |
| }, | |
| { | |
| "name": "typesafe/jev (enriched)", | |
| "acc": { | |
| "sst2": 0.9164, | |
| "ag_news": 0.8864, | |
| "emotion": 0.59, | |
| "banking77": 0.7784 | |
| }, | |
| "lat": { | |
| "sst2": 381.7, | |
| "ag_news": 382.1, | |
| "emotion": 374.0, | |
| "banking77": 392.1 | |
| }, | |
| "ram": null, | |
| "cloud": true | |
| }, | |
| { | |
| "name": "Cross-encoder 149M (bare)", | |
| "acc": { | |
| "sst2": 0.8548, | |
| "ag_news": 0.8444, | |
| "emotion": 0.762, | |
| "banking77": 0.6884 | |
| }, | |
| "lat": { | |
| "sst2": 50.0, | |
| "ag_news": 311.2, | |
| "emotion": 216.0, | |
| "banking77": 615.8 | |
| }, | |
| "ram": null | |
| }, | |
| { | |
| "name": "Cross-encoder 149M (enriched)", | |
| "acc": { | |
| "sst2": 0.8448, | |
| "ag_news": 0.8464, | |
| "emotion": 0.7172, | |
| "banking77": 0.5688 | |
| }, | |
| "lat": { | |
| "sst2": 71.9, | |
| "ag_news": 378.7, | |
| "emotion": 269.5, | |
| "banking77": 707.4 | |
| }, | |
| "ram": null | |
| }, | |
| { | |
| "name": "Bi-encoder + 8-float head (bare)", | |
| "acc": { | |
| "sst2": 0.8112, | |
| "ag_news": 0.6632, | |
| "emotion": 0.4364, | |
| "banking77": 0.6324 | |
| }, | |
| "lat": { | |
| "sst2": 15.685, | |
| "ag_news": 15.685, | |
| "emotion": 15.685, | |
| "banking77": 15.685 | |
| }, | |
| "ram": 490 | |
| }, | |
| { | |
| "name": "Bi-encoder + 8-float head (enriched)", | |
| "acc": { | |
| "sst2": 0.8284, | |
| "ag_news": 0.8, | |
| "emotion": 0.5084, | |
| "banking77": 0.6576 | |
| }, | |
| "lat": { | |
| "sst2": 15.685, | |
| "ag_news": 15.685, | |
| "emotion": 15.685, | |
| "banking77": 15.685 | |
| }, | |
| "ram": 490 | |
| }, | |
| { | |
| "name": "Qwen2.5-0.5B generative", | |
| "acc": { | |
| "sst2": 0.844, | |
| "ag_news": 0.6556, | |
| "emotion": 0.3412, | |
| "banking77": 0.142 | |
| }, | |
| "lat": { | |
| "sst2": 103.5, | |
| "ag_news": 110.1, | |
| "emotion": 109.9, | |
| "banking77": 314.2 | |
| }, | |
| "ram": 2382 | |
| }, | |
| { | |
| "name": "Laya 421M (open weights, MPS)", | |
| "acc": { | |
| "sst2": 0.8652, | |
| "ag_news": 0.9392, | |
| "emotion": 0.582, | |
| "banking77": 0.5428 | |
| }, | |
| "lat": { | |
| "sst2": 19.4, | |
| "ag_news": 25.8, | |
| "emotion": 21.8, | |
| "banking77": 86.3 | |
| }, | |
| "ram": null, | |
| "device": "mps", | |
| "note": "laya.load() auto-selects cuda>mps>cpu and chose mps; every other row here is pure CPU, batch=1. CPU re-measure in `laya_cpu`." | |
| } | |
| ], | |
| "few": [ | |
| { | |
| "name": "static + 8-float head (2k labels)", | |
| "acc": { | |
| "sst2": 0.7832, | |
| "ag_news": 0.8772, | |
| "emotion": 0.6728, | |
| "banking77": 0.7936 | |
| }, | |
| "lat": { | |
| "sst2": 0.1067, | |
| "ag_news": 0.1067, | |
| "emotion": 0.1067, | |
| "banking77": 0.1067 | |
| }, | |
| "ram": 82.5 | |
| }, | |
| { | |
| "name": "minilm + 8-float head (2k labels)", | |
| "acc": { | |
| "sst2": 0.7972, | |
| "ag_news": 0.8872, | |
| "emotion": 0.6424, | |
| "banking77": 0.8512 | |
| }, | |
| "lat": { | |
| "sst2": 2.8222, | |
| "ag_news": 2.8222, | |
| "emotion": 2.8222, | |
| "banking77": 2.8222 | |
| }, | |
| "ram": 451.7 | |
| }, | |
| { | |
| "name": "mbert-embed + 8-float head (2k labels, proper score)", | |
| "acc": { | |
| "sst2": 0.876, | |
| "ag_news": 0.8976, | |
| "emotion": 0.6824, | |
| "banking77": 0.8892 | |
| }, | |
| "lat": { | |
| "sst2": 15.767900000000001, | |
| "ag_news": 15.767900000000001, | |
| "emotion": 15.767900000000001, | |
| "banking77": 15.767900000000001 | |
| }, | |
| "ram": 489.5 | |
| } | |
| ], | |
| "failures": [ | |
| [ | |
| "Template ensembling (3 NLI hypotheses)", | |
| "0.0 pp", | |
| "errors across templates are correlated" | |
| ], | |
| [ | |
| "Evolutionary weight search, honest LODO", | |
| "-1.5 pp", | |
| "negative transfer; helps only when the base signal is weak" | |
| ], | |
| [ | |
| "Cross-encoder 149M -> 395M", | |
| "+2.8 / -3.2 / +0.2 pp", | |
| "no net gain for 3x the compute" | |
| ], | |
| [ | |
| "Listwise rerank with Qwen2.5-1.5B (top-5)", | |
| "-1.0 pp", | |
| "weaker than the cross-encoder alone" | |
| ], | |
| [ | |
| "Column-centering NLI scores", | |
| "0.0 pp macro", | |
| "BANKING77 +2.6, Emotion -3.1" | |
| ], | |
| [ | |
| "Asymmetric query/document prefixes", | |
| "-1.8 pp", | |
| "classification by similarity is not retrieval" | |
| ], | |
| [ | |
| "Distil cross-encoder into bi-encoder head", | |
| "-5.4 pp", | |
| "collapses on Emotion (-18.8 pp)" | |
| ], | |
| [ | |
| "Raw ModernBERT-base (MLM) as sentence encoder", | |
| "35.0 %", | |
| "worse than an 8M static embedding" | |
| ], | |
| [ | |
| "ES objective = accuracy only", | |
| "+1.0 pp when swapped", | |
| "log score is a smoother signal; calibration was already fine" | |
| ] | |
| ], | |
| "jev_cost_usd": 0.2841, | |
| "jev_tokens": 6764108, | |
| "extra": { | |
| "laya_shortlist": { | |
| "base": 0.5428, | |
| "base_lat": 86.3, | |
| "fixed": 0.608, | |
| "fixed_lat": 39.8, | |
| "ceiling": 0.9784, | |
| "device": "mps", | |
| "cpu": { | |
| "base": 0.536, | |
| "base_lat": 309.0, | |
| "fixed": 0.62, | |
| "fixed_lat": 169.9, | |
| "n": 250, | |
| "ceiling": 0.98 | |
| } | |
| }, | |
| "chinese": { | |
| "multilingual": { | |
| "acc": 0.39166666666666666, | |
| "conf": 0.7564826666666666, | |
| "lat": 14.3 | |
| }, | |
| "english": { | |
| "acc": 0.225, | |
| "conf": 0.9681070000000002, | |
| "lat": 42.5 | |
| }, | |
| "random": 0.06666666666666667 | |
| }, | |
| "objective": { | |
| "acc": { | |
| "per": { | |
| "sst2": { | |
| "acc": 0.858, | |
| "ece": 0.013683630180358876, | |
| "temp": 4.51, | |
| "nll": 0.3143 | |
| }, | |
| "ag_news": { | |
| "acc": 0.8948, | |
| "ece": 0.03082646280527116, | |
| "temp": 4.51, | |
| "nll": 0.3369 | |
| }, | |
| "emotion": { | |
| "acc": 0.6616, | |
| "ece": 0.01806434515714644, | |
| "temp": 4.87, | |
| "nll": 0.9243 | |
| }, | |
| "banking77": { | |
| "acc": 0.888, | |
| "ece": 0.010833000487089142, | |
| "temp": 3.59, | |
| "nll": 0.384 | |
| } | |
| }, | |
| "macro_acc": 0.8256, | |
| "mean_ece": 0.018351859657466406, | |
| "mean_nll": 0.48987499999999995, | |
| "weights": { | |
| "proto": 3.9147, | |
| "desc": 0.8717, | |
| "knn1": 2.5684, | |
| "knnk": 2.8151, | |
| "margin": 0.9435, | |
| "prior": 0.3723, | |
| "white": -0.9152, | |
| "tight": 0.0329 | |
| } | |
| }, | |
| "proper": { | |
| "per": { | |
| "sst2": { | |
| "acc": 0.876, | |
| "ece": 0.01733291234970094, | |
| "temp": 5.25, | |
| "nll": 0.2758 | |
| }, | |
| "ag_news": { | |
| "acc": 0.8976, | |
| "ece": 0.02284402102231979, | |
| "temp": 4.18, | |
| "nll": 0.3222 | |
| }, | |
| "emotion": { | |
| "acc": 0.6824, | |
| "ece": 0.021511850953102124, | |
| "temp": 4.51, | |
| "nll": 0.8761 | |
| }, | |
| "banking77": { | |
| "acc": 0.8892, | |
| "ece": 0.010253664314746886, | |
| "temp": 3.33, | |
| "nll": 0.375 | |
| } | |
| }, | |
| "macro_acc": 0.8363, | |
| "mean_ece": 0.017985612159967437, | |
| "mean_nll": 0.462275, | |
| "weights": { | |
| "proto": 1.5202, | |
| "desc": 0.8578, | |
| "knn1": 1.6066, | |
| "knnk": 2.7418, | |
| "margin": 2.3915, | |
| "prior": 0.4453, | |
| "white": 0.4608, | |
| "tight": -4.7245 | |
| } | |
| } | |
| }, | |
| "laya_cpu": { | |
| "device": "cpu", | |
| "n_per_dataset": 250, | |
| "acc": { | |
| "sst2": 0.856, | |
| "ag_news": 0.948, | |
| "emotion": 0.588, | |
| "banking77": 0.536 | |
| }, | |
| "lat": { | |
| "sst2": 59.5, | |
| "ag_news": 78.0, | |
| "emotion": 61.9, | |
| "banking77": 309.0 | |
| }, | |
| "note": "Like-for-like with every other CPU row. ~3x the MPS latencies." | |
| } | |
| } | |
| } |