| { |
| "model": "models/laya-multilingual-v11", |
| "validation": { |
| "banking77": { |
| "acc": 0.908, |
| "n": 500 |
| }, |
| "massive": { |
| "acc": 0.8267, |
| "n": 600 |
| }, |
| "sentiment": { |
| "acc": 0.5983, |
| "n": 600 |
| }, |
| "emotion": { |
| "acc": 0.504, |
| "n": 500 |
| }, |
| "ag_news": { |
| "acc": 0.91, |
| "n": 500 |
| } |
| }, |
| "heldout": { |
| "test/ag_news": { |
| "acc": 0.921, |
| "n": 2000, |
| "ece": 0.0078 |
| }, |
| "test/emotion": { |
| "acc": 0.5305, |
| "n": 2000, |
| "ece": 0.148 |
| }, |
| "test/banking77": { |
| "acc": 0.9185, |
| "n": 2000, |
| "ece": 0.0762 |
| }, |
| "test/typed_decisions": { |
| "acc": 0.775, |
| "n": 2000, |
| "ece": null |
| }, |
| "amazon_massive_intent/de": { |
| "acc": 0.8267, |
| "n": 150, |
| "ece": 0.0715 |
| }, |
| "amazon_massive_intent/en": { |
| "acc": 0.8467, |
| "n": 150, |
| "ece": 0.053 |
| }, |
| "amazon_massive_intent/fr": { |
| "acc": 0.8067, |
| "n": 150, |
| "ece": 0.0371 |
| }, |
| "amazon_massive_intent/es": { |
| "acc": 0.8333, |
| "n": 150, |
| "ece": 0.0745 |
| }, |
| "amazon_massive_intent/it": { |
| "acc": 0.8133, |
| "n": 150, |
| "ece": 0.0607 |
| }, |
| "amazon_massive_intent/tr": { |
| "acc": 0.8133, |
| "n": 150, |
| "ece": 0.0957 |
| }, |
| "amazon_massive_intent/pl": { |
| "acc": 0.8333, |
| "n": 150, |
| "ece": 0.0897 |
| }, |
| "amazon_massive_intent/ru": { |
| "acc": 0.8467, |
| "n": 150, |
| "ece": 0.0704 |
| }, |
| "amazon_massive_intent/ja": { |
| "acc": 0.8333, |
| "n": 150, |
| "ece": 0.0681 |
| }, |
| "amazon_massive_intent/zh-CN": { |
| "acc": 0.82, |
| "n": 150, |
| "ece": 0.0588 |
| }, |
| "amazon_massive_intent/ar": { |
| "acc": 0.74, |
| "n": 150, |
| "ece": 0.0654 |
| }, |
| "amazon_massive_intent/hi": { |
| "acc": 0.78, |
| "n": 150, |
| "ece": 0.0767 |
| }, |
| "multilingual_sentiments/ar": { |
| "acc": 0.6333, |
| "n": 150, |
| "ece": 0.1504 |
| }, |
| "multilingual_sentiments/zh": { |
| "acc": 0.6133, |
| "n": 150, |
| "ece": 0.1625 |
| }, |
| "multilingual_sentiments/en": { |
| "acc": 0.72, |
| "n": 150, |
| "ece": 0.0776 |
| }, |
| "multilingual_sentiments/fr": { |
| "acc": 0.6267, |
| "n": 150, |
| "ece": 0.0915 |
| }, |
| "multilingual_sentiments/de": { |
| "acc": 0.56, |
| "n": 150, |
| "ece": 0.1318 |
| }, |
| "multilingual_sentiments/hi": { |
| "acc": 0.54, |
| "n": 150, |
| "ece": 0.135 |
| }, |
| "multilingual_sentiments/id": { |
| "acc": 0.78, |
| "n": 150, |
| "ece": 0.0798 |
| }, |
| "multilingual_sentiments/it": { |
| "acc": 0.6467, |
| "n": 150, |
| "ece": 0.0752 |
| }, |
| "multilingual_sentiments/ja": { |
| "acc": 0.68, |
| "n": 150, |
| "ece": 0.0907 |
| }, |
| "multilingual_sentiments/ms": { |
| "acc": 0.5333, |
| "n": 150, |
| "ece": 0.168 |
| }, |
| "multilingual_sentiments/pt": { |
| "acc": 0.64, |
| "n": 150, |
| "ece": 0.1151 |
| }, |
| "multilingual_sentiments/es": { |
| "acc": 0.6, |
| "n": 150, |
| "ece": 0.2087 |
| }, |
| "go_emotions/en": { |
| "acc": 0.4533, |
| "n": 150, |
| "ece": 0.0881 |
| }, |
| "multi_hatecheck/en": { |
| "acc": 0.6467, |
| "n": 150, |
| "ece": 0.1259 |
| }, |
| "multi_hatecheck/de": { |
| "acc": 0.6733, |
| "n": 150, |
| "ece": 0.0606 |
| }, |
| "multi_hatecheck/fr": { |
| "acc": 0.66, |
| "n": 150, |
| "ece": 0.1077 |
| }, |
| "multi_hatecheck/es": { |
| "acc": 0.6467, |
| "n": 150, |
| "ece": 0.1149 |
| }, |
| "multi_hatecheck/it": { |
| "acc": 0.62, |
| "n": 150, |
| "ece": 0.1395 |
| }, |
| "multi_hatecheck/nl": { |
| "acc": 0.6467, |
| "n": 150, |
| "ece": 0.0941 |
| }, |
| "multi_hatecheck/pl": { |
| "acc": 0.6533, |
| "n": 150, |
| "ece": 0.1096 |
| }, |
| "multi_hatecheck/pt": { |
| "acc": 0.6733, |
| "n": 150, |
| "ece": 0.0802 |
| }, |
| "multi_hatecheck/zh": { |
| "acc": 0.64, |
| "n": 150, |
| "ece": 0.1316 |
| }, |
| "multi_hatecheck/ar": { |
| "acc": 0.6667, |
| "n": 150, |
| "ece": 0.0891 |
| }, |
| "multi_hatecheck/hi": { |
| "acc": 0.58, |
| "n": 150, |
| "ece": 0.1111 |
| }, |
| "sib200/en": { |
| "acc": 0.8467, |
| "n": 150, |
| "ece": 0.062 |
| }, |
| "sib200/de": { |
| "acc": 0.8267, |
| "n": 150, |
| "ece": 0.0645 |
| }, |
| "sib200/ar": { |
| "acc": 0.78, |
| "n": 150, |
| "ece": 0.0886 |
| }, |
| "sib200/hi": { |
| "acc": 0.7333, |
| "n": 150, |
| "ece": 0.1103 |
| }, |
| "hwu64/en": { |
| "acc": 0.8533, |
| "n": 150, |
| "ece": 0.0795 |
| }, |
| "indonli/id": { |
| "acc": 0.7, |
| "n": 150, |
| "ece": 0.1273 |
| }, |
| "farstail/fa": { |
| "acc": 0.6533, |
| "n": 150, |
| "ece": 0.1512 |
| }, |
| "belebele/en": { |
| "acc": 0.3133, |
| "n": 150, |
| "ece": 0.2754 |
| }, |
| "belebele/de": { |
| "acc": 0.32, |
| "n": 150, |
| "ece": 0.3055 |
| }, |
| "belebele/ar": { |
| "acc": 0.26, |
| "n": 150, |
| "ece": 0.3235 |
| }, |
| "belebele/hi": { |
| "acc": 0.2267, |
| "n": 150, |
| "ece": 0.3503 |
| }, |
| "semrel/en": { |
| "acc": 0.28, |
| "n": 150, |
| "ece": 0.2875 |
| }, |
| "semrel/ar": { |
| "acc": 0.2733, |
| "n": 150, |
| "ece": 0.3183 |
| }, |
| "semrel/hi": { |
| "acc": 0.2933, |
| "n": 150, |
| "ece": 0.2443 |
| }, |
| "categories:sentiment/en": { |
| "acc": 0.8533, |
| "n": 150, |
| "ece": 0.0819, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "dca5d61987d3fa0388ca894850f455db283f3af8eb2fb7adfedba9c538ac56d2" |
| }, |
| "categories:sentiment/zh": { |
| "acc": 0.9, |
| "n": 150, |
| "ece": 0.0304, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "67bb3b2842114c85ce5c5d39ea9254131114e17489b664f993dae78b31302926" |
| }, |
| "categories:emotion/de": { |
| "acc": 0.4867, |
| "n": 150, |
| "ece": 0.1301, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "7e99e8b186acee316711f3c76148ada3cdbf8d82e0d5168107bdf3aeb1ec7b98" |
| }, |
| "categories:emotion/en": { |
| "acc": 0.64, |
| "n": 150, |
| "ece": 0.0991, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "226fc02462825a0b494407518388c06292c177e6c3941abd626a1262f05194a5" |
| }, |
| "categories:emotion/es": { |
| "acc": 0.62, |
| "n": 150, |
| "ece": 0.0918, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "d9760e612a2e182ee1d4f5521423471a67521dad1d71de62c9fe407be8e99df4" |
| }, |
| "categories:emotion/fr": { |
| "acc": 0.8267, |
| "n": 150, |
| "ece": 0.0822, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "a0dc4d4080bee5f6215ac6ca1d7ff831698cd6fadf195942572e73b79ad45e37" |
| }, |
| "categories:emotion/hi": { |
| "acc": 0.86, |
| "n": 150, |
| "ece": 0.0626, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "082b08b1d0eea74ec0138b1424b76dafc8ee95f773c815e3a0412b0a0da9b697" |
| }, |
| "categories:emotion/pt": { |
| "acc": 0.56, |
| "n": 150, |
| "ece": 0.1237, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "e01c851ad2f100c55168c89ba288fccf1e17466db4265ef7c7e2372b545f3bc1" |
| }, |
| "categories:emotion/ru": { |
| "acc": 0.74, |
| "n": 150, |
| "ece": 0.0869, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "3b1f698d6af7861ea4b4789a79322cd34a9aa691d128768eec837d5fa66a5de6" |
| }, |
| "categories:emotion/zh": { |
| "acc": 0.5533, |
| "n": 150, |
| "ece": 0.1448, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "22aeaf3ca1b64322b3afa8d61b40fc0b2b6a1e002181821a5ac4145b59310a38" |
| }, |
| "categories:complaint/en": { |
| "acc": 0.82, |
| "n": 150, |
| "ece": 0.0824, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "333bda35a5d1cd5d5651e0f24a3f3829753ec834b80c19965f5b59945117807a" |
| }, |
| "categories:nli/en": { |
| "acc": 0.8, |
| "n": 150, |
| "ece": 0.071, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "17b38e41fc99c3c8d4458f0857eb437902adaaeccd26eab81c2a20f3b46ba018" |
| }, |
| "categories:nli/ja": { |
| "acc": 0.66, |
| "n": 150, |
| "ece": 0.1421, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "ea2af8de3f6d5bacfc81d6bc137bce5d7a69c6f272b799ca18a651953abef181" |
| }, |
| "categories:nli/tr": { |
| "acc": 0.8467, |
| "n": 150, |
| "ece": 0.0535, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "032946c87c90d553301050355ae43e8ed095b451670fde3638efa944ec6a8276" |
| }, |
| "categories:safety/en": { |
| "acc": 0.88, |
| "n": 150, |
| "ece": 0.0739, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "1955208993d89e14a54edf34f9e3f7d432ad68c49bdb583083fb512310f945ac" |
| }, |
| "categories:reading/en": { |
| "acc": 0.9333, |
| "n": 150, |
| "ece": 0.0519, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "57cecee470248e19e1ec362b39552c3e3bbaf4ced40909e27b95d6c2d03ed809" |
| }, |
| "categories:similarity/pt": { |
| "acc": 0.88, |
| "n": 150, |
| "ece": 0.0631, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "d8a799fb3b53bda0265399aacfb5fcb7939a670120af64598f1e15db2784c69c" |
| }, |
| "categories:topic/en": { |
| "acc": 0.6467, |
| "n": 150, |
| "ece": 0.0898, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "6515d51e9240f29eaa968c8b6402dabae247d9f04095e295360003ba5db2fcbd" |
| }, |
| "categories:intent/en": { |
| "acc": 0.8467, |
| "n": 150, |
| "ece": 0.065, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "1fa7c10ebe1c68fe4a0ffa0c899de81121294f7423fe60011309b0aea9af8f81" |
| }, |
| "categories:intent/nl": { |
| "acc": 0.5333, |
| "n": 150, |
| "ece": 0.0941, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "1f327bd9c76abec1a83d2b51e9947fc1636364e47cc7b5420f08db664b9dc899" |
| }, |
| "categories:intent/tr": { |
| "acc": 1.0, |
| "n": 150, |
| "ece": 0.0002, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "134c60c45d74b27253775ff1b26d209c6dc916cbda0c150999e13584dcf6cce8" |
| }, |
| "categories:stance/en": { |
| "acc": 0.98, |
| "n": 150, |
| "ece": 0.0696, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "e28f7bba23222687ea8d3112afadc7f6a06270eb3faccf219d8ae5feea3b229c" |
| }, |
| "categories:formality/ja": { |
| "acc": 0.9067, |
| "n": 150, |
| "ece": 0.0558, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "e9fe2a0258eec42bbc6ed0ca325063c31eb44df7f5a6dc117af484a2fc3a2beb" |
| }, |
| "categories:formality/tr": { |
| "acc": 1.0, |
| "n": 150, |
| "ece": 0.0025, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "747695931d2a62fe8c2952a9cc66208110f6d968d63bc3603e5d38cd794e6ecc" |
| }, |
| "categories:urgency/en": { |
| "acc": 0.9867, |
| "n": 150, |
| "ece": 0.0247, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "ace94904043bf4d44160d85f936837595b0cba4a9a617fc5e918062becf70271" |
| }, |
| "categories:fact_check/en": { |
| "acc": 0.4667, |
| "n": 150, |
| "ece": 0.2808, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "ac4c49eb82015d55086f84b8643efdd924bd33880320136b845daa16c0ec8ab1" |
| }, |
| "categories:pii/ar": { |
| "acc": 0.9333, |
| "n": 150, |
| "ece": 0.0495, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "323112f111aa8c9c401e70d0f081ea6628b5230e57d741ac291694e16708c2b9" |
| }, |
| "categories:pii/de": { |
| "acc": 0.8867, |
| "n": 150, |
| "ece": 0.0482, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "52145c7106ce9f85fa46c267cdace80c42181e148d9cb45594263718f5a49d92" |
| }, |
| "categories:pii/en": { |
| "acc": 0.9333, |
| "n": 150, |
| "ece": 0.0413, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "10b8d7d2c36445bbeecdfded7c937f4ca228c43ab80c515dd038be0f658a2cbe" |
| }, |
| "categories:pii/es": { |
| "acc": 0.8867, |
| "n": 150, |
| "ece": 0.07, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "853091fcb462b7b89e00759c1085ad942fb99eb455c18f4c5ecd8db63cb20e6e" |
| }, |
| "categories:pii/fr": { |
| "acc": 0.9133, |
| "n": 150, |
| "ece": 0.0322, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "2be7c20bc628fd30fe60af29d6e457b9ce63cba3d8961d3528007cacba4fcbb0" |
| }, |
| "categories:pii/it": { |
| "acc": 0.9, |
| "n": 150, |
| "ece": 0.0357, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "632ad4c14c949eb65c7261d70f3532ea54f0d930e8df47b6da1a69bd294dc5a3" |
| }, |
| "categories:pii/ja": { |
| "acc": 0.9267, |
| "n": 150, |
| "ece": 0.0359, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "9d0a6eeb1fe27e2e5fcb7d7aa058c3bebfaa4e0fd650fe272e96049037cf03f6" |
| }, |
| "categories:pii/nl": { |
| "acc": 0.8533, |
| "n": 150, |
| "ece": 0.0441, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "72e3d46617d1027c2370304fbfea819c14e7d9e7be8a1e7e21e05137680472bd" |
| }, |
| "categories:pii/ru": { |
| "acc": 0.9667, |
| "n": 150, |
| "ece": 0.0473, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "dde80741bffb81fecd80f3a66d88ea65ff9c6b15c050ece7db985ed910ef2f36" |
| }, |
| "categories:pii/sv": { |
| "acc": 0.8733, |
| "n": 150, |
| "ece": 0.0276, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "f0e3e43965db3fb821f3191ba6572b783dc15cdc927dc4e9680da8c2d07e9c5f" |
| }, |
| "categories:pii/zh": { |
| "acc": 0.9267, |
| "n": 150, |
| "ece": 0.0395, |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "pool_items_sha256": "6a85c938e961cc68abc1e79faf75d91c5af55c9a7c38176f0313bd23bf22b1dc" |
| } |
| }, |
| "reported": {}, |
| "mixture": "/home/belkis/Schreibtisch/My WorkSpace/statim/data/mixture-v10.jsonl.gz", |
| "gate_schema": 2, |
| "strict": true, |
| "definitions": { |
| "registry_sha256": "d24ac1430c94a87d79455893106795d7cc2d64fa90ba40765824ca3e11ea4c16", |
| "suite_sha256": { |
| "tools/finetune/eval_dev.py": "aefbc9f675fa87a5dd8ff1cefb5c080843b05d90b9a453dd9fca3cbdbc4c0383", |
| "tools/finetune/eval_laya.py": "c63b4364b59a9048476c0651682e2d760f810cfb450893133110bf84c853f81a", |
| "bench/eval_multilingual.py": "b5ab1f59a261e2890ff2fc1ad5dad98902e1d2f063b608f8ae4fb8cd41263614", |
| "bench/eval_zeroshot.py": "aa59e80ec8e4fb142739e78fb3eeb90b778537c786fadbeba8a2a2d189438a9c", |
| "bench/eval_categories.py": "f6d94bad762fae027059a29ff212d918c835b9d5d0b64394054594bece1eb78d" |
| } |
| }, |
| "suite_status": { |
| "tools/finetune/eval_dev.py": { |
| "status": "ok", |
| "detail": "", |
| "cells": 5 |
| }, |
| "tools/finetune/eval_laya.py": { |
| "status": "ok", |
| "detail": "", |
| "cells": 4 |
| }, |
| "bench/eval_multilingual.py": { |
| "status": "ok", |
| "cells": 24 |
| }, |
| "bench/eval_zeroshot.py": { |
| "status": "ok", |
| "cells": 26 |
| }, |
| "bench/eval_categories.py": { |
| "status": "ok", |
| "cells": 37 |
| } |
| }, |
| "skipped": { |
| "categories:safety/tr": { |
| "why": "only 120 pooled items, need 150", |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "seed": 20260927, |
| "n": 150 |
| }, |
| "categories:safety/zh": { |
| "why": "only 1 pooled items, need 150", |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "seed": 20260927, |
| "n": 150 |
| }, |
| "categories:pii/hi": { |
| "why": "only 132 pooled items, need 150", |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "seed": 20260927, |
| "n": 150 |
| }, |
| "categories:pii/pl": { |
| "why": "only 42 pooled items, need 150", |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "seed": 20260927, |
| "n": 150 |
| }, |
| "categories:pii/pt": { |
| "why": "only 54 pooled items, need 150", |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "seed": 20260927, |
| "n": 150 |
| }, |
| "categories:pii/tr": { |
| "why": "only 58 pooled items, need 150", |
| "pool": "048c4f8142aa6ff0+13eb89bb4ab24253", |
| "seed": 20260927, |
| "n": 150 |
| } |
| }, |
| "eval_items_sha256": "9fad845c1293218f309edaa819bc5f290011df95d1517d917b8377d90e450fcd" |
| } |
