{
 "model": "models/laya-multilingual-v11",
 "validation": {
  "banking77": {
   "acc": 0.908,
   "n": 500
  },
  "massive": {
   "acc": 0.8267,
   "n": 600
  },
  "sentiment": {
   "acc": 0.5983,
   "n": 600
  },
  "emotion": {
   "acc": 0.504,
   "n": 500
  },
  "ag_news": {
   "acc": 0.91,
   "n": 500
  }
 },
 "heldout": {
  "test/ag_news": {
   "acc": 0.921,
   "n": 2000,
   "ece": 0.0078
  },
  "test/emotion": {
   "acc": 0.5305,
   "n": 2000,
   "ece": 0.148
  },
  "test/banking77": {
   "acc": 0.9185,
   "n": 2000,
   "ece": 0.0762
  },
  "test/typed_decisions": {
   "acc": 0.775,
   "n": 2000,
   "ece": null
  },
  "amazon_massive_intent/de": {
   "acc": 0.8267,
   "n": 150,
   "ece": 0.0715
  },
  "amazon_massive_intent/en": {
   "acc": 0.8467,
   "n": 150,
   "ece": 0.053
  },
  "amazon_massive_intent/fr": {
   "acc": 0.8067,
   "n": 150,
   "ece": 0.0371
  },
  "amazon_massive_intent/es": {
   "acc": 0.8333,
   "n": 150,
   "ece": 0.0745
  },
  "amazon_massive_intent/it": {
   "acc": 0.8133,
   "n": 150,
   "ece": 0.0607
  },
  "amazon_massive_intent/tr": {
   "acc": 0.8133,
   "n": 150,
   "ece": 0.0957
  },
  "amazon_massive_intent/pl": {
   "acc": 0.8333,
   "n": 150,
   "ece": 0.0897
  },
  "amazon_massive_intent/ru": {
   "acc": 0.8467,
   "n": 150,
   "ece": 0.0704
  },
  "amazon_massive_intent/ja": {
   "acc": 0.8333,
   "n": 150,
   "ece": 0.0681
  },
  "amazon_massive_intent/zh-CN": {
   "acc": 0.82,
   "n": 150,
   "ece": 0.0588
  },
  "amazon_massive_intent/ar": {
   "acc": 0.74,
   "n": 150,
   "ece": 0.0654
  },
  "amazon_massive_intent/hi": {
   "acc": 0.78,
   "n": 150,
   "ece": 0.0767
  },
  "multilingual_sentiments/ar": {
   "acc": 0.6333,
   "n": 150,
   "ece": 0.1504
  },
  "multilingual_sentiments/zh": {
   "acc": 0.6133,
   "n": 150,
   "ece": 0.1625
  },
  "multilingual_sentiments/en": {
   "acc": 0.72,
   "n": 150,
   "ece": 0.0776
  },
  "multilingual_sentiments/fr": {
   "acc": 0.6267,
   "n": 150,
   "ece": 0.0915
  },
  "multilingual_sentiments/de": {
   "acc": 0.56,
   "n": 150,
   "ece": 0.1318
  },
  "multilingual_sentiments/hi": {
   "acc": 0.54,
   "n": 150,
   "ece": 0.135
  },
  "multilingual_sentiments/id": {
   "acc": 0.78,
   "n": 150,
   "ece": 0.0798
  },
  "multilingual_sentiments/it": {
   "acc": 0.6467,
   "n": 150,
   "ece": 0.0752
  },
  "multilingual_sentiments/ja": {
   "acc": 0.68,
   "n": 150,
   "ece": 0.0907
  },
  "multilingual_sentiments/ms": {
   "acc": 0.5333,
   "n": 150,
   "ece": 0.168
  },
  "multilingual_sentiments/pt": {
   "acc": 0.64,
   "n": 150,
   "ece": 0.1151
  },
  "multilingual_sentiments/es": {
   "acc": 0.6,
   "n": 150,
   "ece": 0.2087
  },
  "go_emotions/en": {
   "acc": 0.4533,
   "n": 150,
   "ece": 0.0881
  },
  "multi_hatecheck/en": {
   "acc": 0.6467,
   "n": 150,
   "ece": 0.1259
  },
  "multi_hatecheck/de": {
   "acc": 0.6733,
   "n": 150,
   "ece": 0.0606
  },
  "multi_hatecheck/fr": {
   "acc": 0.66,
   "n": 150,
   "ece": 0.1077
  },
  "multi_hatecheck/es": {
   "acc": 0.6467,
   "n": 150,
   "ece": 0.1149
  },
  "multi_hatecheck/it": {
   "acc": 0.62,
   "n": 150,
   "ece": 0.1395
  },
  "multi_hatecheck/nl": {
   "acc": 0.6467,
   "n": 150,
   "ece": 0.0941
  },
  "multi_hatecheck/pl": {
   "acc": 0.6533,
   "n": 150,
   "ece": 0.1096
  },
  "multi_hatecheck/pt": {
   "acc": 0.6733,
   "n": 150,
   "ece": 0.0802
  },
  "multi_hatecheck/zh": {
   "acc": 0.64,
   "n": 150,
   "ece": 0.1316
  },
  "multi_hatecheck/ar": {
   "acc": 0.6667,
   "n": 150,
   "ece": 0.0891
  },
  "multi_hatecheck/hi": {
   "acc": 0.58,
   "n": 150,
   "ece": 0.1111
  },
  "sib200/en": {
   "acc": 0.8467,
   "n": 150,
   "ece": 0.062
  },
  "sib200/de": {
   "acc": 0.8267,
   "n": 150,
   "ece": 0.0645
  },
  "sib200/ar": {
   "acc": 0.78,
   "n": 150,
   "ece": 0.0886
  },
  "sib200/hi": {
   "acc": 0.7333,
   "n": 150,
   "ece": 0.1103
  },
  "hwu64/en": {
   "acc": 0.8533,
   "n": 150,
   "ece": 0.0795
  },
  "indonli/id": {
   "acc": 0.7,
   "n": 150,
   "ece": 0.1273
  },
  "farstail/fa": {
   "acc": 0.6533,
   "n": 150,
   "ece": 0.1512
  },
  "belebele/en": {
   "acc": 0.3133,
   "n": 150,
   "ece": 0.2754
  },
  "belebele/de": {
   "acc": 0.32,
   "n": 150,
   "ece": 0.3055
  },
  "belebele/ar": {
   "acc": 0.26,
   "n": 150,
   "ece": 0.3235
  },
  "belebele/hi": {
   "acc": 0.2267,
   "n": 150,
   "ece": 0.3503
  },
  "semrel/en": {
   "acc": 0.28,
   "n": 150,
   "ece": 0.2875
  },
  "semrel/ar": {
   "acc": 0.2733,
   "n": 150,
   "ece": 0.3183
  },
  "semrel/hi": {
   "acc": 0.2933,
   "n": 150,
   "ece": 0.2443
  },
  "categories:sentiment/en": {
   "acc": 0.8533,
   "n": 150,
   "ece": 0.0819,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "dca5d61987d3fa0388ca894850f455db283f3af8eb2fb7adfedba9c538ac56d2"
  },
  "categories:sentiment/zh": {
   "acc": 0.9,
   "n": 150,
   "ece": 0.0304,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "67bb3b2842114c85ce5c5d39ea9254131114e17489b664f993dae78b31302926"
  },
  "categories:emotion/de": {
   "acc": 0.4867,
   "n": 150,
   "ece": 0.1301,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "7e99e8b186acee316711f3c76148ada3cdbf8d82e0d5168107bdf3aeb1ec7b98"
  },
  "categories:emotion/en": {
   "acc": 0.64,
   "n": 150,
   "ece": 0.0991,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "226fc02462825a0b494407518388c06292c177e6c3941abd626a1262f05194a5"
  },
  "categories:emotion/es": {
   "acc": 0.62,
   "n": 150,
   "ece": 0.0918,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "d9760e612a2e182ee1d4f5521423471a67521dad1d71de62c9fe407be8e99df4"
  },
  "categories:emotion/fr": {
   "acc": 0.8267,
   "n": 150,
   "ece": 0.0822,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "a0dc4d4080bee5f6215ac6ca1d7ff831698cd6fadf195942572e73b79ad45e37"
  },
  "categories:emotion/hi": {
   "acc": 0.86,
   "n": 150,
   "ece": 0.0626,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "082b08b1d0eea74ec0138b1424b76dafc8ee95f773c815e3a0412b0a0da9b697"
  },
  "categories:emotion/pt": {
   "acc": 0.56,
   "n": 150,
   "ece": 0.1237,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "e01c851ad2f100c55168c89ba288fccf1e17466db4265ef7c7e2372b545f3bc1"
  },
  "categories:emotion/ru": {
   "acc": 0.74,
   "n": 150,
   "ece": 0.0869,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "3b1f698d6af7861ea4b4789a79322cd34a9aa691d128768eec837d5fa66a5de6"
  },
  "categories:emotion/zh": {
   "acc": 0.5533,
   "n": 150,
   "ece": 0.1448,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "22aeaf3ca1b64322b3afa8d61b40fc0b2b6a1e002181821a5ac4145b59310a38"
  },
  "categories:complaint/en": {
   "acc": 0.82,
   "n": 150,
   "ece": 0.0824,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "333bda35a5d1cd5d5651e0f24a3f3829753ec834b80c19965f5b59945117807a"
  },
  "categories:nli/en": {
   "acc": 0.8,
   "n": 150,
   "ece": 0.071,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "17b38e41fc99c3c8d4458f0857eb437902adaaeccd26eab81c2a20f3b46ba018"
  },
  "categories:nli/ja": {
   "acc": 0.66,
   "n": 150,
   "ece": 0.1421,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "ea2af8de3f6d5bacfc81d6bc137bce5d7a69c6f272b799ca18a651953abef181"
  },
  "categories:nli/tr": {
   "acc": 0.8467,
   "n": 150,
   "ece": 0.0535,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "032946c87c90d553301050355ae43e8ed095b451670fde3638efa944ec6a8276"
  },
  "categories:safety/en": {
   "acc": 0.88,
   "n": 150,
   "ece": 0.0739,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "1955208993d89e14a54edf34f9e3f7d432ad68c49bdb583083fb512310f945ac"
  },
  "categories:reading/en": {
   "acc": 0.9333,
   "n": 150,
   "ece": 0.0519,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "57cecee470248e19e1ec362b39552c3e3bbaf4ced40909e27b95d6c2d03ed809"
  },
  "categories:similarity/pt": {
   "acc": 0.88,
   "n": 150,
   "ece": 0.0631,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "d8a799fb3b53bda0265399aacfb5fcb7939a670120af64598f1e15db2784c69c"
  },
  "categories:topic/en": {
   "acc": 0.6467,
   "n": 150,
   "ece": 0.0898,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "6515d51e9240f29eaa968c8b6402dabae247d9f04095e295360003ba5db2fcbd"
  },
  "categories:intent/en": {
   "acc": 0.8467,
   "n": 150,
   "ece": 0.065,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "1fa7c10ebe1c68fe4a0ffa0c899de81121294f7423fe60011309b0aea9af8f81"
  },
  "categories:intent/nl": {
   "acc": 0.5333,
   "n": 150,
   "ece": 0.0941,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "1f327bd9c76abec1a83d2b51e9947fc1636364e47cc7b5420f08db664b9dc899"
  },
  "categories:intent/tr": {
   "acc": 1.0,
   "n": 150,
   "ece": 0.0002,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "134c60c45d74b27253775ff1b26d209c6dc916cbda0c150999e13584dcf6cce8"
  },
  "categories:stance/en": {
   "acc": 0.98,
   "n": 150,
   "ece": 0.0696,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "e28f7bba23222687ea8d3112afadc7f6a06270eb3faccf219d8ae5feea3b229c"
  },
  "categories:formality/ja": {
   "acc": 0.9067,
   "n": 150,
   "ece": 0.0558,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "e9fe2a0258eec42bbc6ed0ca325063c31eb44df7f5a6dc117af484a2fc3a2beb"
  },
  "categories:formality/tr": {
   "acc": 1.0,
   "n": 150,
   "ece": 0.0025,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "747695931d2a62fe8c2952a9cc66208110f6d968d63bc3603e5d38cd794e6ecc"
  },
  "categories:urgency/en": {
   "acc": 0.9867,
   "n": 150,
   "ece": 0.0247,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "ace94904043bf4d44160d85f936837595b0cba4a9a617fc5e918062becf70271"
  },
  "categories:fact_check/en": {
   "acc": 0.4667,
   "n": 150,
   "ece": 0.2808,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "ac4c49eb82015d55086f84b8643efdd924bd33880320136b845daa16c0ec8ab1"
  },
  "categories:pii/ar": {
   "acc": 0.9333,
   "n": 150,
   "ece": 0.0495,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "323112f111aa8c9c401e70d0f081ea6628b5230e57d741ac291694e16708c2b9"
  },
  "categories:pii/de": {
   "acc": 0.8867,
   "n": 150,
   "ece": 0.0482,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "52145c7106ce9f85fa46c267cdace80c42181e148d9cb45594263718f5a49d92"
  },
  "categories:pii/en": {
   "acc": 0.9333,
   "n": 150,
   "ece": 0.0413,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "10b8d7d2c36445bbeecdfded7c937f4ca228c43ab80c515dd038be0f658a2cbe"
  },
  "categories:pii/es": {
   "acc": 0.8867,
   "n": 150,
   "ece": 0.07,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "853091fcb462b7b89e00759c1085ad942fb99eb455c18f4c5ecd8db63cb20e6e"
  },
  "categories:pii/fr": {
   "acc": 0.9133,
   "n": 150,
   "ece": 0.0322,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "2be7c20bc628fd30fe60af29d6e457b9ce63cba3d8961d3528007cacba4fcbb0"
  },
  "categories:pii/it": {
   "acc": 0.9,
   "n": 150,
   "ece": 0.0357,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "632ad4c14c949eb65c7261d70f3532ea54f0d930e8df47b6da1a69bd294dc5a3"
  },
  "categories:pii/ja": {
   "acc": 0.9267,
   "n": 150,
   "ece": 0.0359,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "9d0a6eeb1fe27e2e5fcb7d7aa058c3bebfaa4e0fd650fe272e96049037cf03f6"
  },
  "categories:pii/nl": {
   "acc": 0.8533,
   "n": 150,
   "ece": 0.0441,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "72e3d46617d1027c2370304fbfea819c14e7d9e7be8a1e7e21e05137680472bd"
  },
  "categories:pii/ru": {
   "acc": 0.9667,
   "n": 150,
   "ece": 0.0473,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "dde80741bffb81fecd80f3a66d88ea65ff9c6b15c050ece7db985ed910ef2f36"
  },
  "categories:pii/sv": {
   "acc": 0.8733,
   "n": 150,
   "ece": 0.0276,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "f0e3e43965db3fb821f3191ba6572b783dc15cdc927dc4e9680da8c2d07e9c5f"
  },
  "categories:pii/zh": {
   "acc": 0.9267,
   "n": 150,
   "ece": 0.0395,
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "pool_items_sha256": "6a85c938e961cc68abc1e79faf75d91c5af55c9a7c38176f0313bd23bf22b1dc"
  }
 },
 "reported": {},
 "mixture": "/home/belkis/Schreibtisch/My WorkSpace/statim/data/mixture-v10.jsonl.gz",
 "gate_schema": 2,
 "strict": true,
 "definitions": {
  "registry_sha256": "d24ac1430c94a87d79455893106795d7cc2d64fa90ba40765824ca3e11ea4c16",
  "suite_sha256": {
   "tools/finetune/eval_dev.py": "aefbc9f675fa87a5dd8ff1cefb5c080843b05d90b9a453dd9fca3cbdbc4c0383",
   "tools/finetune/eval_laya.py": "c63b4364b59a9048476c0651682e2d760f810cfb450893133110bf84c853f81a",
   "bench/eval_multilingual.py": "b5ab1f59a261e2890ff2fc1ad5dad98902e1d2f063b608f8ae4fb8cd41263614",
   "bench/eval_zeroshot.py": "aa59e80ec8e4fb142739e78fb3eeb90b778537c786fadbeba8a2a2d189438a9c",
   "bench/eval_categories.py": "f6d94bad762fae027059a29ff212d918c835b9d5d0b64394054594bece1eb78d"
  }
 },
 "suite_status": {
  "tools/finetune/eval_dev.py": {
   "status": "ok",
   "detail": "",
   "cells": 5
  },
  "tools/finetune/eval_laya.py": {
   "status": "ok",
   "detail": "",
   "cells": 4
  },
  "bench/eval_multilingual.py": {
   "status": "ok",
   "cells": 24
  },
  "bench/eval_zeroshot.py": {
   "status": "ok",
   "cells": 26
  },
  "bench/eval_categories.py": {
   "status": "ok",
   "cells": 37
  }
 },
 "skipped": {
  "categories:safety/tr": {
   "why": "only 120 pooled items, need 150",
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "seed": 20260927,
   "n": 150
  },
  "categories:safety/zh": {
   "why": "only 1 pooled items, need 150",
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "seed": 20260927,
   "n": 150
  },
  "categories:pii/hi": {
   "why": "only 132 pooled items, need 150",
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "seed": 20260927,
   "n": 150
  },
  "categories:pii/pl": {
   "why": "only 42 pooled items, need 150",
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "seed": 20260927,
   "n": 150
  },
  "categories:pii/pt": {
   "why": "only 54 pooled items, need 150",
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "seed": 20260927,
   "n": 150
  },
  "categories:pii/tr": {
   "why": "only 58 pooled items, need 150",
   "pool": "048c4f8142aa6ff0+13eb89bb4ab24253",
   "seed": 20260927,
   "n": 150
  }
 },
 "eval_items_sha256": "9fad845c1293218f309edaa819bc5f290011df95d1517d917b8377d90e450fcd"
}