model,Open Provocativeness Low EM,Open Provocativeness Medium EM,Open Provocativeness High EM,Open Provocativeness Low F1,Open Provocativeness Medium F1,Open Provocativeness High F1,Open Provocativeness Low LR,Open Provocativeness Medium LR,Open Provocativeness High LR,Not open Provocativeness Low EM,Not open Provocativeness Medium EM,Not open Provocativeness High EM,Not open Provocativeness Low IS,Not open Provocativeness Medium IS,Not open Provocativeness High IS,Not open Provocativeness Low PM,Not open Provocativeness Medium PM,Not open Provocativeness High PM,split,RU_adaptation,Access,Team,Country,"n_params, B",source_type,origin_group,model_family,n_items,raw_accuracy
GigaChat-2,40,40,36,47,48,41,64,62,56,71,70,63,73,71,65,75,75,72,open,RU,API,Sber,Russia,,api,russian,gigachat,2840,0.647887323943662
GigaChat-2-Max,62,54,52,69,64,58,80,74,68,90,88,78,91,89,80,93,91,84,open,RU,API,Sber,Russia,,api,russian,gigachat,2840,0.8225352112676056
GigaChat-2-Pro,47,48,41,54,56,46,69,67,60,83,83,72,84,83,74,85,86,81,open,RU,API,Sber,Russia,,api,russian,gigachat,2840,0.7538732394366198
GigaChat-3-Ultra,63,60,48,72,72,59,81,76,68,91,88,79,91,89,81,93,91,86,open,RU,API,Sber,Russia,,api,russian,gigachat,2840,0.826056338028169
aliceai-llm,64,58,48,73,69,58,81,76,65,90,87,79,90,88,79,93,91,86,open,RU,API,Yandex,Russia,,api,russian,aliceai,2840,0.823943661971831
aliceai-llm-flash,63,58,49,73,69,59,81,75,65,90,87,79,90,87,79,93,91,85,open,RU,API,Yandex,Russia,,api,russian,aliceai,2840,0.8235915492957746
claude-sonnet-5,71,62,60,77,72,68,84,76,73,94,92,87,95,92,88,96,94,91,open,,API,Open Source / External,International,,api,international,claude,2840,0.8605633802816901
deepseek-v4-flash-0731,68,63,56,75,72,64,84,78,72,87,84,76,88,84,77,89,86,81,open,,API,Open Source / External,International,,api,international,deepseek,2840,0.7975352112676056
deepseek-v4-pro,70,62,60,76,73,67,85,78,76,90,85,78,91,86,79,92,88,84,open,,API,Open Source / External,International,,api,international,deepseek,2840,0.8165492957746479
gemini-3.5-flash-lite,71,55,63,78,67,68,86,76,72,93,89,83,94,90,84,95,92,89,open,,API,Open Source / External,International,,api,international,gemini,2840,0.844718309859155
gemini-3.6-flash,70,63,67,75,75,74,86,78,75,97,95,93,97,95,93,98,96,95,open,,API,Open Source / External,International,,api,international,gemini,2840,0.907394366197183
glm-5.2,65,62,55,72,72,62,82,78,72,89,83,77,90,84,78,91,86,82,open,,API,Open Source / External,International,,api,international,glm,2840,0.8045774647887324
gpt-5.4,64,60,56,71,70,64,83,77,72,95,92,88,95,93,90,96,94,92,open,,API,Open Source / External,International,,api,international,gpt-5,2840,0.8785211267605634
gpt-5.6-luna-pro,54,54,48,63,67,58,77,76,70,87,81,73,89,83,76,90,85,81,open,,API,Open Source / External,International,,api,international,gpt-5,2840,0.7665492957746479
gpt-5.6-terra,66,58,53,71,70,60,83,77,74,92,91,84,93,91,85,94,93,89,open,,API,Open Source / External,International,,api,international,gpt-5,2840,0.8531690140845071
grok-4.5,68,67,59,76,74,66,86,80,76,97,94,91,97,95,92,98,96,95,open,,API,Open Source / External,International,,api,international,grok,2840,0.9024647887323943
kimi-k2.6,56,51,47,63,62,54,74,70,62,80,76,69,82,79,72,84,81,78,open,,API,Open Source / External,International,,api,international,kimi,2840,0.7211267605633803
mimo-v2.5,51,45,46,58,57,55,73,66,64,83,81,72,85,83,76,86,85,80,open,,API,Open Source / External,International,,api,international,mimo,2840,0.7496478873239436
minimax-m3,60,57,55,68,67,61,79,75,69,81,78,66,82,79,68,85,82,75,open,,API,Open Source / External,International,,api,international,minimax,2840,0.7214788732394366
qwen3.7-flash,52,44,48,60,55,55,72,66,63,80,78,72,81,80,74,84,83,79,open,,API,Open Source / External,International,,api,international,qwen,2840,0.7362676056338028
qwen3.8-max,68,61,61,77,71,67,84,75,73,97,94,90,98,95,91,98,96,94,open,,API,Open Source / External,International,,api,international,qwen,2840,0.8929577464788733
yandexgpt-5.1,54,44,47,66,59,58,77,70,67,91,86,75,91,86,76,92,89,82,open,RU,API,Yandex,Russia,,api,russian,yandexgpt,2840,0.7926056338028169
Qwen/Qwen3.6-27B,55,54,43,62,63,53,75,73,65,85,83,76,86,84,78,88,87,83,full,,OSS,Open Source / External,International,27.0,oss,international,qwen,14199,0.7712514965842665
Qwen/Qwen3.6-35B-A3B,54,51,43,62,62,54,75,72,65,82,78,70,83,80,72,86,83,79,full,,OSS,Open Source / External,International,35.0,oss,international,qwen,14199,0.7387139939432354
ai-sage/GigaChat3.1-10B-A1.8B-bf16,45,40,31,52,47,41,67,61,55,45,46,25,45,46,25,73,70,64,full,RU,OSS,Sber,Russia,10.0,oss,russian,gigachat,14199,0.3874920769068244
allenai/Olmo-3.1-32B-Instruct,19,20,16,25,27,22,44,46,39,38,39,31,41,41,34,48,47,44,full,,OSS,Open Source / External,International,32.0,oss,international,olmo,14199,0.340728220297204
google/gemma-4-12B-it,47,43,34,53,52,43,69,64,56,70,68,59,72,70,62,75,74,69,full,,OSS,Open Source / External,International,12.0,oss,international,gemma,14199,0.6201845200366223
google/gemma-4-26B-A4B-it,55,52,43,62,62,53,75,72,65,80,78,68,81,79,70,84,83,77,full,,OSS,Open Source / External,International,26.0,oss,international,gemma,14199,0.7155433481231073
google/gemma-4-31B-it,59,59,47,66,69,56,80,76,69,88,86,78,88,86,79,90,90,86,full,,OSS,Open Source / External,International,31.0,oss,international,gemma,14199,0.7997042045214452
microsoft/phi-4,17,16,11,41,40,32,58,54,45,41,44,24,43,46,27,70,68,63,full,,OSS,Open Source / External,International,14.0,oss,international,phi,14199,0.3437566025776463
mistralai/Ministral-3-14B-Instruct-2512,42,39,32,58,55,48,72,67,61,72,70,61,75,73,65,78,76,72,full,,OSS,Open Source / External,International,14.0,oss,international,mistral,14199,0.6422987534333404
mistralai/Mistral-Small-3.2-24B-Instruct-2506,55,56,44,63,65,53,76,74,66,73,69,58,76,72,62,80,78,74,full,,OSS,Open Source / External,International,24.0,oss,international,mistral,14199,0.6508909078104091
nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16,6,6,1,9,10,4,23,23,18,31,32,25,32,34,29,40,41,40,full,,OSS,Open Source / External,International,4.0,oss,international,nemotron,14199,0.2688921755053173
t-tech/T-pro-it-2.1,49,48,35,55,56,45,72,69,61,80,77,68,81,78,71,84,82,78,full,RU,OSS,T-Tech,Russia,,oss,russian,t-pro,14199,0.7154729206282132
zai-org/GLM-4.7-Flash,39,35,34,47,46,42,65,60,57,54,52,38,57,55,43,63,61,54,full,,OSS,Open Source / External,International,30.0,oss,international,glm,14199,0.4670047186421579