| {"model_name": "GoToCompany/gemma3-27b-cpt-gotoai-v3-instruct", "model_size": "27B", "model_type": "Instruct", "overall": {"total": 64.39}, "id": {"nlg": {"translation-en-xx": {"total": 94.12}, "translation-xx-en": {"total": 93.08}, "abssum": {"total": 16.9}, "total": 55.25}, "nlr": {"nli": {"total": 74.65}, "causal": {"total": 89.8}, "total": 82.23}, "nlu": {"metaphor": {"total": 71.24}, "qa": {"total": 81.93}, "sentiment": {"total": 74.02}, "total": 75.73}, "instruction-following": {"total": 90.48}, "multi-turn": {"total": 53.75}, "safety": {"toxicity": {"total": 49.39}, "total": 49.39}, "linguistic-diagnostics": {"syntax": {"total": 32.26}, "pragmatics": {"total": 66.77}, "total": 49.52}, "total": 65.19}, "jv": {"nlg": {"translation-id-xx": {"total": 87.39}, "translation-xx-id": {"total": 91.15}, "total": 89.27}, "nlu": {"metaphor": {"total": 59.29}, "qa-mc": {"total": 87.06}, "sentiment": {"total": 71.39}, "total": 72.58}, "instruction-following": {"total": 73.33}, "multi-turn": {"total": 49.18}, "total": 71.09}, "su": {"nlg": {"translation-id-xx": {"total": 82.75}, "translation-xx-id": {"total": 89.89}, "total": 86.32}, "nlu": {"metaphor": {"total": 48.09}, "qa-mc": {"total": 79.05}, "sentiment": {"total": 71.22}, "total": 66.12}, "instruction-following": {"total": 72.63}, "multi-turn": {"total": 51.01}, "total": 69.02}, "ban": {"nlg": {"translation-id-xx": {"total": 77.08}, "translation-xx-id": {"total": 81.31}, "total": 79.2}, "nlu": {"sentiment": {"total": 66.13}, "total": 66.13}, "instruction-following": {"total": 68.79}, "multi-turn": {"total": 42.23}, "total": 64.09}, "bbc": {"nlg": {"translation-id-xx": {"total": 54.75}, "translation-xx-id": {"total": 73.67}, "total": 64.21}, "nlu": {"qa": {"total": 54.13}, "sentiment": {"total": 56.66}, "total": 55.4}, "instruction-following": {"total": 47.43}, "multi-turn": {"total": 43.18}, "total": 52.55}, "indommlu": {"total": 58.48, "Humanities": {"total": 64.14}, "Indonesian language": {"total": 62.44}, "Local languages and cultures": {"total": 39.83}, "Social science": {"total": 65.17}, "STEM": {"total": 60.82}}} |
| {"model_name": "GoToCompany/Llama-Sahabat-AI-v2-70B-IT", "model_size": "70B", "model_type": "Instruct", "overall": {"total": 60.33}, "id": {"nlg": {"translation-en-xx": {"total": 93.16}, "translation-xx-en": {"total": 92.91}, "abssum": {"total": 18.94}, "total": 55.99}, "nlr": {"nli": {"total": 75.27}, "causal": {"total": 92.07}, "total": 83.67}, "nlu": {"metaphor": {"total": 71.45}, "qa": {"total": 78.39}, "sentiment": {"total": 75.08}, "total": 74.97}, "instruction-following": {"total": 93.62}, "multi-turn": {"total": 37.31}, "safety": {"toxicity": {"total": 49.25}, "total": 49.25}, "linguistic-diagnostics": {"syntax": {"total": 37.77}, "pragmatics": {"total": 62.85}, "total": 50.31}, "total": 63.59}, "jv": {"nlg": {"translation-id-xx": {"total": 86.52}, "translation-xx-id": {"total": 88.86}, "total": 87.69}, "nlu": {"metaphor": {"total": 47.04}, "qa-mc": {"total": 89.54}, "sentiment": {"total": 69.05}, "total": 68.54}, "instruction-following": {"total": 70.73}, "multi-turn": {"total": 28.78}, "total": 63.94}, "su": {"nlg": {"translation-id-xx": {"total": 84.82}, "translation-xx-id": {"total": 86.7}, "total": 85.76}, "nlu": {"metaphor": {"total": 33.22}, "qa-mc": {"total": 81.06}, "sentiment": {"total": 67.21}, "total": 60.5}, "instruction-following": {"total": 71.08}, "multi-turn": {"total": 33.33}, "total": 62.67}, "ban": {"nlg": {"translation-id-xx": {"total": 78.86}, "translation-xx-id": {"total": 79.46}, "total": 79.16}, "nlu": {"sentiment": {"total": 68.05}, "total": 68.05}, "instruction-following": {"total": 64.16}, "multi-turn": {"total": 35.88}, "total": 61.81}, "bbc": {"nlg": {"translation-id-xx": {"total": 55.13}, "translation-xx-id": {"total": 72.71}, "total": 63.92}, "nlu": {"qa": {"total": 52.55}, "sentiment": {"total": 52.88}, "total": 52.72}, "instruction-following": {"total": 48.41}, "multi-turn": {"total": 33.48}, "total": 49.63}, "indommlu": {"total": 62.56, "Humanities": {"total": 70.15}, "Indonesian language": {"total": 65.93}, "Local languages and cultures": {"total": 44.46}, "Social science": {"total": 69.15}, "STEM": {"total": 63.11}}} |
| {"model_name": "GoToCompany/gemma2-9b-cpt-sahabatai-v1-base", "model_size": "9B", "model_type": "Base", "overall": {"total": 51.21}, "id": {"nlg": {"translation-en-xx": {"total": 84.86}, "translation-xx-en": {"total": 86.62}, "abssum": {"total": 19.8}, "total": 52.77}, "nlr": {"nli": {"total": 4.65}, "causal": {"total": 81.2}, "total": 42.92}, "nlu": {"metaphor": {"total": 53.75}, "qa": {"total": 70.22}, "sentiment": {"total": 79.89}, "total": 67.95}, "safety": {"toxicity": {"total": 50.78}, "total": 50.78}, "linguistic-diagnostics": {"syntax": {"total": 5.26}, "pragmatics": {"total": 55.52}, "total": 30.39}, "total": 48.96}, "jv": {"nlg": {"translation-id-xx": {"total": 70.08}, "translation-xx-id": {"total": 85.04}, "total": 77.56}, "nlu": {"metaphor": {"total": 46.39}, "qa-mc": {"total": 66.19}, "sentiment": {"total": 77.32}, "total": 63.3}, "total": 70.43}, "su": {"nlg": {"translation-id-xx": {"total": 53.21}, "translation-xx-id": {"total": 81.39}, "total": 67.3}, "nlu": {"metaphor": {"total": 19.73}, "qa-mc": {"total": 52.86}, "sentiment": {"total": 60.9}, "total": 44.5}, "total": 55.9}, "ban": {"nlg": {"translation-id-xx": {"total": 33.03}, "translation-xx-id": {"total": 64.04}, "total": 48.54}, "nlu": {"sentiment": {"total": 66.16}, "total": 66.16}, "total": 57.35}, "bbc": {"nlg": {"translation-id-xx": {"total": 10.48}, "translation-xx-id": {"total": 44.45}, "total": 27.47}, "nlu": {"qa": {"total": 27.57}, "sentiment": {"total": 11.05}, "total": 19.31}, "total": 23.39}} |
| {"model_name": "GoToCompany/gemma2-9b-cpt-sahabatai-v1-instruct", "model_size": "9B", "model_type": "Instruct", "overall": {"total": 39.87}, "id": {"nlg": {"translation-en-xx": {"total": 89.3}, "translation-xx-en": {"total": 89.49}, "abssum": {"total": 19.1}, "total": 54.25}, "nlr": {"nli": {"total": 69.57}, "causal": {"total": 74.23}, "total": 71.9}, "nlu": {"metaphor": {"total": 58.54}, "qa": {"total": 78.07}, "sentiment": {"total": 76.12}, "total": 70.91}, "instruction-following": {"total": 83.49}, "multi-turn": {"total": 10.23}, "safety": {"toxicity": {"total": 43.82}, "total": 43.82}, "linguistic-diagnostics": {"syntax": {"total": 5.79}, "pragmatics": {"total": 53.76}, "total": 29.78}, "total": 52.05}, "jv": {"nlg": {"translation-id-xx": {"total": 70.5}, "translation-xx-id": {"total": 84.19}, "total": 77.34}, "nlu": {"metaphor": {"total": 39.6}, "qa-mc": {"total": 69.52}, "sentiment": {"total": 75.24}, "total": 61.46}, "instruction-following": {"total": 59.46}, "multi-turn": {"total": 10.07}, "total": 52.08}, "su": {"nlg": {"translation-id-xx": {"total": 60.67}, "translation-xx-id": {"total": 77.91}, "total": 69.29}, "nlu": {"metaphor": {"total": 16.99}, "qa-mc": {"total": 65.71}, "sentiment": {"total": 70.07}, "total": 50.93}, "instruction-following": {"total": 55.24}, "multi-turn": {"total": 8.52}, "total": 45.99}, "ban": {"nlg": {"translation-id-xx": {"total": 45.6}, "translation-xx-id": {"total": 63.62}, "total": 54.61}, "nlu": {"sentiment": {"total": 59.08}, "total": 59.08}, "instruction-following": {"total": 9.52}, "multi-turn": {"total": 5.73}, "total": 32.24}, "bbc": {"nlg": {"translation-id-xx": {"total": 39.56}, "translation-xx-id": {"total": 43.01}, "total": 41.28}, "nlu": {"qa": {"total": 22.56}, "sentiment": {"total": 5.96}, "total": 14.26}, "instruction-following": {"total": 8.57}, "multi-turn": {"total": 3.89}, "total": 17.0}, "indommlu": {"total": 45.63, "Humanities": {"total": 51.93}, "Indonesian language": {"total": 50.2}, "Local languages and cultures": {"total": 24.29}, "Social science": {"total": 55.03}, "STEM": {"total": 46.68}}} |
| {"model_name": "GoToCompany/llama3-8b-cpt-sahabatai-v1-base", "model_size": "8B", "model_type": "Base", "overall": {"total": 47.45}, "id": {"nlg": {"translation-en-xx": {"total": 83.14}, "translation-xx-en": {"total": 84.94}, "abssum": {"total": 19.15}, "total": 51.59}, "nlr": {"nli": {"total": 41.72}, "causal": {"total": 79.79}, "total": 60.75}, "nlu": {"metaphor": {"total": 66.77}, "qa": {"total": 65.68}, "sentiment": {"total": 78.93}, "total": 70.46}, "safety": {"toxicity": {"total": 39.63}, "total": 39.63}, "linguistic-diagnostics": {"syntax": {"total": 28.42}, "pragmatics": {"total": 22.87}, "total": 25.65}, "total": 49.62}, "jv": {"nlg": {"translation-id-xx": {"total": 58.39}, "translation-xx-id": {"total": 81.42}, "total": 69.9}, "nlu": {"metaphor": {"total": 51.07}, "qa-mc": {"total": 59.52}, "sentiment": {"total": 76.94}, "total": 62.51}, "total": 66.21}, "su": {"nlg": {"translation-id-xx": {"total": 50.83}, "translation-xx-id": {"total": 80.49}, "total": 65.66}, "nlu": {"metaphor": {"total": 38.26}, "qa-mc": {"total": 51.52}, "sentiment": {"total": 29.9}, "total": 39.89}, "total": 52.78}, "ban": {"nlg": {"translation-id-xx": {"total": 24.14}, "translation-xx-id": {"total": 56.0}, "total": 40.07}, "nlu": {"sentiment": {"total": 59.48}, "total": 59.48}, "total": 49.77}, "bbc": {"nlg": {"translation-id-xx": {"total": 9.58}, "translation-xx-id": {"total": 38.23}, "total": 23.91}, "nlu": {"qa": {"total": 23.98}, "sentiment": {"total": 3.64}, "total": 13.81}, "total": 18.86}} |
| {"model_name": "GoToCompany/llama3-8b-cpt-sahabatai-v1-instruct", "model_size": "8B", "model_type": "Instruct", "overall": {"total": 28.83}, "id": {"nlg": {"translation-en-xx": {"total": 83.08}, "translation-xx-en": {"total": 76.26}, "abssum": {"total": 18.32}, "total": 48.99}, "nlr": {"nli": {"total": 32.55}, "causal": {"total": 67.39}, "total": 49.97}, "nlu": {"metaphor": {"total": 53.08}, "qa": {"total": 70.91}, "sentiment": {"total": 61.94}, "total": 61.98}, "instruction-following": {"total": 70.29}, "multi-turn": {"total": 5.95}, "safety": {"toxicity": {"total": 43.71}, "total": 43.71}, "linguistic-diagnostics": {"syntax": {"total": 9.77}, "pragmatics": {"total": 46.52}, "total": 28.15}, "total": 44.15}, "jv": {"nlg": {"translation-id-xx": {"total": 55.9}, "translation-xx-id": {"total": 69.5}, "total": 62.7}, "nlu": {"metaphor": {"total": 34.82}, "qa-mc": {"total": 53.81}, "sentiment": {"total": 49.99}, "total": 46.2}, "instruction-following": {"total": 27.11}, "multi-turn": {"total": 4.25}, "total": 35.07}, "su": {"nlg": {"translation-id-xx": {"total": 49.77}, "translation-xx-id": {"total": 64.68}, "total": 57.22}, "nlu": {"metaphor": {"total": 9.01}, "qa-mc": {"total": 36.67}, "sentiment": {"total": 47.31}, "total": 31.0}, "instruction-following": {"total": 25.71}, "multi-turn": {"total": 5.57}, "total": 29.88}, "ban": {"nlg": {"translation-id-xx": {"total": 49.39}, "translation-xx-id": {"total": 47.34}, "total": 48.37}, "nlu": {"sentiment": {"total": 34.82}, "total": 34.82}, "instruction-following": {"total": 6.67}, "multi-turn": {"total": 2.34}, "total": 23.05}, "bbc": {"nlg": {"translation-id-xx": {"total": 34.46}, "translation-xx-id": {"total": 24.12}, "total": 29.29}, "nlu": {"qa": {"total": 20.14}, "sentiment": {"total": 0.37}, "total": 10.26}, "instruction-following": {"total": 6.67}, "multi-turn": {"total": 1.75}, "total": 11.99}, "indommlu": {"total": 45.24, "Humanities": {"total": 52.06}, "Indonesian language": {"total": 50.03}, "Local languages and cultures": {"total": 31.87}, "Social science": {"total": 52.69}, "STEM": {"total": 39.55}}} |
| {"model_name": "Qwen/Qwen2-7B", "model_size": "7B", "model_type": "Base", "overall": {"total": 27.88}, "id": {"nlg": {"translation-en-xx": {"total": 67.88}, "translation-xx-en": {"total": 87.83}, "abssum": {"total": 14.52}, "total": 46.19}, "nlr": {"nli": {"total": 47.44}, "causal": {"total": 82.11}, "total": 64.77}, "nlu": {"metaphor": {"total": 55.87}, "qa": {"total": 61.8}, "sentiment": {"total": 71.32}, "total": 63.0}, "safety": {"toxicity": {"total": 26.21}, "total": 26.21}, "linguistic-diagnostics": {"syntax": {"total": 15.26}, "pragmatics": {"total": 22.53}, "total": 18.9}, "total": 43.81}, "jv": {"nlg": {"translation-id-xx": {"total": 12.37}, "translation-xx-id": {"total": 41.85}, "total": 27.11}, "nlu": {"metaphor": {"total": 25.01}, "qa-mc": {"total": 36.24}, "sentiment": {"total": 53.33}, "total": 38.19}, "total": 32.65}, "su": {"nlg": {"translation-id-xx": {"total": 12.71}, "translation-xx-id": {"total": 35.8}, "total": 24.26}, "nlu": {"metaphor": {"total": 2.38}, "qa-mc": {"total": 33.81}, "sentiment": {"total": 30.44}, "total": 22.21}, "total": 23.23}, "ban": {"nlg": {"translation-id-xx": {"total": 15.26}, "translation-xx-id": {"total": 29.72}, "total": 22.49}, "nlu": {"sentiment": {"total": 30.24}, "total": 30.24}, "total": 26.36}, "bbc": {"nlg": {"translation-id-xx": {"total": 4.03}, "translation-xx-id": {"total": 16.77}, "total": 10.4}, "nlu": {"qa": {"total": 24.71}, "sentiment": {"total": 7.82}, "total": 16.26}, "total": 13.33}} |
| {"model_name": "Qwen/Qwen2-7B-Instruct", "model_size": "7B", "model_type": "Instruct", "overall": {"total": 26.06}, "id": {"nlg": {"translation-en-xx": {"total": 82.15}, "translation-xx-en": {"total": 78.58}, "abssum": {"total": 12.32}, "total": 46.34}, "nlr": {"nli": {"total": 62.75}, "causal": {"total": 80.59}, "total": 71.67}, "nlu": {"metaphor": {"total": 57.08}, "qa": {"total": 65.52}, "sentiment": {"total": 75.98}, "total": 66.19}, "instruction-following": {"total": 62.86}, "multi-turn": {"total": 8.97}, "safety": {"toxicity": {"total": 22.74}, "total": 22.74}, "linguistic-diagnostics": {"syntax": {"total": 1.7}, "pragmatics": {"total": 51.02}, "total": 26.36}, "total": 43.59}, "jv": {"nlg": {"translation-id-xx": {"total": 70.87}, "translation-xx-id": {"total": 53.5}, "total": 62.18}, "nlu": {"metaphor": {"total": 16.25}, "qa-mc": {"total": 42.95}, "sentiment": {"total": 57.53}, "total": 38.91}, "instruction-following": {"total": 3.81}, "multi-turn": {"total": 3.03}, "total": 26.98}, "su": {"nlg": {"translation-id-xx": {"total": 29.71}, "translation-xx-id": {"total": 42.28}, "total": 36.0}, "nlu": {"metaphor": {"total": 14.74}, "qa-mc": {"total": 32.86}, "sentiment": {"total": 28.08}, "total": 25.22}, "instruction-following": {"total": 1.78}, "multi-turn": {"total": 2.46}, "total": 16.36}, "ban": {"nlg": {"translation-id-xx": {"total": 77.27}, "translation-xx-id": {"total": 45.72}, "total": 61.49}, "nlu": {"sentiment": {"total": 34.09}, "total": 34.09}, "instruction-following": {"total": 5.71}, "multi-turn": {"total": 2.59}, "total": 25.97}, "bbc": {"nlg": {"translation-id-xx": {"total": 64.0}, "translation-xx-id": {"total": 30.21}, "total": 47.11}, "nlu": {"qa": {"total": 27.37}, "sentiment": {"total": 13.01}, "total": 20.19}, "instruction-following": {"total": 0.95}, "multi-turn": {"total": 1.38}, "total": 17.41}, "indommlu": {"total": 34.58, "Humanities": {"total": 37.74}, "Indonesian language": {"total": 42.31}, "Local languages and cultures": {"total": 15.93}, "Social science": {"total": 41.53}, "STEM": {"total": 35.42}}} |
| {"model_name": "Qwen/Qwen2.5-72B-Instruct", "model_size": "72B", "model_type": "Instruct", "overall": {"total": 42.78}, "id": {"nlg": {"translation-en-xx": {"total": 91.96}, "translation-xx-en": {"total": 92.93}, "abssum": {"total": 17.0}, "total": 54.72}, "nlr": {"nli": {"total": 72.69}, "causal": {"total": 94.4}, "total": 83.54}, "nlu": {"metaphor": {"total": 72.15}, "qa": {"total": 76.46}, "sentiment": {"total": 78.59}, "total": 75.73}, "instruction-following": {"total": 91.49}, "multi-turn": {"total": 34.2}, "safety": {"toxicity": {"total": 49.22}, "total": 49.22}, "linguistic-diagnostics": {"syntax": {"total": 50.12}, "pragmatics": {"total": 72.42}, "total": 61.27}, "total": 64.31}, "jv": {"nlg": {"translation-id-xx": {"total": 52.36}, "translation-xx-id": {"total": 79.96}, "total": 66.16}, "nlu": {"metaphor": {"total": 37.79}, "qa-mc": {"total": 75.78}, "sentiment": {"total": 66.27}, "total": 59.95}, "instruction-following": {"total": 43.4}, "multi-turn": {"total": 24.11}, "total": 48.4}, "su": {"nlg": {"translation-id-xx": {"total": 47.37}, "translation-xx-id": {"total": 72.92}, "total": 60.15}, "nlu": {"metaphor": {"total": 23.88}, "qa-mc": {"total": 65.24}, "sentiment": {"total": 56.86}, "total": 48.66}, "instruction-following": {"total": 43.21}, "multi-turn": {"total": 26.18}, "total": 44.55}, "ban": {"nlg": {"translation-id-xx": {"total": 54.3}, "translation-xx-id": {"total": 65.57}, "total": 59.93}, "nlu": {"sentiment": {"total": 51.71}, "total": 51.71}, "instruction-following": {"total": 5.3}, "multi-turn": {"total": 14.44}, "total": 32.85}, "bbc": {"nlg": {"translation-id-xx": {"total": 35.55}, "translation-xx-id": {"total": 52.13}, "total": 43.84}, "nlu": {"qa": {"total": 40.18}, "sentiment": {"total": 22.34}, "total": 31.26}, "instruction-following": {"total": 6.29}, "multi-turn": {"total": 13.76}, "total": 23.79}, "indommlu": {"total": 57.51, "Humanities": {"total": 66.03}, "Indonesian language": {"total": 63.0}, "Local languages and cultures": {"total": 27.51}, "Social science": {"total": 66.3}, "STEM": {"total": 64.74}}} |
| {"model_name": "Qwen/Qwen2.5-7B", "model_size": "7B", "model_type": "Base", "overall": {"total": 29.36}, "id": {"nlg": {"translation-en-xx": {"total": 71.95}, "translation-xx-en": {"total": 88.16}, "abssum": {"total": 14.02}, "total": 47.03}, "nlr": {"nli": {"total": 54.29}, "causal": {"total": 84.8}, "total": 69.55}, "nlu": {"metaphor": {"total": 65.37}, "qa": {"total": 67.65}, "sentiment": {"total": 68.94}, "total": 67.32}, "safety": {"toxicity": {"total": 14.5}, "total": 14.5}, "linguistic-diagnostics": {"syntax": {"total": 25.79}, "pragmatics": {"total": 41.19}, "total": 33.49}, "total": 46.38}, "jv": {"nlg": {"translation-id-xx": {"total": 16.98}, "translation-xx-id": {"total": 49.01}, "total": 33.0}, "nlu": {"metaphor": {"total": 21.31}, "qa-mc": {"total": 44.76}, "sentiment": {"total": 37.19}, "total": 34.42}, "total": 33.71}, "su": {"nlg": {"translation-id-xx": {"total": 16.2}, "translation-xx-id": {"total": 39.47}, "total": 27.83}, "nlu": {"metaphor": {"total": 9.23}, "qa-mc": {"total": 39.05}, "sentiment": {"total": 31.32}, "total": 26.53}, "total": 27.18}, "ban": {"nlg": {"translation-id-xx": {"total": 19.5}, "translation-xx-id": {"total": 38.18}, "total": 28.84}, "nlu": {"sentiment": {"total": 23.79}, "total": 23.79}, "total": 26.32}, "bbc": {"nlg": {"translation-id-xx": {"total": 6.62}, "translation-xx-id": {"total": 19.49}, "total": 13.05}, "nlu": {"qa": {"total": 23.14}, "sentiment": {"total": 3.64}, "total": 13.39}, "total": 13.22}} |
| {"model_name": "Qwen/Qwen2.5-7B-Instruct", "model_size": "7B", "model_type": "Instruct", "overall": {"total": 28.83}, "id": {"nlg": {"translation-en-xx": {"total": 84.27}, "translation-xx-en": {"total": 91.1}, "abssum": {"total": 17.52}, "total": 52.6}, "nlr": {"nli": {"total": 68.46}, "causal": {"total": 78.92}, "total": 73.69}, "nlu": {"metaphor": {"total": 62.53}, "qa": {"total": 76.56}, "sentiment": {"total": 75.06}, "total": 71.38}, "instruction-following": {"total": 78.1}, "multi-turn": {"total": 15.49}, "safety": {"toxicity": {"total": 44.75}, "total": 44.75}, "linguistic-diagnostics": {"syntax": {"total": 11.54}, "pragmatics": {"total": 44.65}, "total": 28.1}, "total": 52.02}, "jv": {"nlg": {"translation-id-xx": {"total": 37.37}, "translation-xx-id": {"total": 60.07}, "total": 48.72}, "nlu": {"metaphor": {"total": 15.72}, "qa-mc": {"total": 43.29}, "sentiment": {"total": 60.43}, "total": 39.81}, "instruction-following": {"total": 7.62}, "multi-turn": {"total": 5.92}, "total": 25.52}, "su": {"nlg": {"translation-id-xx": {"total": 38.8}, "translation-xx-id": {"total": 51.11}, "total": 44.96}, "nlu": {"metaphor": {"total": 13.08}, "qa-mc": {"total": 32.86}, "sentiment": {"total": 38.89}, "total": 28.28}, "instruction-following": {"total": 12.32}, "multi-turn": {"total": 6.82}, "total": 23.09}, "ban": {"nlg": {"translation-id-xx": {"total": 71.88}, "translation-xx-id": {"total": 50.78}, "total": 61.33}, "nlu": {"sentiment": {"total": 35.98}, "total": 35.98}, "instruction-following": {"total": 1.9}, "multi-turn": {"total": 5.89}, "total": 26.28}, "bbc": {"nlg": {"translation-id-xx": {"total": 32.73}, "translation-xx-id": {"total": 35.21}, "total": 33.97}, "nlu": {"qa": {"total": 31.18}, "sentiment": {"total": 11.76}, "total": 21.47}, "instruction-following": {"total": 7.62}, "multi-turn": {"total": 5.98}, "total": 17.26}, "indommlu": {"total": 42.11, "Humanities": {"total": 48.04}, "Indonesian language": {"total": 49.61}, "Local languages and cultures": {"total": 15.32}, "Social science": {"total": 52.35}, "STEM": {"total": 45.25}}} |
| {"model_name": "Qwen/Qwen3-30B-A3B", "model_size": "30B", "model_type": "Instruct", "overall": {"total": 43.93}, "id": {"nlg": {"translation-en-xx": {"total": 91.12}, "translation-xx-en": {"total": 91.58}, "abssum": {"total": 17.46}, "total": 54.4}, "nlr": {"nli": {"total": 67.82}, "causal": {"total": 89.2}, "total": 78.51}, "nlu": {"metaphor": {"total": 66.03}, "qa": {"total": 79.25}, "sentiment": {"total": 76.24}, "total": 73.84}, "instruction-following": {"total": 83.81}, "multi-turn": {"total": 41.98}, "safety": {"toxicity": {"total": 48.5}, "total": 48.5}, "linguistic-diagnostics": {"syntax": {"total": 38.95}, "pragmatics": {"total": 45.14}, "total": 42.05}, "total": 60.44}, "jv": {"nlg": {"translation-id-xx": {"total": 70.05}, "translation-xx-id": {"total": 82.28}, "total": 76.16}, "nlu": {"metaphor": {"total": 39.52}, "qa-mc": {"total": 82.38}, "sentiment": {"total": 66.41}, "total": 62.77}, "instruction-following": {"total": 38.29}, "multi-turn": {"total": 37.31}, "total": 53.63}, "su": {"nlg": {"translation-id-xx": {"total": 56.72}, "translation-xx-id": {"total": 75.2}, "total": 65.96}, "nlu": {"metaphor": {"total": 32.83}, "qa-mc": {"total": 71.43}, "sentiment": {"total": 60.2}, "total": 54.82}, "instruction-following": {"total": 48.63}, "multi-turn": {"total": 36.62}, "total": 51.51}, "ban": {"nlg": {"translation-id-xx": {"total": 64.9}, "translation-xx-id": {"total": 57.8}, "total": 61.35}, "nlu": {"sentiment": {"total": 40.54}, "total": 40.54}, "instruction-following": {"total": 11.3}, "multi-turn": {"total": 18.56}, "total": 32.94}, "bbc": {"nlg": {"translation-id-xx": {"total": 37.35}, "translation-xx-id": {"total": 43.75}, "total": 40.55}, "nlu": {"qa": {"total": 30.46}, "sentiment": {"total": 0.84}, "total": 15.65}, "instruction-following": {"total": 12.38}, "multi-turn": {"total": 15.99}, "total": 21.14}, "indommlu": {"total": 57.48, "Humanities": {"total": 60.91}, "Indonesian language": {"total": 62.79}, "Local languages and cultures": {"total": 30.98}, "Social science": {"total": 64.91}, "STEM": {"total": 67.79}}} |
| {"model_name": "Qwen/Qwen3-32B", "model_size": "32B", "model_type": "Instruct", "overall": {"total": 49.76}, "id": {"nlg": {"translation-en-xx": {"total": 91.93}, "translation-xx-en": {"total": 92.65}, "abssum": {"total": 16.57}, "total": 54.43}, "nlr": {"nli": {"total": 76.33}, "causal": {"total": 90.0}, "total": 83.17}, "nlu": {"metaphor": {"total": 78.91}, "qa": {"total": 77.9}, "sentiment": {"total": 79.88}, "total": 78.9}, "instruction-following": {"total": 89.52}, "multi-turn": {"total": 56.35}, "safety": {"toxicity": {"total": 47.45}, "total": 47.45}, "linguistic-diagnostics": {"syntax": {"total": 54.81}, "pragmatics": {"total": 63.37}, "total": 59.09}, "total": 66.99}, "jv": {"nlg": {"translation-id-xx": {"total": 72.1}, "translation-xx-id": {"total": 84.72}, "total": 78.41}, "nlu": {"metaphor": {"total": 52.27}, "qa-mc": {"total": 87.14}, "sentiment": {"total": 75.51}, "total": 71.64}, "instruction-following": {"total": 48.73}, "multi-turn": {"total": 48.02}, "total": 61.7}, "su": {"nlg": {"translation-id-xx": {"total": 61.69}, "translation-xx-id": {"total": 78.5}, "total": 70.09}, "nlu": {"metaphor": {"total": 32.09}, "qa-mc": {"total": 77.14}, "sentiment": {"total": 68.89}, "total": 59.37}, "instruction-following": {"total": 52.54}, "multi-turn": {"total": 46.71}, "total": 57.18}, "ban": {"nlg": {"translation-id-xx": {"total": 58.37}, "translation-xx-id": {"total": 60.47}, "total": 59.42}, "nlu": {"sentiment": {"total": 61.36}, "total": 61.36}, "instruction-following": {"total": 3.94}, "multi-turn": {"total": 27.96}, "total": 38.17}, "bbc": {"nlg": {"translation-id-xx": {"total": 42.56}, "translation-xx-id": {"total": 47.05}, "total": 44.8}, "nlu": {"qa": {"total": 34.49}, "sentiment": {"total": 10.38}, "total": 22.44}, "instruction-following": {"total": 2.86}, "multi-turn": {"total": 29.02}, "total": 24.78}, "indommlu": {"total": 59.9, "Humanities": {"total": 63.75}, "Indonesian language": {"total": 65.68}, "Local languages and cultures": {"total": 33.99}, "Social science": {"total": 66.09}, "STEM": {"total": 69.98}}} |
| {"model_name": "aisingapore/Gemma-SEA-LION-v3-9B", "model_size": "9B", "model_type": "Base", "overall": {"total": 44.99}, "id": {"nlg": {"translation-en-xx": {"total": 82.02}, "translation-xx-en": {"total": 86.44}, "abssum": {"total": 18.85}, "total": 51.54}, "nlr": {"nli": {"total": 37.12}, "causal": {"total": 74.8}, "total": 55.96}, "nlu": {"metaphor": {"total": 59.87}, "qa": {"total": 68.34}, "sentiment": {"total": 77.06}, "total": 68.42}, "safety": {"toxicity": {"total": 24.59}, "total": 24.59}, "linguistic-diagnostics": {"syntax": {"total": 2.35}, "pragmatics": {"total": 37.74}, "total": 20.04}, "total": 44.11}, "jv": {"nlg": {"translation-id-xx": {"total": 38.46}, "translation-xx-id": {"total": 75.05}, "total": 56.76}, "nlu": {"metaphor": {"total": 33.64}, "qa-mc": {"total": 60.33}, "sentiment": {"total": 75.79}, "total": 56.59}, "total": 56.67}, "su": {"nlg": {"translation-id-xx": {"total": 29.85}, "translation-xx-id": {"total": 70.26}, "total": 50.06}, "nlu": {"metaphor": {"total": 12.24}, "qa-mc": {"total": 54.29}, "sentiment": {"total": 59.29}, "total": 41.94}, "total": 46.0}, "ban": {"nlg": {"translation-id-xx": {"total": 30.01}, "translation-xx-id": {"total": 58.58}, "total": 44.3}, "nlu": {"sentiment": {"total": 60.1}, "total": 60.1}, "total": 52.2}, "bbc": {"nlg": {"translation-id-xx": {"total": 12.69}, "translation-xx-id": {"total": 41.94}, "total": 27.31}, "nlu": {"qa": {"total": 25.26}, "sentiment": {"total": 23.93}, "total": 24.6}, "total": 25.96}} |
| {"model_name": "aisingapore/Gemma-SEA-LION-v3-9B-IT", "model_size": "9B", "model_type": "Instruct", "overall": {"total": 43.51}, "id": {"nlg": {"translation-en-xx": {"total": 92.49}, "translation-xx-en": {"total": 92.75}, "abssum": {"total": 17.48}, "total": 55.05}, "nlr": {"nli": {"total": 73.54}, "causal": {"total": 92.4}, "total": 82.97}, "nlu": {"metaphor": {"total": 66.71}, "qa": {"total": 77.03}, "sentiment": {"total": 72.34}, "total": 72.02}, "instruction-following": {"total": 89.52}, "multi-turn": {"total": 32.41}, "safety": {"toxicity": {"total": 42.42}, "total": 42.42}, "linguistic-diagnostics": {"syntax": {"total": 26.32}, "pragmatics": {"total": 66.7}, "total": 46.51}, "total": 60.13}, "jv": {"nlg": {"translation-id-xx": {"total": 73.76}, "translation-xx-id": {"total": 85.55}, "total": 79.66}, "nlu": {"metaphor": {"total": 43.02}, "qa-mc": {"total": 81.49}, "sentiment": {"total": 70.0}, "total": 64.84}, "instruction-following": {"total": 25.71}, "multi-turn": {"total": 25.49}, "total": 48.92}, "su": {"nlg": {"translation-id-xx": {"total": 61.3}, "translation-xx-id": {"total": 80.87}, "total": 71.09}, "nlu": {"metaphor": {"total": 31.22}, "qa-mc": {"total": 69.52}, "sentiment": {"total": 63.53}, "total": 54.76}, "instruction-following": {"total": 40.95}, "multi-turn": {"total": 23.74}, "total": 47.63}, "ban": {"nlg": {"translation-id-xx": {"total": 47.56}, "translation-xx-id": {"total": 74.44}, "total": 61.0}, "nlu": {"sentiment": {"total": 52.05}, "total": 52.05}, "instruction-following": {"total": 11.97}, "multi-turn": {"total": 17.27}, "total": 35.57}, "bbc": {"nlg": {"translation-id-xx": {"total": 38.14}, "translation-xx-id": {"total": 54.81}, "total": 46.48}, "nlu": {"qa": {"total": 34.97}, "sentiment": {"total": 21.62}, "total": 28.29}, "instruction-following": {"total": 12.38}, "multi-turn": {"total": 13.98}, "total": 25.28}, "indommlu": {"total": 50.31, "Humanities": {"total": 56.97}, "Indonesian language": {"total": 56.65}, "Local languages and cultures": {"total": 27.75}, "Social science": {"total": 58.5}, "STEM": {"total": 51.69}}} |
| {"model_name": "aisingapore/Gemma-SEA-LION-v4-27B-IT", "model_size": "27B", "model_type": "Instruct", "overall": {"total": 59.57}, "id": {"nlg": {"translation-en-xx": {"total": 93.9}, "translation-xx-en": {"total": 93.06}, "abssum": {"total": 16.05}, "total": 54.77}, "nlr": {"nli": {"total": 73.41}, "causal": {"total": 90.0}, "total": 81.7}, "nlu": {"metaphor": {"total": 69.42}, "qa": {"total": 81.54}, "sentiment": {"total": 74.54}, "total": 75.17}, "instruction-following": {"total": 91.43}, "multi-turn": {"total": 54.34}, "safety": {"toxicity": {"total": 47.17}, "total": 47.17}, "linguistic-diagnostics": {"syntax": {"total": 33.16}, "pragmatics": {"total": 65.1}, "total": 49.13}, "total": 64.82}, "jv": {"nlg": {"translation-id-xx": {"total": 84.35}, "translation-xx-id": {"total": 91.09}, "total": 87.72}, "nlu": {"metaphor": {"total": 56.59}, "qa-mc": {"total": 89.05}, "sentiment": {"total": 71.27}, "total": 72.3}, "instruction-following": {"total": 65.71}, "multi-turn": {"total": 49.74}, "total": 68.87}, "su": {"nlg": {"translation-id-xx": {"total": 77.0}, "translation-xx-id": {"total": 88.75}, "total": 82.88}, "nlu": {"metaphor": {"total": 41.57}, "qa-mc": {"total": 80.0}, "sentiment": {"total": 70.56}, "total": 64.04}, "instruction-following": {"total": 73.33}, "multi-turn": {"total": 45.22}, "total": 66.37}, "ban": {"nlg": {"translation-id-xx": {"total": 58.89}, "translation-xx-id": {"total": 80.95}, "total": 69.92}, "nlu": {"sentiment": {"total": 65.75}, "total": 65.75}, "instruction-following": {"total": 51.43}, "multi-turn": {"total": 40.63}, "total": 56.93}, "bbc": {"nlg": {"translation-id-xx": {"total": 20.08}, "translation-xx-id": {"total": 70.29}, "total": 45.19}, "nlu": {"qa": {"total": 50.68}, "sentiment": {"total": 49.28}, "total": 49.98}, "instruction-following": {"total": 31.43}, "multi-turn": {"total": 36.9}, "total": 40.87}, "indommlu": {"total": 56.72, "Humanities": {"total": 62.11}, "Indonesian language": {"total": 60.09}, "Local languages and cultures": {"total": 37.98}, "Social science": {"total": 63.51}, "STEM": {"total": 59.92}}} |
| {"model_name": "aisingapore/Llama-SEA-LION-v2-8B", "model_size": "8B", "model_type": "Base", "overall": {"total": 46.49}, "id": {"nlg": {"translation-en-xx": {"total": 91.19}, "translation-xx-en": {"total": 91.54}, "abssum": {"total": 23.5}, "total": 57.43}, "nlr": {"nli": {"total": 51.41}, "causal": {"total": 74.47}, "total": 62.94}, "nlu": {"metaphor": {"total": 62.64}, "qa": {"total": 77.31}, "sentiment": {"total": 71.22}, "total": 70.39}, "safety": {"toxicity": {"total": 34.44}, "total": 34.44}, "linguistic-diagnostics": {"syntax": {"total": 28.95}, "pragmatics": {"total": 18.52}, "total": 23.74}, "total": 49.79}, "jv": {"nlg": {"translation-id-xx": {"total": 58.57}, "translation-xx-id": {"total": 80.59}, "total": 69.58}, "nlu": {"metaphor": {"total": 29.68}, "qa-mc": {"total": 51.37}, "sentiment": {"total": 57.71}, "total": 46.25}, "total": 57.92}, "su": {"nlg": {"translation-id-xx": {"total": 50.08}, "translation-xx-id": {"total": 72.7}, "total": 61.39}, "nlu": {"metaphor": {"total": 7.57}, "qa-mc": {"total": 39.52}, "sentiment": {"total": 49.76}, "total": 32.29}, "total": 46.84}, "ban": {"nlg": {"translation-id-xx": {"total": 51.48}, "translation-xx-id": {"total": 65.58}, "total": 58.53}, "nlu": {"sentiment": {"total": 41.07}, "total": 41.07}, "total": 49.8}, "bbc": {"nlg": {"translation-id-xx": {"total": 21.61}, "translation-xx-id": {"total": 56.79}, "total": 39.2}, "nlu": {"qa": {"total": 34.1}, "sentiment": {"total": 0.0}, "total": 17.05}, "total": 28.12}} |
| {"model_name": "aisingapore/Llama-SEA-LION-v2-8B-IT", "model_size": "8B", "model_type": "Instruct", "overall": {"total": 28.43}, "id": {"nlg": {"translation-en-xx": {"total": 90.94}, "translation-xx-en": {"total": 83.39}, "abssum": {"total": 20.38}, "total": 53.77}, "nlr": {"nli": {"total": 56.08}, "causal": {"total": 77.37}, "total": 66.72}, "nlu": {"metaphor": {"total": 59.47}, "qa": {"total": 76.93}, "sentiment": {"total": 74.46}, "total": 70.29}, "instruction-following": {"total": 74.29}, "multi-turn": {"total": 9.83}, "safety": {"toxicity": {"total": 41.11}, "total": 41.11}, "linguistic-diagnostics": {"syntax": {"total": 16.84}, "pragmatics": {"total": 37.08}, "total": 26.96}, "total": 49.0}, "jv": {"nlg": {"translation-id-xx": {"total": 57.44}, "translation-xx-id": {"total": 74.84}, "total": 66.14}, "nlu": {"metaphor": {"total": 12.57}, "qa-mc": {"total": 53.81}, "sentiment": {"total": 62.92}, "total": 43.1}, "instruction-following": {"total": 2.86}, "multi-turn": {"total": 3.32}, "total": 28.85}, "su": {"nlg": {"translation-id-xx": {"total": 54.75}, "translation-xx-id": {"total": 68.02}, "total": 61.38}, "nlu": {"metaphor": {"total": 8.88}, "qa-mc": {"total": 32.86}, "sentiment": {"total": 51.08}, "total": 30.94}, "instruction-following": {"total": 4.76}, "multi-turn": {"total": 3.25}, "total": 25.08}, "ban": {"nlg": {"translation-id-xx": {"total": 56.21}, "translation-xx-id": {"total": 58.02}, "total": 57.12}, "nlu": {"sentiment": {"total": 30.02}, "total": 30.02}, "instruction-following": {"total": 5.68}, "multi-turn": {"total": 3.71}, "total": 24.13}, "bbc": {"nlg": {"translation-id-xx": {"total": 29.44}, "translation-xx-id": {"total": 42.98}, "total": 36.21}, "nlu": {"qa": {"total": 21.54}, "sentiment": {"total": 9.54}, "total": 15.54}, "instruction-following": {"total": 5.71}, "multi-turn": {"total": 2.83}, "total": 15.07}, "indommlu": {"total": 45.15, "Humanities": {"total": 54.0}, "Indonesian language": {"total": 50.66}, "Local languages and cultures": {"total": 24.54}, "Social science": {"total": 55.28}, "STEM": {"total": 41.27}}} |
| {"model_name": "aisingapore/Llama-SEA-LION-v3-70B-IT", "model_size": "70B", "model_type": "Instruct", "overall": {"total": 50.71}, "id": {"nlg": {"translation-en-xx": {"total": 93.08}, "translation-xx-en": {"total": 92.85}, "abssum": {"total": 18.98}, "total": 55.97}, "nlr": {"nli": {"total": 77.94}, "causal": {"total": 90.8}, "total": 84.37}, "nlu": {"metaphor": {"total": 70.08}, "qa": {"total": 79.88}, "sentiment": {"total": 74.32}, "total": 74.76}, "instruction-following": {"total": 93.49}, "multi-turn": {"total": 31.01}, "safety": {"toxicity": {"total": 49.36}, "total": 49.36}, "linguistic-diagnostics": {"syntax": {"total": 43.68}, "pragmatics": {"total": 57.79}, "total": 50.74}, "total": 62.81}, "jv": {"nlg": {"translation-id-xx": {"total": 83.66}, "translation-xx-id": {"total": 87.14}, "total": 85.4}, "nlu": {"metaphor": {"total": 40.03}, "qa-mc": {"total": 88.1}, "sentiment": {"total": 67.97}, "total": 65.36}, "instruction-following": {"total": 42.57}, "multi-turn": {"total": 20.65}, "total": 53.5}, "su": {"nlg": {"translation-id-xx": {"total": 80.28}, "translation-xx-id": {"total": 83.44}, "total": 81.86}, "nlu": {"metaphor": {"total": 26.56}, "qa-mc": {"total": 77.62}, "sentiment": {"total": 65.21}, "total": 56.46}, "instruction-following": {"total": 64.19}, "multi-turn": {"total": 24.86}, "total": 56.84}, "ban": {"nlg": {"translation-id-xx": {"total": 72.81}, "translation-xx-id": {"total": 76.51}, "total": 74.66}, "nlu": {"sentiment": {"total": 68.84}, "total": 68.84}, "instruction-following": {"total": 43.05}, "multi-turn": {"total": 19.5}, "total": 51.51}, "bbc": {"nlg": {"translation-id-xx": {"total": 48.21}, "translation-xx-id": {"total": 60.79}, "total": 54.5}, "nlu": {"qa": {"total": 43.84}, "sentiment": {"total": 15.37}, "total": 29.6}, "instruction-following": {"total": 15.21}, "multi-turn": {"total": 16.22}, "total": 28.88}, "indommlu": {"total": 61.48, "Humanities": {"total": 70.28}, "Indonesian language": {"total": 64.78}, "Local languages and cultures": {"total": 42.1}, "Social science": {"total": 67.94}, "STEM": {"total": 62.29}}} |
| {"model_name": "google/gemma-2-9b", "model_size": "9B", "model_type": "Base", "overall": {"total": 44.41}, "id": {"nlg": {"translation-en-xx": {"total": 85.98}, "translation-xx-en": {"total": 88.69}, "abssum": {"total": 17.4}, "total": 52.37}, "nlr": {"nli": {"total": 3.47}, "causal": {"total": 75.6}, "total": 39.53}, "nlu": {"metaphor": {"total": 62.59}, "qa": {"total": 71.2}, "sentiment": {"total": 79.54}, "total": 71.11}, "safety": {"toxicity": {"total": 52.45}, "total": 52.45}, "linguistic-diagnostics": {"syntax": {"total": 2.11}, "pragmatics": {"total": 34.55}, "total": 18.33}, "total": 46.76}, "jv": {"nlg": {"translation-id-xx": {"total": 40.55}, "translation-xx-id": {"total": 74.96}, "total": 57.75}, "nlu": {"metaphor": {"total": 28.59}, "qa-mc": {"total": 54.29}, "sentiment": {"total": 69.95}, "total": 50.94}, "total": 54.35}, "su": {"nlg": {"translation-id-xx": {"total": 31.64}, "translation-xx-id": {"total": 67.53}, "total": 49.59}, "nlu": {"metaphor": {"total": 14.31}, "qa-mc": {"total": 50.48}, "sentiment": {"total": 57.16}, "total": 40.65}, "total": 45.12}, "ban": {"nlg": {"translation-id-xx": {"total": 35.39}, "translation-xx-id": {"total": 61.19}, "total": 48.29}, "nlu": {"sentiment": {"total": 48.6}, "total": 48.6}, "total": 48.44}, "bbc": {"nlg": {"translation-id-xx": {"total": 17.25}, "translation-xx-id": {"total": 45.97}, "total": 31.61}, "nlu": {"qa": {"total": 26.26}, "sentiment": {"total": 20.0}, "total": 23.13}, "total": 27.37}} |
| {"model_name": "google/gemma-2-9b-it", "model_size": "9B", "model_type": "Instruct", "overall": {"total": 35.19}, "id": {"nlg": {"translation-en-xx": {"total": 91.1}, "translation-xx-en": {"total": 92.5}, "abssum": {"total": 19.18}, "total": 55.49}, "nlr": {"nli": {"total": 65.49}, "causal": {"total": 89.6}, "total": 77.54}, "nlu": {"metaphor": {"total": 68.74}, "qa": {"total": 77.45}, "sentiment": {"total": 67.83}, "total": 71.34}, "instruction-following": {"total": 80.1}, "multi-turn": {"total": 16.31}, "safety": {"toxicity": {"total": 37.97}, "total": 37.97}, "linguistic-diagnostics": {"syntax": {"total": 24.21}, "pragmatics": {"total": 58.9}, "total": 41.56}, "total": 54.33}, "jv": {"nlg": {"translation-id-xx": {"total": 53.1}, "translation-xx-id": {"total": 81.5}, "total": 67.3}, "nlu": {"metaphor": {"total": 37.85}, "qa-mc": {"total": 73.81}, "sentiment": {"total": 61.22}, "total": 57.63}, "instruction-following": {"total": 17.24}, "multi-turn": {"total": 8.82}, "total": 37.75}, "su": {"nlg": {"translation-id-xx": {"total": 37.26}, "translation-xx-id": {"total": 75.34}, "total": 56.3}, "nlu": {"metaphor": {"total": 28.27}, "qa-mc": {"total": 66.67}, "sentiment": {"total": 60.37}, "total": 51.77}, "instruction-following": {"total": 20.95}, "multi-turn": {"total": 10.09}, "total": 34.78}, "ban": {"nlg": {"translation-id-xx": {"total": 28.03}, "translation-xx-id": {"total": 68.04}, "total": 48.04}, "nlu": {"sentiment": {"total": 51.33}, "total": 51.33}, "instruction-following": {"total": 19.05}, "multi-turn": {"total": 5.37}, "total": 30.95}, "bbc": {"nlg": {"translation-id-xx": {"total": 23.26}, "translation-xx-id": {"total": 47.28}, "total": 35.27}, "nlu": {"qa": {"total": 26.64}, "sentiment": {"total": 19.41}, "total": 23.02}, "instruction-following": {"total": 8.57}, "multi-turn": {"total": 5.76}, "total": 18.16}, "indommlu": {"total": 48.54, "Humanities": {"total": 54.59}, "Indonesian language": {"total": 54.96}, "Local languages and cultures": {"total": 25.28}, "Social science": {"total": 57.65}, "STEM": {"total": 50.18}}} |
| {"model_name": "google/gemma-3-27b-it", "model_size": "27B", "model_type": "Instruct", "overall": {"total": 59.69}, "id": {"nlg": {"translation-en-xx": {"total": 94.0}, "translation-xx-en": {"total": 93.09}, "abssum": {"total": 16.72}, "total": 55.13}, "nlr": {"nli": {"total": 72.84}, "causal": {"total": 89.0}, "total": 80.92}, "nlu": {"metaphor": {"total": 68.74}, "qa": {"total": 82.54}, "sentiment": {"total": 74.54}, "total": 75.27}, "instruction-following": {"total": 85.94}, "multi-turn": {"total": 54.21}, "safety": {"toxicity": {"total": 47.3}, "total": 47.3}, "linguistic-diagnostics": {"syntax": {"total": 34.95}, "pragmatics": {"total": 71.51}, "total": 53.23}, "total": 64.57}, "jv": {"nlg": {"translation-id-xx": {"total": 84.22}, "translation-xx-id": {"total": 91.24}, "total": 87.73}, "nlu": {"metaphor": {"total": 53.52}, "qa-mc": {"total": 87.14}, "sentiment": {"total": 70.94}, "total": 70.53}, "instruction-following": {"total": 64.76}, "multi-turn": {"total": 48.3}, "total": 67.83}, "su": {"nlg": {"translation-id-xx": {"total": 76.38}, "translation-xx-id": {"total": 88.75}, "total": 82.57}, "nlu": {"metaphor": {"total": 43.77}, "qa-mc": {"total": 80.0}, "sentiment": {"total": 71.1}, "total": 64.96}, "instruction-following": {"total": 71.02}, "multi-turn": {"total": 46.01}, "total": 66.14}, "ban": {"nlg": {"translation-id-xx": {"total": 59.77}, "translation-xx-id": {"total": 80.9}, "total": 70.33}, "nlu": {"sentiment": {"total": 66.5}, "total": 66.5}, "instruction-following": {"total": 48.44}, "multi-turn": {"total": 43.79}, "total": 57.27}, "bbc": {"nlg": {"translation-id-xx": {"total": 23.26}, "translation-xx-id": {"total": 70.56}, "total": 46.91}, "nlu": {"qa": {"total": 51.85}, "sentiment": {"total": 51.52}, "total": 51.68}, "instruction-following": {"total": 36.89}, "multi-turn": {"total": 35.06}, "total": 42.64}, "indommlu": {"total": 57.13, "Humanities": {"total": 62.02}, "Indonesian language": {"total": 60.62}, "Local languages and cultures": {"total": 38.62}, "Social science": {"total": 64.21}, "STEM": {"total": 60.16}}} |
| {"model_name": "meta-llama/Llama-3.1-70B-Instruct", "model_size": "70B", "model_type": "Instruct", "overall": {"total": 42.05}, "id": {"nlg": {"translation-en-xx": {"total": 91.72}, "translation-xx-en": {"total": 92.2}, "abssum": {"total": 22.31}, "total": 57.13}, "nlr": {"nli": {"total": 67.37}, "causal": {"total": 90.8}, "total": 79.09}, "nlu": {"metaphor": {"total": 72.35}, "qa": {"total": 75.27}, "sentiment": {"total": 73.45}, "total": 73.69}, "instruction-following": {"total": 79.62}, "multi-turn": {"total": 12.34}, "safety": {"toxicity": {"total": 53.5}, "total": 53.5}, "linguistic-diagnostics": {"syntax": {"total": 36.84}, "pragmatics": {"total": 65.76}, "total": 51.3}, "total": 58.1}, "jv": {"nlg": {"translation-id-xx": {"total": 79.24}, "translation-xx-id": {"total": 85.84}, "total": 82.54}, "nlu": {"metaphor": {"total": 38.98}, "qa-mc": {"total": 83.7}, "sentiment": {"total": 63.47}, "total": 62.05}, "instruction-following": {"total": 23.62}, "multi-turn": {"total": 10.82}, "total": 44.76}, "su": {"nlg": {"translation-id-xx": {"total": 74.43}, "translation-xx-id": {"total": 79.83}, "total": 77.13}, "nlu": {"metaphor": {"total": 25.89}, "qa-mc": {"total": 72.78}, "sentiment": {"total": 57.65}, "total": 52.1}, "instruction-following": {"total": 35.56}, "multi-turn": {"total": 9.61}, "total": 43.6}, "ban": {"nlg": {"translation-id-xx": {"total": 63.69}, "translation-xx-id": {"total": 74.28}, "total": 68.99}, "nlu": {"sentiment": {"total": 63.89}, "total": 63.89}, "instruction-following": {"total": 24.92}, "multi-turn": {"total": 8.18}, "total": 41.49}, "bbc": {"nlg": {"translation-id-xx": {"total": 39.56}, "translation-xx-id": {"total": 54.95}, "total": 47.26}, "nlu": {"qa": {"total": 37.55}, "sentiment": {"total": 19.32}, "total": 28.44}, "instruction-following": {"total": 5.78}, "multi-turn": {"total": 7.66}, "total": 22.28}, "indommlu": {"total": 60.61, "Humanities": {"total": 69.6}, "Indonesian language": {"total": 63.69}, "Local languages and cultures": {"total": 39.04}, "Social science": {"total": 68.13}, "STEM": {"total": 62.56}}} |
| {"model_name": "meta-llama/Llama-3.1-8B", "model_size": "8B", "model_type": "Base", "overall": {"total": 49.58}, "id": {"nlg": {"translation-en-xx": {"total": 88.27}, "translation-xx-en": {"total": 91.47}, "abssum": {"total": 21.95}, "total": 55.91}, "nlr": {"nli": {"total": 45.4}, "causal": {"total": 76.59}, "total": 61.0}, "nlu": {"metaphor": {"total": 62.68}, "qa": {"total": 76.06}, "sentiment": {"total": 76.98}, "total": 71.91}, "safety": {"toxicity": {"total": 36.33}, "total": 36.33}, "linguistic-diagnostics": {"syntax": {"total": 33.68}, "pragmatics": {"total": 17.0}, "total": 25.34}, "total": 50.1}, "jv": {"nlg": {"translation-id-xx": {"total": 60.9}, "translation-xx-id": {"total": 75.88}, "total": 68.39}, "nlu": {"metaphor": {"total": 21.73}, "qa-mc": {"total": 60.0}, "sentiment": {"total": 69.21}, "total": 50.31}, "total": 59.35}, "su": {"nlg": {"translation-id-xx": {"total": 53.8}, "translation-xx-id": {"total": 70.73}, "total": 62.27}, "nlu": {"metaphor": {"total": 7.07}, "qa-mc": {"total": 42.79}, "sentiment": {"total": 65.56}, "total": 38.47}, "total": 50.37}, "ban": {"nlg": {"translation-id-xx": {"total": 51.07}, "translation-xx-id": {"total": 63.42}, "total": 57.25}, "nlu": {"sentiment": {"total": 52.96}, "total": 52.96}, "total": 55.1}, "bbc": {"nlg": {"translation-id-xx": {"total": 21.92}, "translation-xx-id": {"total": 53.43}, "total": 37.68}, "nlu": {"qa": {"total": 33.1}, "sentiment": {"total": 23.56}, "total": 28.33}, "total": 33.0}} |
| {"model_name": "meta-llama/Llama-3.1-8B-Instruct", "model_size": "8B", "model_type": "Instruct", "overall": {"total": 28.92}, "id": {"nlg": {"translation-en-xx": {"total": 88.99}, "translation-xx-en": {"total": 91.23}, "abssum": {"total": 18.36}, "total": 54.24}, "nlr": {"nli": {"total": 45.62}, "causal": {"total": 76.76}, "total": 61.19}, "nlu": {"metaphor": {"total": 59.99}, "qa": {"total": 78.23}, "sentiment": {"total": 65.71}, "total": 67.98}, "instruction-following": {"total": 79.24}, "multi-turn": {"total": 10.98}, "safety": {"toxicity": {"total": 39.97}, "total": 39.97}, "linguistic-diagnostics": {"syntax": {"total": 17.18}, "pragmatics": {"total": 40.56}, "total": 28.87}, "total": 48.92}, "jv": {"nlg": {"translation-id-xx": {"total": 60.11}, "translation-xx-id": {"total": 74.39}, "total": 67.25}, "nlu": {"metaphor": {"total": 26.69}, "qa-mc": {"total": 62.0}, "sentiment": {"total": 49.45}, "total": 46.04}, "instruction-following": {"total": 12.89}, "multi-turn": {"total": 6.45}, "total": 33.16}, "su": {"nlg": {"translation-id-xx": {"total": 52.56}, "translation-xx-id": {"total": 64.91}, "total": 58.74}, "nlu": {"metaphor": {"total": 5.74}, "qa-mc": {"total": 42.87}, "sentiment": {"total": 35.27}, "total": 27.96}, "instruction-following": {"total": 7.75}, "multi-turn": {"total": 4.67}, "total": 24.78}, "ban": {"nlg": {"translation-id-xx": {"total": 50.75}, "translation-xx-id": {"total": 61.53}, "total": 56.14}, "nlu": {"sentiment": {"total": 20.36}, "total": 20.36}, "instruction-following": {"total": 7.11}, "multi-turn": {"total": 3.66}, "total": 21.82}, "bbc": {"nlg": {"translation-id-xx": {"total": 50.44}, "translation-xx-id": {"total": 40.52}, "total": 45.48}, "nlu": {"qa": {"total": 26.78}, "sentiment": {"total": 0.0}, "total": 13.39}, "instruction-following": {"total": 0.95}, "multi-turn": {"total": 3.82}, "total": 15.91}, "indommlu": {"total": 43.9, "Humanities": {"total": 49.56}, "Indonesian language": {"total": 51.35}, "Local languages and cultures": {"total": 26.67}, "Social science": {"total": 51.92}, "STEM": {"total": 40.03}}} |
| {"model_name": "meta-llama/Llama-3.3-70B-Instruct", "model_size": "70B", "model_type": "Instruct", "overall": {"total": 44.5}, "id": {"nlg": {"translation-en-xx": {"total": 91.33}, "translation-xx-en": {"total": 92.25}, "abssum": {"total": 18.41}, "total": 55.1}, "nlr": {"nli": {"total": 77.22}, "causal": {"total": 93.33}, "total": 85.28}, "nlu": {"metaphor": {"total": 75.22}, "qa": {"total": 79.74}, "sentiment": {"total": 72.99}, "total": 75.98}, "instruction-following": {"total": 93.33}, "multi-turn": {"total": 16.67}, "safety": {"toxicity": {"total": 51.55}, "total": 51.55}, "linguistic-diagnostics": {"syntax": {"total": 37.68}, "pragmatics": {"total": 70.06}, "total": 53.87}, "total": 61.68}, "jv": {"nlg": {"translation-id-xx": {"total": 77.27}, "translation-xx-id": {"total": 87.23}, "total": 82.25}, "nlu": {"metaphor": {"total": 47.49}, "qa-mc": {"total": 85.52}, "sentiment": {"total": 66.22}, "total": 66.41}, "instruction-following": {"total": 27.59}, "multi-turn": {"total": 15.89}, "total": 48.03}, "su": {"nlg": {"translation-id-xx": {"total": 71.81}, "translation-xx-id": {"total": 81.53}, "total": 76.67}, "nlu": {"metaphor": {"total": 29.09}, "qa-mc": {"total": 76.38}, "sentiment": {"total": 60.54}, "total": 55.34}, "instruction-following": {"total": 43.9}, "multi-turn": {"total": 15.37}, "total": 47.82}, "ban": {"nlg": {"translation-id-xx": {"total": 63.34}, "translation-xx-id": {"total": 74.66}, "total": 69.0}, "nlu": {"sentiment": {"total": 56.9}, "total": 56.9}, "instruction-following": {"total": 25.37}, "multi-turn": {"total": 12.93}, "total": 41.05}, "bbc": {"nlg": {"translation-id-xx": {"total": 40.44}, "translation-xx-id": {"total": 57.2}, "total": 48.82}, "nlu": {"qa": {"total": 35.99}, "sentiment": {"total": 10.67}, "total": 23.33}, "instruction-following": {"total": 13.27}, "multi-turn": {"total": 10.14}, "total": 23.89}, "indommlu": {"total": 60.28, "Humanities": {"total": 70.92}, "Indonesian language": {"total": 63.92}, "Local languages and cultures": {"total": 37.24}, "Social science": {"total": 67.82}, "STEM": {"total": 61.5}}} |
| {"model_name": "meta-llama/Meta-Llama-3-8B", "model_size": "8B", "model_type": "Base", "overall": {"total": 46.79}, "id": {"nlg": {"translation-en-xx": {"total": 88.0}, "translation-xx-en": {"total": 91.32}, "abssum": {"total": 21.67}, "total": 55.66}, "nlr": {"nli": {"total": 11.84}, "causal": {"total": 68.77}, "total": 40.31}, "nlu": {"metaphor": {"total": 57.26}, "qa": {"total": 76.1}, "sentiment": {"total": 81.24}, "total": 71.53}, "safety": {"toxicity": {"total": 39.02}, "total": 39.02}, "linguistic-diagnostics": {"syntax": {"total": 24.21}, "pragmatics": {"total": 17.0}, "total": 20.61}, "total": 45.43}, "jv": {"nlg": {"translation-id-xx": {"total": 61.14}, "translation-xx-id": {"total": 75.46}, "total": 68.3}, "nlu": {"metaphor": {"total": 23.75}, "qa-mc": {"total": 50.0}, "sentiment": {"total": 67.78}, "total": 47.18}, "total": 57.74}, "su": {"nlg": {"translation-id-xx": {"total": 52.41}, "translation-xx-id": {"total": 69.89}, "total": 61.15}, "nlu": {"metaphor": {"total": 1.06}, "qa-mc": {"total": 38.57}, "sentiment": {"total": 62.94}, "total": 34.19}, "total": 47.67}, "ban": {"nlg": {"translation-id-xx": {"total": 51.47}, "translation-xx-id": {"total": 64.43}, "total": 57.95}, "nlu": {"sentiment": {"total": 46.9}, "total": 46.9}, "total": 52.43}, "bbc": {"nlg": {"translation-id-xx": {"total": 19.17}, "translation-xx-id": {"total": 51.16}, "total": 35.17}, "nlu": {"qa": {"total": 35.02}, "sentiment": {"total": 17.32}, "total": 26.17}, "total": 30.67}} |
| {"model_name": "meta-llama/Meta-Llama-3-8B-Instruct", "model_size": "8B", "model_type": "Instruct", "overall": {"total": 24.34}, "id": {"nlg": {"translation-en-xx": {"total": 87.08}, "translation-xx-en": {"total": 89.92}, "abssum": {"total": 18.17}, "total": 53.33}, "nlr": {"nli": {"total": 42.87}, "causal": {"total": 60.16}, "total": 51.51}, "nlu": {"metaphor": {"total": 53.81}, "qa": {"total": 77.8}, "sentiment": {"total": 62.27}, "total": 64.63}, "instruction-following": {"total": 31.43}, "multi-turn": {"total": 3.99}, "safety": {"toxicity": {"total": 30.11}, "total": 30.11}, "linguistic-diagnostics": {"syntax": {"total": 11.56}, "pragmatics": {"total": 36.81}, "total": 24.19}, "total": 37.03}, "jv": {"nlg": {"translation-id-xx": {"total": 48.67}, "translation-xx-id": {"total": 69.71}, "total": 59.19}, "nlu": {"metaphor": {"total": 23.08}, "qa-mc": {"total": 47.14}, "sentiment": {"total": 45.89}, "total": 38.71}, "instruction-following": {"total": 3.62}, "multi-turn": {"total": 5.4}, "total": 26.73}, "su": {"nlg": {"translation-id-xx": {"total": 59.72}, "translation-xx-id": {"total": 60.39}, "total": 60.05}, "nlu": {"metaphor": {"total": 6.18}, "qa-mc": {"total": 30.48}, "sentiment": {"total": 28.07}, "total": 21.57}, "instruction-following": {"total": 2.86}, "multi-turn": {"total": 4.09}, "total": 22.15}, "ban": {"nlg": {"translation-id-xx": {"total": 52.5}, "translation-xx-id": {"total": 53.75}, "total": 53.13}, "nlu": {"sentiment": {"total": 31.75}, "total": 31.75}, "instruction-following": {"total": 0.95}, "multi-turn": {"total": 5.19}, "total": 22.76}, "bbc": {"nlg": {"translation-id-xx": {"total": 31.46}, "translation-xx-id": {"total": 35.87}, "total": 33.66}, "nlu": {"qa": {"total": 31.17}, "sentiment": {"total": 0.0}, "total": 15.58}, "instruction-following": {"total": 1.9}, "multi-turn": {"total": 0.92}, "total": 13.02}, "indommlu": {"total": 38.82, "Humanities": {"total": 45.29}, "Indonesian language": {"total": 44.26}, "Local languages and cultures": {"total": 21.23}, "Social science": {"total": 47.58}, "STEM": {"total": 35.75}}} |
|
|
