Added JEV 27B VL AWQ

data/results.json

@@ -1,475 +1,490 @@
-{
-  "schema_version": "1.0",
-  "benchmark_version": "1.0",
-  "updated": "2026-10-06",
-  "models": [
-    {
-      "id": "openai-gpt-6-astra",
-      "name": "GPT-6 Astra",
-      "organization": "OpenAI",
-      "params": "—",
-      "type": "closed",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-10-06",
-      "model_url": "https://deploymentsafety.openai.com/gpt-6-astra",
-      "overall": 75.26,
-      "spatial": 83.12,
-      "spatio_temporal": 81.2,
-      "temporal": 51.64,
-      "semantic": 85.08,
-      "2d_localization": 82.2,
-      "2d_referring_expressions": 80.18,
-      "2d_spatial_pointing": 86.97,
-      "single_object_tracking": 81.2,
-      "temporal_localization": 65.1,
-      "dense_video_captioning": 38.17,
-      "event_verification": 86.9,
-      "video_qa": 83.26
-    },
-    {
-      "id": "google-gemini-3-6-flash",
-      "name": "Gemini 3.6 Flash",
-      "organization": "Google",
-      "params": "—",
-      "type": "closed",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-21",
-      "model_url": "https://deepmind.google/technologies/gemini/",
-      "overall": 69.52,
-      "spatial": 78.85,
-      "spatio_temporal": 75.99,
-      "temporal": 43.83,
-      "semantic": 79.41,
-      "2d_localization": 81.57,
-      "2d_referring_expressions": 75.78,
-      "2d_spatial_pointing": 79.2,
-      "single_object_tracking": 75.99,
-      "temporal_localization": 51.5,
-      "dense_video_captioning": 36.15,
-      "event_verification": 82.0,
-      "video_qa": 76.82
-    },
-    {
-      "id": "openai-gpt-5-6-sol",
-      "name": "GPT-5.6 Sol",
-      "organization": "OpenAI",
-      "params": "—",
-      "type": "closed",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-09",
-      "model_url": "https://openai.com/index/gpt-5-6/",
-      "overall": 68.04,
-      "spatial": 73.55,
-      "spatio_temporal": 75.02,
-      "temporal": 46.5,
-      "semantic": 77.11,
-      "2d_localization": 69.3,
-      "2d_referring_expressions": 69.44,
-      "2d_spatial_pointing": 81.9,
-      "single_object_tracking": 75.02,
-      "temporal_localization": 55.71,
-      "dense_video_captioning": 37.28,
-      "event_verification": 76.14,
-      "video_qa": 78.08
-    },
-    {
-      "id": "nvidia-cosmos3-super",
-      "name": "Cosmos3-Super",
-      "organization": "NVIDIA",
-      "params": "64B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-05-29",
-      "model_url": "https://huggingface.co/nvidia/Cosmos3-Super",
-      "overall": 63.62,
-      "spatial": 78.71,
-      "spatio_temporal": 64.66,
-      "temporal": 40.72,
-      "semantic": 70.37,
-      "2d_localization": 86.97,
-      "2d_referring_expressions": 76.23,
-      "2d_spatial_pointing": 72.94,
-      "single_object_tracking": 64.66,
-      "temporal_localization": 51.9,
-      "dense_video_captioning": 29.54,
-      "event_verification": 71.28,
-      "video_qa": 69.46
-    },
-    {
-      "id": "google-gemini-3-1-pro",
-      "name": "Gemini 3.1 Pro",
-      "organization": "Google",
-      "params": "—",
-      "type": "closed",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://deepmind.google/technologies/gemini/",
-      "overall": 62.66,
-      "spatial": 72.14,
-      "spatio_temporal": 67.88,
-      "temporal": 40.62,
-      "semantic": 70.01,
-      "2d_localization": 77.21,
-      "2d_referring_expressions": 67.17,
-      "2d_spatial_pointing": 72.04,
-      "single_object_tracking": 67.88,
-      "temporal_localization": 45.69,
-      "dense_video_captioning": 35.55,
-      "event_verification": 68.57,
-      "video_qa": 71.46
-    },
-    {
-      "id": "nvidia-cosmos3-nano",
-      "name": "Cosmos3-Nano",
-      "organization": "NVIDIA",
-      "params": "16B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-05-29",
-      "model_url": "https://huggingface.co/nvidia/Cosmos3-Nano",
-      "overall": 60.67,
-      "spatial": 74.84,
-      "spatio_temporal": 59.21,
-      "temporal": 39.73,
-      "semantic": 68.91,
-      "2d_localization": 74.11,
-      "2d_referring_expressions": 75.57,
-      "2d_spatial_pointing": 74.83,
-      "single_object_tracking": 59.21,
-      "temporal_localization": 48.04,
-      "dense_video_captioning": 31.42,
-      "event_verification": 68.88,
-      "video_qa": 68.95
-    },
-    {
-      "id": "alibaba-qwen3-vl-32b",
-      "name": "Qwen3-VL-32B-Instruct",
-      "organization": "Alibaba",
-      "params": "32B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct",
-      "overall": 55.38,
-      "spatial": 73.56,
-      "spatio_temporal": 44.19,
-      "temporal": 38.1,
-      "semantic": 65.67,
-      "2d_localization": 72.67,
-      "2d_referring_expressions": 72.39,
-      "2d_spatial_pointing": 75.62,
-      "single_object_tracking": 44.19,
-      "temporal_localization": 46.8,
-      "dense_video_captioning": 29.4,
-      "event_verification": 60.04,
-      "video_qa": 71.3
-    },
-    {
-      "id": "nvidia-cosmos-reason2-8b",
-      "name": "Cosmos-Reason2-8B",
-      "organization": "NVIDIA",
-      "params": "8B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-8B",
-      "overall": 54.46,
-      "spatial": 74.25,
-      "spatio_temporal": 37.69,
-      "temporal": 39.9,
-      "semantic": 66.02,
-      "2d_localization": 83.88,
-      "2d_referring_expressions": 70.26,
-      "2d_spatial_pointing": 68.6,
-      "single_object_tracking": 37.69,
-      "temporal_localization": 47.3,
-      "dense_video_captioning": 32.5,
-      "event_verification": 64.09,
-      "video_qa": 67.95
-    },
-    {
-      "id": "google-gemini-3-5-flash-lite",
-      "name": "Gemini 3.5 Flash-Lite",
-      "organization": "Google",
-      "params": "—",
-      "type": "closed",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-21",
-      "model_url": "https://deepmind.google/models/gemini/flash-lite/",
-      "overall": 53.25,
-      "spatial": 61.09,
-      "spatio_temporal": 55.77,
-      "temporal": 28.14,
-      "semantic": 68.01,
-      "2d_localization": 65.73,
-      "2d_referring_expressions": 74.46,
-      "2d_spatial_pointing": 43.08,
-      "single_object_tracking": 55.77,
-      "temporal_localization": 24.0,
-      "dense_video_captioning": 32.29,
-      "event_verification": 70.41,
-      "video_qa": 65.61
-    },
-    {
-      "id": "google-gemini-3-2-flash-lite",
-      "name": "Gemini 3.1 Flash Lite",
-      "organization": "Google",
-      "params": "—",
-      "type": "closed",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://deepmind.google/technologies/gemini/",
-      "overall": 52.61,
-      "spatial": 62.71,
-      "spatio_temporal": 46.68,
-      "temporal": 35.13,
-      "semantic": 65.91,
-      "2d_localization": 71.74,
-      "2d_referring_expressions": 52.7,
-      "2d_spatial_pointing": 63.68,
-      "single_object_tracking": 46.68,
-      "temporal_localization": 37.72,
-      "dense_video_captioning": 32.54,
-      "event_verification": 63.78,
-      "video_qa": 68.03
-    },
-    {
-      "id": "alibaba-qwen3-vl-8b",
-      "name": "Qwen3-VL-8B-Instruct",
-      "organization": "Alibaba",
-      "params": "8B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct",
-      "overall": 50.07,
-      "spatial": 67.26,
-      "spatio_temporal": 33.12,
-      "temporal": 36.98,
-      "semantic": 62.91,
-      "2d_localization": 59.93,
-      "2d_referring_expressions": 73.29,
-      "2d_spatial_pointing": 68.56,
-      "single_object_tracking": 33.12,
-      "temporal_localization": 44.32,
-      "dense_video_captioning": 29.64,
-      "event_verification": 59.39,
-      "video_qa": 66.44
-    },
-    {
-      "id": "alibaba-qwen3-5-vl-27b",
-      "name": "Qwen3.5-27B",
-      "organization": "Alibaba",
-      "params": "27B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-27B-Instruct",
-      "overall": 49.25,
-      "spatial": 79.04,
-      "spatio_temporal": 24.4,
-      "temporal": 31.93,
-      "semantic": 61.65,
-      "2d_localization": 85.46,
-      "2d_referring_expressions": 76.35,
-      "2d_spatial_pointing": 75.32,
-      "single_object_tracking": 24.4,
-      "temporal_localization": 36.88,
-      "dense_video_captioning": 26.97,
-      "event_verification": 55.34,
-      "video_qa": 67.95
-    },
-    {
-      "id": "alibaba-qwen3-5-vl-9b",
-      "name": "Qwen3.5-9B",
-      "organization": "Alibaba",
-      "params": "9B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-9B-Instruct",
-      "overall": 48.3,
-      "spatial": 75.26,
-      "spatio_temporal": 23.52,
-      "temporal": 35.2,
-      "semantic": 59.2,
-      "2d_localization": 76.87,
-      "2d_referring_expressions": 77.16,
-      "2d_spatial_pointing": 71.74,
-      "single_object_tracking": 23.52,
-      "temporal_localization": 42.53,
-      "dense_video_captioning": 27.88,
-      "event_verification": 46.78,
-      "video_qa": 71.63
-    },
-    {
-      "id": "nvidia-cosmos-reason2-32b",
-      "name": "Cosmos-Reason2-32B",
-      "organization": "NVIDIA",
-      "params": "32B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": false,
-      "date_evaluated": "2026-05-27",
-      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-32B",
-      "overall": 47.03,
-      "spatial": 42.81,
-      "spatio_temporal": 33.12,
-      "temporal": 40.25,
-      "semantic": 71.94,
-      "2d_localization": 4.35,
-      "2d_referring_expressions": 50.04,
-      "2d_spatial_pointing": 74.03,
-      "single_object_tracking": 33.12,
-      "temporal_localization": 50.47,
-      "dense_video_captioning": 30.04,
-      "event_verification": 73.58,
-      "video_qa": 70.29
-    },
-    {
-      "id": "nvidia-cosmos3-edge",
-      "name": "Cosmos3-Edge",
-      "organization": "NVIDIA",
-      "params": "4B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-16",
-      "model_url": "https://huggingface.co/nvidia/Cosmos3-Edge",
-      "overall": 46.94,
-      "spatial": 72.02,
-      "spatio_temporal": 24.65,
-      "temporal": 28.07,
-      "semantic": 63.02,
-      "2d_localization": 79.02,
-      "2d_referring_expressions": 69.27,
-      "2d_spatial_pointing": 67.76,
-      "single_object_tracking": 24.65,
-      "temporal_localization": 39.51,
-      "dense_video_captioning": 16.64,
-      "event_verification": 63.36,
-      "video_qa": 62.68
-    },
-    {
-      "id": "nvidia-cosmos-reason2-2b",
-      "name": "Cosmos-Reason2-2B",
-      "organization": "NVIDIA",
-      "params": "2B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-16",
-      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-2B",
-      "overall": 45.94,
-      "spatial": 63.2,
-      "spatio_temporal": 26.85,
-      "temporal": 33.73,
-      "semantic": 59.98,
-      "2d_localization": 73.56,
-      "2d_referring_expressions": 56.33,
-      "2d_spatial_pointing": 59.7,
-      "single_object_tracking": 26.85,
-      "temporal_localization": 38.95,
-      "dense_video_captioning": 28.52,
-      "event_verification": 55.27,
-      "video_qa": 64.69
-    },
-    {
-      "id": "alibaba-qwen3-vl-2b",
-      "name": "Qwen3-VL-2B-Instruct",
-      "organization": "Alibaba",
-      "params": "2B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-16",
-      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct",
-      "overall": 41.86,
-      "spatial": 55.22,
-      "spatio_temporal": 30.38,
-      "temporal": 27.55,
-      "semantic": 54.29,
-      "2d_localization": 47.42,
-      "2d_referring_expressions": 65.12,
-      "2d_spatial_pointing": 53.13,
-      "single_object_tracking": 30.38,
-      "temporal_localization": 35.21,
-      "dense_video_captioning": 19.9,
-      "event_verification": 44.73,
-      "video_qa": 63.85
-    },
-    {
-      "id": "google-gemma-4-e2b-it",
-      "name": "Gemma-4-E2B-it",
-      "organization": "Google",
-      "params": "2B",
-      "type": "open",
-      "result_type": "single",
-      "verified": true,
-      "ensemble": false,
-      "is_new": true,
-      "date_evaluated": "2026-07-16",
-      "model_url": "https://huggingface.co/google/gemma-4-e2b-it",
-      "overall": 21.28,
-      "spatial": 27.55,
-      "spatio_temporal": 11.58,
-      "temporal": 9.08,
-      "semantic": 36.92,
-      "2d_localization": 45.4,
-      "2d_referring_expressions": 5.52,
-      "2d_spatial_pointing": 31.74,
-      "single_object_tracking": 11.58,
-      "temporal_localization": 9.02,
-      "dense_video_captioning": 9.14,
-      "event_verification": 27.65,
-      "video_qa": 46.19
-    }
-  ]
-}
+{
+  "schema_version": "1.0",
+  "benchmark_version": "1.0",
+  "updated": "2026-10-21",
+  "models": [
+    {
+      "id": "openai-gpt-6-astra",
+      "name": "GPT-6 Astra",
+      "organization": "OpenAI",
+      "params": "—",
+      "type": "closed",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-10-06",
+      "model_url": "https://deploymentsafety.openai.com/gpt-6-astra",
+      "overall": 75.26,
+      "spatial": 83.12,
+      "spatio_temporal": 81.2,
+      "temporal": 51.64,
+      "semantic": 85.08,
+      "2d_localization": 82.2,
+      "2d_referring_expressions": 80.18,
+      "2d_spatial_pointing": 86.97,
+      "single_object_tracking": 81.2,
+      "temporal_localization": 65.1,
+      "dense_video_captioning": 38.17,
+      "event_verification": 86.9,
+      "video_qa": 83.26
+    },
+    {
+      "id": "google-gemini-3-6-flash",
+      "name": "Gemini 3.6 Flash",
+      "organization": "Google",
+      "params": "—",
+      "type": "closed",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-21",
+      "model_url": "https://deepmind.google/technologies/gemini/",
+      "overall": 69.52,
+      "spatial": 78.85,
+      "spatio_temporal": 75.99,
+      "temporal": 43.83,
+      "semantic": 79.41,
+      "2d_localization": 81.57,
+      "2d_referring_expressions": 75.78,
+      "2d_spatial_pointing": 79.2,
+      "single_object_tracking": 75.99,
+      "temporal_localization": 51.5,
+      "dense_video_captioning": 36.15,
+      "event_verification": 82.0,
+      "video_qa": 76.82
+    },
+    {
+      "id": "openai-gpt-5-6-sol",
+      "name": "GPT-5.6 Sol",
+      "organization": "OpenAI",
+      "params": "—",
+      "type": "closed",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-09",
+      "model_url": "https://openai.com/index/gpt-5-6/",
+      "overall": 68.04,
+      "spatial": 73.55,
+      "spatio_temporal": 75.02,
+      "temporal": 46.5,
+      "semantic": 77.11,
+      "2d_localization": 69.3,
+      "2d_referring_expressions": 69.44,
+      "2d_spatial_pointing": 81.9,
+      "single_object_tracking": 75.02,
+      "temporal_localization": 55.71,
+      "dense_video_captioning": 37.28,
+      "event_verification": 76.14,
+      "video_qa": 78.08
+    },
+    {
+      "id": "nvidia-cosmos3-super",
+      "name": "Cosmos3-Super",
+      "organization": "NVIDIA",
+      "params": "64B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-05-29",
+      "model_url": "https://huggingface.co/nvidia/Cosmos3-Super",
+      "overall": 63.62,
+      "spatial": 78.71,
+      "spatio_temporal": 64.66,
+      "temporal": 40.72,
+      "semantic": 70.37,
+      "2d_localization": 86.97,
+      "2d_referring_expressions": 76.23,
+      "2d_spatial_pointing": 72.94,
+      "single_object_tracking": 64.66,
+      "temporal_localization": 51.9,
+      "dense_video_captioning": 29.54,
+      "event_verification": 71.28,
+      "video_qa": 69.46
+    },
+    {
+      "id": "google-gemini-3-1-pro",
+      "name": "Gemini 3.1 Pro",
+      "organization": "Google",
+      "params": "—",
+      "type": "closed",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://deepmind.google/technologies/gemini/",
+      "overall": 62.66,
+      "spatial": 72.14,
+      "spatio_temporal": 67.88,
+      "temporal": 40.62,
+      "semantic": 70.01,
+      "2d_localization": 77.21,
+      "2d_referring_expressions": 67.17,
+      "2d_spatial_pointing": 72.04,
+      "single_object_tracking": 67.88,
+      "temporal_localization": 45.69,
+      "dense_video_captioning": 35.55,
+      "event_verification": 68.57,
+      "video_qa": 71.46
+    },
+    {
+      "id": "nvidia-cosmos3-nano",
+      "name": "Cosmos3-Nano",
+      "organization": "NVIDIA",
+      "params": "16B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-05-29",
+      "model_url": "https://huggingface.co/nvidia/Cosmos3-Nano",
+      "overall": 60.67,
+      "spatial": 74.84,
+      "spatio_temporal": 59.21,
+      "temporal": 39.73,
+      "semantic": 68.91,
+      "2d_localization": 74.11,
+      "2d_referring_expressions": 75.57,
+      "2d_spatial_pointing": 74.83,
+      "single_object_tracking": 59.21,
+      "temporal_localization": 48.04,
+      "dense_video_captioning": 31.42,
+      "event_verification": 68.88,
+      "video_qa": 68.95
+    },
+    {
+      "id": "alibaba-qwen3-vl-32b",
+      "name": "Qwen3-VL-32B-Instruct",
+      "organization": "Alibaba",
+      "params": "32B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct",
+      "overall": 55.38,
+      "spatial": 73.56,
+      "spatio_temporal": 44.19,
+      "temporal": 38.1,
+      "semantic": 65.67,
+      "2d_localization": 72.67,
+      "2d_referring_expressions": 72.39,
+      "2d_spatial_pointing": 75.62,
+      "single_object_tracking": 44.19,
+      "temporal_localization": 46.8,
+      "dense_video_captioning": 29.4,
+      "event_verification": 60.04,
+      "video_qa": 71.3
+    },
+    {
+      "id": "nvidia-cosmos-reason2-8b",
+      "name": "Cosmos-Reason2-8B",
+      "organization": "NVIDIA",
+      "params": "8B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-8B",
+      "overall": 54.46,
+      "spatial": 74.25,
+      "spatio_temporal": 37.69,
+      "temporal": 39.9,
+      "semantic": 66.02,
+      "2d_localization": 83.88,
+      "2d_referring_expressions": 70.26,
+      "2d_spatial_pointing": 68.6,
+      "single_object_tracking": 37.69,
+      "temporal_localization": 47.3,
+      "dense_video_captioning": 32.5,
+      "event_verification": 64.09,
+      "video_qa": 67.95
+    },
+    {
+      "id": "google-gemini-3-5-flash-lite",
+      "name": "Gemini 3.5 Flash-Lite",
+      "organization": "Google",
+      "params": "—",
+      "type": "closed",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-21",
+      "model_url": "https://deepmind.google/models/gemini/flash-lite/",
+      "overall": 53.25,
+      "spatial": 61.09,
+      "spatio_temporal": 55.77,
+      "temporal": 28.14,
+      "semantic": 68.01,
+      "2d_localization": 65.73,
+      "2d_referring_expressions": 74.46,
+      "2d_spatial_pointing": 43.08,
+      "single_object_tracking": 55.77,
+      "temporal_localization": 24.0,
+      "dense_video_captioning": 32.29,
+      "event_verification": 70.41,
+      "video_qa": 65.61
+    },
+    {
+      "id": "google-gemini-3-2-flash-lite",
+      "name": "Gemini 3.1 Flash Lite",
+      "organization": "Google",
+      "params": "—",
+      "type": "closed",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://deepmind.google/technologies/gemini/",
+      "overall": 52.61,
+      "spatial": 62.71,
+      "spatio_temporal": 46.68,
+      "temporal": 35.13,
+      "semantic": 65.91,
+      "2d_localization": 71.74,
+      "2d_referring_expressions": 52.7,
+      "2d_spatial_pointing": 63.68,
+      "single_object_tracking": 46.68,
+      "temporal_localization": 37.72,
+      "dense_video_captioning": 32.54,
+      "event_verification": 63.78,
+      "video_qa": 68.03
+    },
+    {
+      "id": "alibaba-qwen3-vl-8b",
+      "name": "Qwen3-VL-8B-Instruct",
+      "organization": "Alibaba",
+      "params": "8B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct",
+      "overall": 50.07,
+      "spatial": 67.26,
+      "spatio_temporal": 33.12,
+      "temporal": 36.98,
+      "semantic": 62.91,
+      "2d_localization": 59.93,
+      "2d_referring_expressions": 73.29,
+      "2d_spatial_pointing": 68.56,
+      "single_object_tracking": 33.12,
+      "temporal_localization": 44.32,
+      "dense_video_captioning": 29.64,
+      "event_verification": 59.39,
+      "video_qa": 66.44
+    },
+    {
+      "id": "alibaba-qwen3-5-vl-27b",
+      "name": "Qwen3.5-27B",
+      "organization": "Alibaba",
+      "params": "27B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-27B-Instruct",
+      "overall": 49.25,
+      "spatial": 79.04,
+      "spatio_temporal": 24.4,
+      "temporal": 31.93,
+      "semantic": 61.65,
+      "2d_localization": 85.46,
+      "2d_referring_expressions": 76.35,
+      "2d_spatial_pointing": 75.32,
+      "single_object_tracking": 24.4,
+      "temporal_localization": 36.88,
+      "dense_video_captioning": 26.97,
+      "event_verification": 55.34,
+      "video_qa": 67.95
+    },
+    {
+      "id": "alibaba-qwen3-5-vl-9b",
+      "name": "Qwen3.5-9B",
+      "organization": "Alibaba",
+      "params": "9B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://huggingface.co/Qwen/Qwen3.5-VL-9B-Instruct",
+      "overall": 48.3,
+      "spatial": 75.26,
+      "spatio_temporal": 23.52,
+      "temporal": 35.2,
+      "semantic": 59.2,
+      "2d_localization": 76.87,
+      "2d_referring_expressions": 77.16,
+      "2d_spatial_pointing": 71.74,
+      "single_object_tracking": 23.52,
+      "temporal_localization": 42.53,
+      "dense_video_captioning": 27.88,
+      "event_verification": 46.78,
+      "video_qa": 71.63
+    },
+    {
+      "id": "nvidia-cosmos-reason2-32b",
+      "name": "Cosmos-Reason2-32B",
+      "organization": "NVIDIA",
+      "params": "32B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": false,
+      "date_evaluated": "2026-05-27",
+      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-32B",
+      "overall": 47.03,
+      "spatial": 42.81,
+      "spatio_temporal": 33.12,
+      "temporal": 40.25,
+      "semantic": 71.94,
+      "2d_localization": 4.35,
+      "2d_referring_expressions": 50.04,
+      "2d_spatial_pointing": 74.03,
+      "single_object_tracking": 33.12,
+      "temporal_localization": 50.47,
+      "dense_video_captioning": 30.04,
+      "event_verification": 73.58,
+      "video_qa": 70.29
+    },
+    {
+      "id": "nvidia-cosmos3-edge",
+      "name": "Cosmos3-Edge",
+      "organization": "NVIDIA",
+      "params": "4B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-16",
+      "model_url": "https://huggingface.co/nvidia/Cosmos3-Edge",
+      "overall": 46.94,
+      "spatial": 72.02,
+      "spatio_temporal": 24.65,
+      "temporal": 28.07,
+      "semantic": 63.02,
+      "2d_localization": 79.02,
+      "2d_referring_expressions": 69.27,
+      "2d_spatial_pointing": 67.76,
+      "single_object_tracking": 24.65,
+      "temporal_localization": 39.51,
+      "dense_video_captioning": 16.64,
+      "event_verification": 63.36,
+      "video_qa": 62.68
+    },
+    {
+      "id": "nvidia-cosmos-reason2-2b",
+      "name": "Cosmos-Reason2-2B",
+      "organization": "NVIDIA",
+      "params": "2B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-16",
+      "model_url": "https://huggingface.co/nvidia/Cosmos-Reason2-2B",
+      "overall": 45.94,
+      "spatial": 63.2,
+      "spatio_temporal": 26.85,
+      "temporal": 33.73,
+      "semantic": 59.98,
+      "2d_localization": 73.56,
+      "2d_referring_expressions": 56.33,
+      "2d_spatial_pointing": 59.7,
+      "single_object_tracking": 26.85,
+      "temporal_localization": 38.95,
+      "dense_video_captioning": 28.52,
+      "event_verification": 55.27,
+      "video_qa": 64.69
+    },
+    {
+      "id": "alibaba-qwen3-vl-2b",
+      "name": "Qwen3-VL-2B-Instruct",
+      "organization": "Alibaba",
+      "params": "2B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-16",
+      "model_url": "https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct",
+      "overall": 41.86,
+      "spatial": 55.22,
+      "spatio_temporal": 30.38,
+      "temporal": 27.55,
+      "semantic": 54.29,
+      "2d_localization": 47.42,
+      "2d_referring_expressions": 65.12,
+      "2d_spatial_pointing": 53.13,
+      "single_object_tracking": 30.38,
+      "temporal_localization": 35.21,
+      "dense_video_captioning": 19.9,
+      "event_verification": 44.73,
+      "video_qa": 63.85
+    },
+    {
+      "id": "google-gemma-4-e2b-it",
+      "name": "Gemma-4-E2B-it",
+      "organization": "Google",
+      "params": "2B",
+      "type": "open",
+      "result_type": "single",
+      "verified": true,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-07-16",
+      "model_url": "https://huggingface.co/google/gemma-4-e2b-it",
+      "overall": 21.28,
+      "spatial": 27.55,
+      "spatio_temporal": 11.58,
+      "temporal": 9.08,
+      "semantic": 36.92,
+      "2d_localization": 45.4,
+      "2d_referring_expressions": 5.52,
+      "2d_spatial_pointing": 31.74,
+      "single_object_tracking": 11.58,
+      "temporal_localization": 9.02,
+      "dense_video_captioning": 9.14,
+      "event_verification": 27.65,
+      "video_qa": 46.19
+    },
+    {
+      "id": "antrino-labs-jev-27b-vl-awq",
+      "name": "JEV 27B VL AWQ",
+      "organization": "Antrino Labs",
+      "params": "27B",
+      "type": "open",
+      "result_type": "single",
+      "verified": false,
+      "ensemble": false,
+      "is_new": true,
+      "date_evaluated": "2026-10-10",
+      "semantic": 61.63,
+      "event_verification": 56.98,
+      "video_qa": 66.28
+    }
+  ]
+}

util/data.py

@@ -1,359 +1,363 @@
-"""Load and validate the static leaderboard JSON.
-
-Public surface:
-    ModelRecord       — frozen dataclass for one row of the leaderboard.
-    LeaderboardData   — frozen dataclass wrapping the full dataset + index.
-    load_results_json — read + validate + build the dataset.
-
-Validation is strict: any schema violation raises ValueError with a
-single multi-line message listing every problem encountered. The app
-should let this propagate so the HF Space build fails loudly when the
-JSON is bad.
-"""
-
-from __future__ import annotations
-
-import json
-import re
-from dataclasses import dataclass
-from pathlib import Path
-
-from .config import (
-    ALLOWED_RESULT_TYPES,
-    ALLOWED_TYPES,
-    OPTIONAL_TASK_FIELDS,
-    REQUIRED_MODEL_FIELDS,
-    REQUIRED_SCORE_FIELDS,
-    SCHEMA_VERSION,
-    TYPE_DISPLAY,
-)
-
-
-# -- Dataclasses -----------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class ModelRecord:
-    """One model row, post-validation.
-
-    `scores` is a flat dict keyed by JSON field name (e.g. "overall",
-    "spatial", "2d_localization"). Optional task scores may be absent;
-    callers should use ``.score(field)`` which returns None when missing.
-    """
-
-    # Required fields
-    id: str
-    name: str
-    organization: str
-    params: str
-    type: str            # "open" or "closed"
-    result_type: str     # "single" or "ensemble"
-    scores: dict[str, float]
-    param_value: float | None
-    param_bucket: str | None
-    type_display: str
-
-    # Optional fields with defaults
-    url: str | None = None
-    verified: bool = False
-    ensemble: bool = False
-    is_new: bool = False
-    date_evaluated: str = ""
-    model_url: str = ""
-
-    def score(self, field_name: str) -> float | None:
-        return self.scores.get(field_name)
-
-
-@dataclass(frozen=True)
-class LeaderboardData:
-    schema_version: str
-    benchmark_version: str
-    updated: str  # YYYY-MM-DD as in the source JSON
-    models: list[ModelRecord]
-    model_by_id: dict[str, ModelRecord]
-
-
-# -- Internals -------------------------------------------------------------
-
-
-_PARAMS_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*B\s*$")
-_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
-_EM_DASH = "—"
-
-
-def _parse_param_value(params: str) -> float | None:
-    """Return the numeric param count in billions, or None if undisclosed/unparseable."""
-    if params.strip() == _EM_DASH:
-        return None
-    m = _PARAMS_RE.match(params)
-    return float(m.group(1)) if m else None
-
-
-def _param_bucket(value: float | None) -> str | None:
-    """Map a numeric param count (in billions) to a coarse bucket label.
-
-    Boundaries match util.config.PARAM_BUCKETS:
-        v <= 4        → "<4B"
-        4 < v < 10    → "4B–10B"
-        10 ≤ v ≤ 40   → "10B–40B"
-        v > 40        → ">40B"
-
-    None inputs return None (used to mark models with undisclosed sizes —
-    those models are visible only under the "All sizes" filter, never
-    under a numeric bucket).
-    """
-    if value is None:
-        return None
-    if value <= 4.0:
-        return "<4B"
-    if value < 10.0:
-        return "4B–10B"
-    if value <= 40.0:
-        return "10B–40B"
-    return ">40B"
-
-
-def _id_label(model_dict: dict, index: int) -> str:
-    raw = model_dict.get("id")
-    if isinstance(raw, str) and raw:
-        return f'models[{index}].id="{raw}"'
-    return f"models[{index}].id=<missing>"
-
-
-def _validate_score(
-    model_dict: dict, field: str, prefix: str, errors: list[str], required: bool
-) -> float | None:
-    """Validate one score field. Returns the float on success, None on failure
-    (or when an optional field is absent)."""
-    if field not in model_dict:
-        if required:
-            errors.append(f'{prefix}: missing required field "{field}"')
-        return None
-    v = model_dict[field]
-    if v is None:
-        errors.append(
-            f'{prefix}: field "{field}" is null; omit the field instead of using null'
-        )
-        return None
-    # bool is a subclass of int; reject it explicitly.
-    if isinstance(v, bool) or not isinstance(v, (int, float)):
-        errors.append(
-            f'{prefix}: field "{field}" must be numeric, got {type(v).__name__}'
-        )
-        return None
-    fv = float(v)
-    if not (0.0 <= fv <= 100.0):
-        errors.append(f'{prefix}: field "{field}" must be in [0, 100], got {fv}')
-        return None
-    return fv
-
-
-def _validate_optional_bool(
-    model_dict: dict, field_name: str, prefix: str, errors: list[str]
-) -> bool:
-    """Validate an optional bool field. Returns the value if valid, False if absent."""
-    if field_name not in model_dict:
-        return False
-    v = model_dict[field_name]
-    if not isinstance(v, bool):
-        errors.append(
-            f'{prefix}: field "{field_name}" must be a boolean, got {type(v).__name__}'
-        )
-        return False
-    return v
-
-
-def _fail(errors: list[str]) -> None:
-    msg = "[VANTAGE-Bench leaderboard] Schema validation failed:\n" + "\n".join(
-        f"  - {e}" for e in errors
-    )
-    raise ValueError(msg)
-
-
-# -- Public loader ---------------------------------------------------------
-
-
-def load_results_json(path: str | Path) -> LeaderboardData:
-    """Read, validate, and build the in-memory leaderboard dataset.
-
-    Hard-fails with ValueError listing every problem if validation does not pass.
-    """
-    p = Path(path)
-    try:
-        raw_text = p.read_text(encoding="utf-8")
-    except OSError as e:
-        raise ValueError(
-            f"[VANTAGE-Bench leaderboard] Could not read {p}: {e}"
-        ) from e
-
-    try:
-        data = json.loads(raw_text)
-    except json.JSONDecodeError as e:
-        raise ValueError(
-            f"[VANTAGE-Bench leaderboard] JSON parse error in {p}: {e}"
-        ) from e
-
-    if not isinstance(data, dict):
-        raise ValueError(
-            "[VANTAGE-Bench leaderboard] Schema validation failed:\n"
-            "  - top-level JSON must be an object"
-        )
-
-    errors: list[str] = []
-
-    # -- Top-level fields ---------------------------------------------------
-
-    sv = data.get("schema_version")
-    if sv != SCHEMA_VERSION:
-        errors.append(
-            f'schema_version must equal "{SCHEMA_VERSION}", got {sv!r}'
-        )
-
-    bv = data.get("benchmark_version")
-    if not isinstance(bv, str) or not bv:
-        errors.append("benchmark_version must be a non-empty string")
-
-    updated = data.get("updated")
-    if not isinstance(updated, str) or not _DATE_RE.match(updated):
-        errors.append(f"updated must be a YYYY-MM-DD string, got {updated!r}")
-
-    models_raw = data.get("models")
-    if not isinstance(models_raw, list) or not models_raw:
-        errors.append("models must be a non-empty list")
-        _fail(errors)
-        return  # unreachable; satisfies type checkers
-
-    # -- Per-model validation ----------------------------------------------
-
-    seen_ids: set[str] = set()
-
-    for i, m in enumerate(models_raw):
-        if not isinstance(m, dict):
-            errors.append(
-                f"models[{i}] must be an object, got {type(m).__name__}"
-            )
-            continue
-
-        prefix = _id_label(m, i)
-
-        # Required string fields
-        for fld in REQUIRED_MODEL_FIELDS:
-            if fld not in m:
-                errors.append(f'{prefix}: missing required field "{fld}"')
-            elif not isinstance(m[fld], str) or not m[fld]:
-                errors.append(
-                    f'{prefix}: field "{fld}" must be a non-empty string, got {m[fld]!r}'
-                )
-
-        # type must be in allowed set
-        if isinstance(m.get("type"), str) and m["type"] not in ALLOWED_TYPES:
-            errors.append(
-                f'{prefix}: field "type" must be one of {ALLOWED_TYPES}, got {m["type"]!r}'
-            )
-
-        # result_type must be in allowed set
-        rt = m.get("result_type")
-        if isinstance(rt, str) and rt not in ALLOWED_RESULT_TYPES:
-            errors.append(
-                f'{prefix}: field "result_type" must be one of {ALLOWED_RESULT_TYPES}, got {rt!r}'
-            )
-
-        # Optional bool fields
-        _validate_optional_bool(m, "verified", prefix, errors)
-        _validate_optional_bool(m, "ensemble", prefix, errors)
-        _validate_optional_bool(m, "is_new", prefix, errors)
-
-        # Uniqueness on id
-        mid = m.get("id")
-        if isinstance(mid, str) and mid:
-            if mid in seen_ids:
-                errors.append(f"{prefix}: duplicate id")
-            seen_ids.add(mid)
-
-        # url (optional legacy field — model name link)
-        if "url" in m:
-            url = m["url"]
-            if url is None:
-                errors.append(
-                    f'{prefix}: field "url" is null; omit the field instead of using null'
-                )
-            elif not isinstance(url, str) or not url:
-                errors.append(
-                    f'{prefix}: field "url" must be a non-empty string when present, got {url!r}'
-                )
-
-        # model_url (optional — HF model card or project page link)
-        if "model_url" in m:
-            mu = m["model_url"]
-            if not isinstance(mu, str):
-                errors.append(
-                    f'{prefix}: field "model_url" must be a string, got {type(mu).__name__}'
-                )
-
-        # date_evaluated (optional — YYYY-MM-DD or empty string)
-        if "date_evaluated" in m:
-            de = m["date_evaluated"]
-            if not isinstance(de, str):
-                errors.append(
-                    f'{prefix}: field "date_evaluated" must be a string, got {type(de).__name__}'
-                )
-            elif de and not _DATE_RE.match(de):
-                errors.append(
-                    f'{prefix}: field "date_evaluated" must be YYYY-MM-DD or empty, got {de!r}'
-                )
-
-        # Required score fields
-        for fld in REQUIRED_SCORE_FIELDS:
-            _validate_score(m, fld, prefix, errors, required=True)
-
-        # Optional task score fields
-        for fld in OPTIONAL_TASK_FIELDS:
-            _validate_score(m, fld, prefix, errors, required=False)
-
-    if errors:
-        _fail(errors)
-
-    # -- Build records (guaranteed valid past this point) ------------------
-
-    records: list[ModelRecord] = []
-    for m in models_raw:
-        scores: dict[str, float] = {}
-        for fld in REQUIRED_SCORE_FIELDS:
-            scores[fld] = float(m[fld])
-        for fld in OPTIONAL_TASK_FIELDS:
-            if fld in m:
-                scores[fld] = float(m[fld])
-
-        params_str = m["params"]
-        pv = _parse_param_value(params_str)
-        pb = _param_bucket(pv)
-
-        records.append(
-            ModelRecord(
-                id=m["id"],
-                name=m["name"],
-                organization=m["organization"],
-                params=params_str,
-                type=m["type"],
-                result_type=m["result_type"],
-                scores=scores,
-                param_value=pv,
-                param_bucket=pb,
-                type_display=TYPE_DISPLAY[m["type"]],
-                url=m.get("url"),
-                verified=bool(m.get("verified", False)),
-                ensemble=bool(m.get("ensemble", False)),
-                is_new=bool(m.get("is_new", False)),
-                date_evaluated=m.get("date_evaluated", ""),
-                model_url=m.get("model_url", ""),
-            )
-        )
-
-    return LeaderboardData(
-        schema_version=sv,
-        benchmark_version=bv,
-        updated=updated,
-        models=records,
-        model_by_id={r.id: r for r in records},
-    )
+"""Load and validate the static leaderboard JSON.
+
+Public surface:
+    ModelRecord       — frozen dataclass for one row of the leaderboard.
+    LeaderboardData   — frozen dataclass wrapping the full dataset + index.
+    load_results_json — read + validate + build the dataset.
+
+Validation is strict: any schema violation raises ValueError with a
+single multi-line message listing every problem encountered. The app
+should let this propagate so the HF Space build fails loudly when the
+JSON is bad.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from .config import (
+    ALLOWED_RESULT_TYPES,
+    ALLOWED_TYPES,
+    OPTIONAL_TASK_FIELDS,
+    REQUIRED_MODEL_FIELDS,
+    REQUIRED_SCORE_FIELDS,
+    SCHEMA_VERSION,
+    TYPE_DISPLAY,
+)
+
+
+# -- Dataclasses -----------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class ModelRecord:
+    """One model row, post-validation.
+
+    `scores` is a flat dict keyed by JSON field name (e.g. "overall",
+    "spatial", "2d_localization"). Any score may be absent;
+    callers should use ``.score(field)`` which returns None when missing.
+    """
+
+    # Required fields
+    id: str
+    name: str
+    organization: str
+    params: str
+    type: str            # "open" or "closed"
+    result_type: str     # "single" or "ensemble"
+    scores: dict[str, float]
+    param_value: float | None
+    param_bucket: str | None
+    type_display: str
+
+    # Optional fields with defaults
+    url: str | None = None
+    verified: bool = False
+    ensemble: bool = False
+    is_new: bool = False
+    date_evaluated: str = ""
+    model_url: str = ""
+
+    def score(self, field_name: str) -> float | None:
+        return self.scores.get(field_name)
+
+
+@dataclass(frozen=True)
+class LeaderboardData:
+    schema_version: str
+    benchmark_version: str
+    updated: str  # YYYY-MM-DD as in the source JSON
+    models: list[ModelRecord]
+    model_by_id: dict[str, ModelRecord]
+
+
+# -- Internals -------------------------------------------------------------
+
+
+_PARAMS_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*B\s*$")
+_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+_EM_DASH = "—"
+
+
+def _parse_param_value(params: str) -> float | None:
+    """Return the numeric param count in billions, or None if undisclosed/unparseable."""
+    if params.strip() == _EM_DASH:
+        return None
+    m = _PARAMS_RE.match(params)
+    return float(m.group(1)) if m else None
+
+
+def _param_bucket(value: float | None) -> str | None:
+    """Map a numeric param count (in billions) to a coarse bucket label.
+
+    Boundaries match util.config.PARAM_BUCKETS:
+        v <= 4        → "<4B"
+        4 < v < 10    → "4B–10B"
+        10 ≤ v ≤ 40   → "10B–40B"
+        v > 40        → ">40B"
+
+    None inputs return None (used to mark models with undisclosed sizes —
+    those models are visible only under the "All sizes" filter, never
+    under a numeric bucket).
+    """
+    if value is None:
+        return None
+    if value <= 4.0:
+        return "<4B"
+    if value < 10.0:
+        return "4B–10B"
+    if value <= 40.0:
+        return "10B–40B"
+    return ">40B"
+
+
+def _id_label(model_dict: dict, index: int) -> str:
+    raw = model_dict.get("id")
+    if isinstance(raw, str) and raw:
+        return f'models[{index}].id="{raw}"'
+    return f"models[{index}].id=<missing>"
+
+
+def _validate_score(
+    model_dict: dict, field: str, prefix: str, errors: list[str], required: bool
+) -> float | None:
+    """Validate one score field. Returns the float on success, None on failure
+    (or when an optional field is absent)."""
+    if field not in model_dict:
+        if required:
+            errors.append(f'{prefix}: missing required field "{field}"')
+        return None
+    v = model_dict[field]
+    if v is None:
+        errors.append(
+            f'{prefix}: field "{field}" is null; omit the field instead of using null'
+        )
+        return None
+    # bool is a subclass of int; reject it explicitly.
+    if isinstance(v, bool) or not isinstance(v, (int, float)):
+        errors.append(
+            f'{prefix}: field "{field}" must be numeric, got {type(v).__name__}'
+        )
+        return None
+    fv = float(v)
+    if not (0.0 <= fv <= 100.0):
+        errors.append(f'{prefix}: field "{field}" must be in [0, 100], got {fv}')
+        return None
+    return fv
+
+
+def _validate_optional_bool(
+    model_dict: dict, field_name: str, prefix: str, errors: list[str]
+) -> bool:
+    """Validate an optional bool field. Returns the value if valid, False if absent."""
+    if field_name not in model_dict:
+        return False
+    v = model_dict[field_name]
+    if not isinstance(v, bool):
+        errors.append(
+            f'{prefix}: field "{field_name}" must be a boolean, got {type(v).__name__}'
+        )
+        return False
+    return v
+
+
+def _fail(errors: list[str]) -> None:
+    msg = "[VANTAGE-Bench leaderboard] Schema validation failed:\n" + "\n".join(
+        f"  - {e}" for e in errors
+    )
+    raise ValueError(msg)
+
+
+# -- Public loader ---------------------------------------------------------
+
+
+def load_results_json(path: str | Path) -> LeaderboardData:
+    """Read, validate, and build the in-memory leaderboard dataset.
+
+    Hard-fails with ValueError listing every problem if validation does not pass.
+    """
+    p = Path(path)
+    try:
+        raw_text = p.read_text(encoding="utf-8")
+    except OSError as e:
+        raise ValueError(
+            f"[VANTAGE-Bench leaderboard] Could not read {p}: {e}"
+        ) from e
+
+    try:
+        data = json.loads(raw_text)
+    except json.JSONDecodeError as e:
+        raise ValueError(
+            f"[VANTAGE-Bench leaderboard] JSON parse error in {p}: {e}"
+        ) from e
+
+    if not isinstance(data, dict):
+        raise ValueError(
+            "[VANTAGE-Bench leaderboard] Schema validation failed:\n"
+            "  - top-level JSON must be an object"
+        )
+
+    errors: list[str] = []
+
+    # -- Top-level fields ---------------------------------------------------
+
+    sv = data.get("schema_version")
+    if sv != SCHEMA_VERSION:
+        errors.append(
+            f'schema_version must equal "{SCHEMA_VERSION}", got {sv!r}'
+        )
+
+    bv = data.get("benchmark_version")
+    if not isinstance(bv, str) or not bv:
+        errors.append("benchmark_version must be a non-empty string")
+
+    updated = data.get("updated")
+    if not isinstance(updated, str) or not _DATE_RE.match(updated):
+        errors.append(f"updated must be a YYYY-MM-DD string, got {updated!r}")
+
+    models_raw = data.get("models")
+    if not isinstance(models_raw, list) or not models_raw:
+        errors.append("models must be a non-empty list")
+        _fail(errors)
+        return  # unreachable; satisfies type checkers
+
+    # -- Per-model validation ----------------------------------------------
+
+    seen_ids: set[str] = set()
+
+    for i, m in enumerate(models_raw):
+        if not isinstance(m, dict):
+            errors.append(
+                f"models[{i}] must be an object, got {type(m).__name__}"
+            )
+            continue
+
+        prefix = _id_label(m, i)
+
+        # Required string fields
+        for fld in REQUIRED_MODEL_FIELDS:
+            if fld not in m:
+                errors.append(f'{prefix}: missing required field "{fld}"')
+            elif not isinstance(m[fld], str) or not m[fld]:
+                errors.append(
+                    f'{prefix}: field "{fld}" must be a non-empty string, got {m[fld]!r}'
+                )
+
+        # type must be in allowed set
+        if isinstance(m.get("type"), str) and m["type"] not in ALLOWED_TYPES:
+            errors.append(
+                f'{prefix}: field "type" must be one of {ALLOWED_TYPES}, got {m["type"]!r}'
+            )
+
+        # result_type must be in allowed set
+        rt = m.get("result_type")
+        if isinstance(rt, str) and rt not in ALLOWED_RESULT_TYPES:
+            errors.append(
+                f'{prefix}: field "result_type" must be one of {ALLOWED_RESULT_TYPES}, got {rt!r}'
+            )
+
+        # Optional bool fields
+        _validate_optional_bool(m, "verified", prefix, errors)
+        _validate_optional_bool(m, "ensemble", prefix, errors)
+        _validate_optional_bool(m, "is_new", prefix, errors)
+
+        # Uniqueness on id
+        mid = m.get("id")
+        if isinstance(mid, str) and mid:
+            if mid in seen_ids:
+                errors.append(f"{prefix}: duplicate id")
+            seen_ids.add(mid)
+
+        # url (optional legacy field — model name link)
+        if "url" in m:
+            url = m["url"]
+            if url is None:
+                errors.append(
+                    f'{prefix}: field "url" is null; omit the field instead of using null'
+                )
+            elif not isinstance(url, str) or not url:
+                errors.append(
+                    f'{prefix}: field "url" must be a non-empty string when present, got {url!r}'
+                )
+
+        # model_url (optional — HF model card or project page link)
+        if "model_url" in m:
+            mu = m["model_url"]
+            if not isinstance(mu, str):
+                errors.append(
+                    f'{prefix}: field "model_url" must be a string, got {type(mu).__name__}'
+                )
+
+        # date_evaluated (optional — YYYY-MM-DD or empty string)
+        if "date_evaluated" in m:
+            de = m["date_evaluated"]
+            if not isinstance(de, str):
+                errors.append(
+                    f'{prefix}: field "date_evaluated" must be a string, got {type(de).__name__}'
+                )
+            elif de and not _DATE_RE.match(de):
+                errors.append(
+                    f'{prefix}: field "date_evaluated" must be YYYY-MM-DD or empty, got {de!r}'
+                )
+
+        # Pillar and task scores are all optional: a model scored on only
+        # some tasks is listed with "—" for the rest. It needs at least one
+        # score, and may carry "overall" only when all four pillars are present.
+        for fld in REQUIRED_SCORE_FIELDS + OPTIONAL_TASK_FIELDS:
+            _validate_score(m, fld, prefix, errors, required=False)
+        if not any(f in m for f in REQUIRED_SCORE_FIELDS + OPTIONAL_TASK_FIELDS):
+            errors.append(f"{prefix}: model has no scores")
+        if "overall" in m:
+            missing = [f for f in REQUIRED_SCORE_FIELDS if f not in m]
+            if missing:
+                errors.append(
+                    f'{prefix}: "overall" requires all pillar scores, missing {missing}'
+                )
+
+    if errors:
+        _fail(errors)
+
+    # -- Build records (guaranteed valid past this point) ------------------
+
+    records: list[ModelRecord] = []
+    for m in models_raw:
+        scores: dict[str, float] = {}
+        for fld in REQUIRED_SCORE_FIELDS + OPTIONAL_TASK_FIELDS:
+            if fld in m:
+                scores[fld] = float(m[fld])
+
+        params_str = m["params"]
+        pv = _parse_param_value(params_str)
+        pb = _param_bucket(pv)
+
+        records.append(
+            ModelRecord(
+                id=m["id"],
+                name=m["name"],
+                organization=m["organization"],
+                params=params_str,
+                type=m["type"],
+                result_type=m["result_type"],
+                scores=scores,
+                param_value=pv,
+                param_bucket=pb,
+                type_display=TYPE_DISPLAY[m["type"]],
+                url=m.get("url"),
+                verified=bool(m.get("verified", False)),
+                ensemble=bool(m.get("ensemble", False)),
+                is_new=bool(m.get("is_new", False)),
+                date_evaluated=m.get("date_evaluated", ""),
+                model_url=m.get("model_url", ""),
+            )
+        )
+
+    return LeaderboardData(
+        schema_version=sv,
+        benchmark_version=bv,
+        updated=updated,
+        models=records,
+        model_by_id={r.id: r for r in records},
+    )

util/ranking.py

@@ -1,116 +1,116 @@
-"""Global rank computation for the leaderboard tabs.
-
-Ranks are computed once at startup over the full model list, before any
-filtering or partitioning into Open-Weight / Proprietary tables.
-They are properties of the model on this benchmark version and do not
-change when users apply filters or re-sort interactively.
-
-Tie handling: standard competition ranking ("1224").
-    scores 70, 65, 65, 60 → ranks 1, 2, 2, 4
-
-Module-level export
--------------------
-GLOBAL_RANKS : dict[str, dict[str, int]]
-    Keyed by the short pillar names from config.PILLARS:
-    'overall', 'spatial', 'st', 'temporal', 'semantic'.
-    Populated the first time compute_global_ranks() is called at startup.
-    Import directly::
-
-        from util.ranking import GLOBAL_RANKS
-"""
-
-from __future__ import annotations
-
-from typing import Iterable
-
-from .config import HEADLINE_FIELD_BY_TAB
-from .data import ModelRecord
-
-# Map short pillar key (config.PILLARS) → score field in ModelRecord.scores.
-# 'st' is the short form of 'spatio_temporal' used by the new UI layer.
-_PILLAR_SCORE_FIELD: dict[str, str] = {
-    "overall":  "overall",
-    "spatial":  "spatial",
-    "st":       "spatio_temporal",
-    "temporal": "temporal",
-    "semantic": "semantic",
-}
-
-# Module-level export. Starts empty; populated by compute_global_ranks()
-# on the first (and only) call at app startup.
-GLOBAL_RANKS: dict[str, dict[str, int]] = {}
-
-
-# -- Core ranking primitive ------------------------------------------------
-
-
-def _rank_models(models: list[ModelRecord], score_field: str) -> dict[str, int]:
-    """Return ``{model_id: rank}`` using 1-2-2-4 competition ranking.
-
-    Sort key: descending score, ascending name as deterministic tiebreaker.
-    The score field must be present in every model's scores dict.
-    """
-    sorted_models = sorted(
-        models,
-        key=lambda m: (-_headline(m, score_field), m.name),
-    )
-    ranks: dict[str, int] = {}
-    prev_score: float | None = None
-    prev_rank: int = 0
-    for position, m in enumerate(sorted_models, start=1):
-        s = m.scores[score_field]
-        if prev_score is not None and s == prev_score:
-            ranks[m.id] = prev_rank
-        else:
-            ranks[m.id] = position
-            prev_rank = position
-            prev_score = s
-    return ranks
-
-
-# -- Public API ------------------------------------------------------------
-
-
-def compute_global_ranks(
-    models: list[ModelRecord], tabs: Iterable[str]
-) -> dict[str, dict[str, int]]:
-    """Return ``{tab: {model_id: rank}}`` with global standard-competition ranks.
-
-    Sort key per tab: descending by the tab's headline score, ascending by
-    model name as deterministic tiebreaker. Required score fields are
-    guaranteed present by upstream validation.
-
-    Side-effect: populates the module-level ``GLOBAL_RANKS`` dict using the
-    short pillar keys from ``config.PILLARS`` ('overall', 'spatial', 'st',
-    'temporal', 'semantic'). This happens once at app startup so any module
-    can ``from util.ranking import GLOBAL_RANKS`` after startup.
-    """
-    # Build return value keyed by caller-supplied tab names (backward-compat).
-    result: dict[str, dict[str, int]] = {}
-    for tab in tabs:
-        field = HEADLINE_FIELD_BY_TAB[tab]
-        result[tab] = _rank_models(models, field)
-
-    # Populate GLOBAL_RANKS with short pillar keys for the new UI layer.
-    for pillar_key, score_field in _PILLAR_SCORE_FIELD.items():
-        GLOBAL_RANKS[pillar_key] = _rank_models(models, score_field)
-
-    return result
-
-
-def sort_model_ids_by_rank(
-    model_ids: Iterable[str], rank_map: dict[str, int]
-) -> list[str]:
-    """Return ``model_ids`` sorted by ascending rank.
-
-    Ids missing from ``rank_map`` are placed last in their input order.
-    """
-    sentinel = float("inf")
-    return sorted(model_ids, key=lambda mid: rank_map.get(mid, sentinel))
-
-
-def _headline(m: ModelRecord, field: str) -> float:
-    # Required scores guaranteed by validation; defensive default keeps
-    # this function total in case it is called pre-validation.
-    v = m.scores.get(field)
-    return v if v is not None else float("-inf")
+"""Global rank computation for the leaderboard tabs.
+
+Ranks are computed once at startup over the full model list, before any
+filtering or partitioning into Open-Weight / Proprietary tables.
+They are properties of the model on this benchmark version and do not
+change when users apply filters or re-sort interactively.
+
+Tie handling: standard competition ranking ("1224").
+    scores 70, 65, 65, 60 → ranks 1, 2, 2, 4
+
+Module-level export
+-------------------
+GLOBAL_RANKS : dict[str, dict[str, int]]
+    Keyed by the short pillar names from config.PILLARS:
+    'overall', 'spatial', 'st', 'temporal', 'semantic'.
+    Populated the first time compute_global_ranks() is called at startup.
+    Import directly::
+
+        from util.ranking import GLOBAL_RANKS
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from .config import HEADLINE_FIELD_BY_TAB
+from .data import ModelRecord
+
+# Map short pillar key (config.PILLARS) → score field in ModelRecord.scores.
+# 'st' is the short form of 'spatio_temporal' used by the new UI layer.
+_PILLAR_SCORE_FIELD: dict[str, str] = {
+    "overall":  "overall",
+    "spatial":  "spatial",
+    "st":       "spatio_temporal",
+    "temporal": "temporal",
+    "semantic": "semantic",
+}
+
+# Module-level export. Starts empty; populated by compute_global_ranks()
+# on the first (and only) call at app startup.
+GLOBAL_RANKS: dict[str, dict[str, int]] = {}
+
+
+# -- Core ranking primitive ------------------------------------------------
+
+
+def _rank_models(models: list[ModelRecord], score_field: str) -> dict[str, int]:
+    """Return ``{model_id: rank}`` using 1-2-2-4 competition ranking.
+
+    Sort key: descending score, ascending name as deterministic tiebreaker.
+    Models without the score field (partial submissions) get no rank.
+    """
+    sorted_models = sorted(
+        (m for m in models if score_field in m.scores),
+        key=lambda m: (-_headline(m, score_field), m.name),
+    )
+    ranks: dict[str, int] = {}
+    prev_score: float | None = None
+    prev_rank: int = 0
+    for position, m in enumerate(sorted_models, start=1):
+        s = m.scores[score_field]
+        if prev_score is not None and s == prev_score:
+            ranks[m.id] = prev_rank
+        else:
+            ranks[m.id] = position
+            prev_rank = position
+            prev_score = s
+    return ranks
+
+
+# -- Public API ------------------------------------------------------------
+
+
+def compute_global_ranks(
+    models: list[ModelRecord], tabs: Iterable[str]
+) -> dict[str, dict[str, int]]:
+    """Return ``{tab: {model_id: rank}}`` with global standard-competition ranks.
+
+    Sort key per tab: descending by the tab's headline score, ascending by
+    model name as deterministic tiebreaker. Models missing a tab's score
+    are left out of that tab's ranking.
+
+    Side-effect: populates the module-level ``GLOBAL_RANKS`` dict using the
+    short pillar keys from ``config.PILLARS`` ('overall', 'spatial', 'st',
+    'temporal', 'semantic'). This happens once at app startup so any module
+    can ``from util.ranking import GLOBAL_RANKS`` after startup.
+    """
+    # Build return value keyed by caller-supplied tab names (backward-compat).
+    result: dict[str, dict[str, int]] = {}
+    for tab in tabs:
+        field = HEADLINE_FIELD_BY_TAB[tab]
+        result[tab] = _rank_models(models, field)
+
+    # Populate GLOBAL_RANKS with short pillar keys for the new UI layer.
+    for pillar_key, score_field in _PILLAR_SCORE_FIELD.items():
+        GLOBAL_RANKS[pillar_key] = _rank_models(models, score_field)
+
+    return result
+
+
+def sort_model_ids_by_rank(
+    model_ids: Iterable[str], rank_map: dict[str, int]
+) -> list[str]:
+    """Return ``model_ids`` sorted by ascending rank.
+
+    Ids missing from ``rank_map`` are placed last in their input order.
+    """
+    sentinel = float("inf")
+    return sorted(model_ids, key=lambda mid: rank_map.get(mid, sentinel))
+
+
+def _headline(m: ModelRecord, field: str) -> float:
+    # Required scores guaranteed by validation; defensive default keeps
+    # this function total in case it is called pre-validation.
+    v = m.scores.get(field)
+    return v if v is not None else float("-inf")