Upload 8 files

runs/corpus1_2026-08-29/avg_latency.csv

@@ -0,0 +1,21 @@
+model_name,computer_science,philosophy,social_science,language,science,technology,arts,literature,history,agentic,Average (All Topics)
+Claude-sonnet-5,104.1285,40.0797,44.0383,52.2072,84.0212,45.2519,29.8389,30.7315,21.5157,9.1996,45.0655
+Deepseek-v4-flash-0731,124.6763,51.8759,49.0708,64.5170,82.1244,58.6729,52.9496,29.7742,26.0048,12.0519,53.4702
+GLM-5.2,115.3036,33.1635,34.7832,90.2493,92.8141,51.1301,57.4331,21.5099,18.6140,11.3857,52.3773
+GLM-5.3,78.6642,25.3735,55.6592,36.6172,51.4355,22.5974,43.4911,44.1527,22.7423,10.1134,38.2621
+Gemini-3.7-flash,40.6056,16.8653,20.7072,21.1582,27.4686,27.4393,27.7861,16.5619,15.8902,6.0907,21.5635
+Gemma-4-31b-it,29.7570,8.7522,17.0807,13.1580,27.0728,18.4476,17.1738,14.3495,11.5384,3.1334,15.8917
+Gpt-5.6-luna,110.4560,48.7502,73.7323,55.3611,55.2018,87.7245,62.8368,52.0553,30.9942,16.3886,56.5227
+Gpt-5.6-sol,146.2571,36.8419,75.4219,79.5431,75.5003,89.3168,75.4008,55.6614,39.1577,10.1769,66.3198
+Gpt-oss-120b,47.0243,29.0562,42.4483,31.5265,40.1419,42.9978,38.7071,29.9871,21.0147,9.7212,32.2931
+Grok-4.3,36.2627,12.2636,14.9860,21.1912,38.8928,19.9250,14.7165,14.1027,10.6336,9.2493,19.3693
+Grok-4.6,105.9274,30.9835,45.8555,49.8322,65.8972,46.9488,60.1150,39.7996,34.7181,12.9576,48.4345
+Hy3,46.5265,24.1220,25.1871,27.9365,41.4173,32.9733,27.5396,15.5697,16.9130,10.6467,26.5821
+Kimi-k2.6,199.9390,75.5453,78.2259,126.0239,78.1248,144.2096,101.7710,64.0730,57.0748,46.1749,93.0601
+Minimax-m3,93.0344,50.9386,42.1184,40.5628,44.1448,48.8331,70.6174,33.0846,32.7577,10.9857,44.2021
+Muse-glimmer-30b,23.9669,15.2687,15.8275,19.1767,28.5719,19.9078,15.6491,11.5667,11.5080,8.7542,17.0245
+Muse-spark-1.2,54.6906,30.8114,27.2576,29.8689,50.0040,40.6275,23.5305,37.7279,29.7687,17.4049,33.3163
+Nemotron-3-ultra-550b-a55b,85.5339,32.1617,32.2411,54.2057,39.7960,58.8242,44.1815,18.5844,17.2569,11.5445,38.0708
+Nemotron-3.5-lightning,109.4723,17.1492,33.1787,57.9600,88.2577,61.4016,36.6644,19.3933,22.6521,27.6037,48.2569
+Qwen3.5-9b,131.5601,138.8030,131.9078,202.5656,183.4021,176.4795,149.7241,137.2679,122.2912,92.3986,144.2203
+Qwen3.8-max,87.5096,50.2276,51.3803,74.8060,63.7113,65.6762,85.4906,39.1906,30.4111,14.1274,54.8266

runs/corpus1_2026-08-29/correlations.json

@@ -0,0 +1,6 @@
+{
+  "correlations": {
+    "Artificial Analysis Intelligence Index": 88.13
+  },
+  "description": "Correlation percentage between AutoBench scores and the Artificial Analysis Intelligence Index"
+}

runs/corpus1_2026-08-29/cost_data.csv

@@ -0,0 +1,21 @@
+model_name,computer_science,philosophy,social_science,language,science,technology,arts,literature,history,agentic,Average (All Topics)
+Claude-sonnet-5,0.100033000,0.030445429,0.035476519,0.044583067,0.088654296,0.045905368,0.022036000,0.021949000,0.014784647,0.009004067,0.040498091
+Deepseek-v4-flash-0731,0.001187238,0.000408442,0.000436091,0.000715858,0.000779890,0.000550249,0.000564028,0.000179154,0.000256069,0.000151916,0.000517397
+GLM-5.2,0.011182684,0.002783043,0.002891602,0.008541058,0.009912209,0.005607376,0.005924516,0.001835299,0.001520731,0.001098392,0.005111099
+GLM-5.3,0.011749750,0.004468121,0.007140109,0.008432784,0.006149523,0.005576853,0.005039362,0.003705922,0.003500347,0.001845237,0.005702562
+Gemini-3.7-flash,0.024673364,0.005382214,0.007471861,0.012524964,0.018451800,0.014588546,0.010639197,0.006708422,0.005221301,0.002696766,0.010703945
+Gemma-4-31b-it,0.000397057,0.000181809,0.000245579,0.000224744,0.000333946,0.000300895,0.000211487,0.000154348,0.000152829,0.000122933,0.000230341
+Gpt-5.6-luna,0.008954479,0.003653869,0.004813916,0.004699617,0.004704679,0.006353109,0.005400223,0.003235693,0.002109371,0.001196322,0.004334039
+Gpt-5.6-sol,0.227527357,0.049823346,0.098272385,0.080121274,0.094974323,0.091065658,0.074781500,0.038270688,0.037361015,0.019223828,0.078654744
+Gpt-oss-120b,0.000341553,0.000216294,0.000259617,0.000237908,0.000272558,0.000255874,0.000230396,0.000170703,0.000175405,0.000075282,0.000219573
+Grok-4.3,0.009463395,0.002990929,0.003897046,0.005365011,0.009729115,0.005680482,0.003603604,0.003416572,0.002590157,0.003178928,0.005036845
+Grok-4.6,0.037498091,0.009382429,0.014259630,0.016466000,0.023565778,0.018089579,0.018757923,0.011689000,0.010698059,0.006132312,0.016396672
+Hy3,0.001768374,0.000801159,0.000819156,0.000965307,0.001579511,0.001262618,0.000862956,0.000489122,0.000536228,0.000434849,0.000942565
+Kimi-k2.6,0.032332930,0.011152740,0.012407466,0.019959517,0.013965612,0.024363118,0.014585366,0.008995756,0.007640027,0.006910723,0.014548843
+Minimax-m3,0.010059726,0.003419193,0.003737113,0.004351812,0.006881629,0.003893425,0.006522411,0.002832289,0.002490695,0.001173037,0.004374423
+Muse-glimmer-30b,0.001588969,0.000930900,0.001001579,0.001256039,0.001827673,0.001338138,0.001015123,0.000708606,0.000688004,0.000569177,0.001092813
+Muse-spark-1.2,0.016187682,0.008409911,0.009093722,0.011075847,0.023779306,0.012538903,0.009902279,0.008169594,0.007378971,0.006906320,0.011496686
+Nemotron-3-ultra-550b-a55b,0.017285455,0.004811871,0.005232556,0.010798283,0.007561296,0.011142726,0.007515254,0.002368238,0.002114471,0.002146312,0.006871648
+Nemotron-3.5-lightning,0.003253451,0.000481606,0.000982873,0.002070814,0.002807913,0.002164553,0.001024351,0.000587450,0.000491460,0.000858536,0.001497770
+Qwen3.5-9b,0.001295631,0.000901008,0.000825323,0.001437293,0.001330398,0.001258903,0.001100667,0.000877927,0.000822777,0.000629244,0.001026152
+Qwen3.8-max,0.027363619,0.012933714,0.013929926,0.022899742,0.020605538,0.021456842,0.024741231,0.010080625,0.007509882,0.004854500,0.016261683

runs/corpus1_2026-08-29/domain_ranks.csv

@@ -0,0 +1,21 @@
+model_name,computer_science,philosophy,social_science,language,science,technology,arts,literature,history,agentic,Average (All Topics)
+Claude-sonnet-5,4.5979,4.8641,4.8350,4.7430,4.7931,4.7686,4.5734,4.7631,4.6530,4.7941,4.7329
+Deepseek-v4-flash-0731,4.3247,4.7181,4.7730,4.6946,4.7729,4.6214,4.3121,4.4405,4.6925,4.4561,4.5873
+GLM-5.2,4.4793,4.6928,4.7445,4.6309,4.6734,4.8167,4.5067,4.7027,4.7164,4.7656,4.6739
+GLM-5.3,4.3649,4.8080,4.7566,4.6296,4.6215,4.6536,4.5350,4.6166,4.7158,4.7716,4.6506
+Gemini-3.7-flash,4.4496,4.8064,4.7818,4.6520,4.7715,4.7334,4.4493,4.7244,4.7498,4.6616,4.6754
+Gemma-4-31b-it,4.0035,4.6314,4.5189,4.0819,4.2238,4.5565,4.3160,4.5486,4.6468,4.1596,4.3455
+Gpt-5.6-luna,4.7919,4.8422,4.8577,4.7364,4.8252,4.7073,4.7099,4.7302,4.8281,4.4283,4.7386
+Gpt-5.6-sol,4.7511,4.8586,4.8423,4.7896,4.8069,4.7113,4.6203,4.7368,4.8344,4.8353,4.7832
+Gpt-oss-120b,3.9671,4.3366,4.3181,4.0662,3.9606,4.5045,3.6427,4.0499,4.0639,4.4490,4.1232
+Grok-4.3,4.2710,4.7021,4.7461,4.4431,4.4103,4.4242,4.0756,4.4834,4.6374,4.3090,4.4433
+Grok-4.6,4.6202,4.7780,4.7581,4.6781,4.6693,4.5362,4.3955,4.6590,4.7343,4.2365,4.5950
+Hy3,4.4544,4.6640,4.7638,4.6508,4.5344,4.6413,4.2501,4.6054,4.7083,4.7593,4.6088
+Kimi-k2.6,4.4461,4.8073,4.7954,4.6576,4.8396,4.7509,4.5495,4.6651,4.6960,4.6893,4.6908
+Minimax-m3,4.5243,4.7499,4.7825,4.5643,4.6955,4.5767,4.4278,4.7261,4.6961,4.7623,4.6518
+Muse-glimmer-30b,4.0622,4.7332,4.6255,4.1769,4.1846,4.6147,3.8686,4.3277,4.4530,4.6354,4.3530
+Muse-spark-1.2,4.4337,4.7659,4.6920,4.6429,4.6468,4.4158,4.3553,4.6847,4.7075,4.3114,4.5603
+Nemotron-3-ultra-550b-a55b,4.3387,4.7399,4.7312,4.4890,4.7421,4.4829,4.1111,4.6197,4.6694,4.1746,4.4943
+Nemotron-3.5-lightning,4.0732,4.6024,4.5882,4.2177,4.2980,4.2487,4.0998,4.4620,4.5074,4.5129,4.3583
+Qwen3.5-9b,3.4353,4.6154,4.4137,4.1456,4.5118,4.2612,3.5378,4.0834,4.0477,4.4726,4.1545
+Qwen3.8-max,4.6542,4.8227,4.8020,4.6729,4.8592,4.5951,4.5886,4.7601,4.7871,4.8160,4.7388

runs/corpus1_2026-08-29/metadata.json

@@ -0,0 +1,10 @@
+{
+  "run_id": "run_2026-08-29",
+  "title": "AutoBench August 2026",
+  "date": "2026-08-29",
+  "description": "Twenty models scored on 252 questions across ten subject domains. Score, cost, and latency are the per-answer committee rank, billed cost, and generation time.",
+  "blog_url": "",
+  "model_count": 20,
+  "is_latest": false,
+  "benchmark_comparison_note": "Reference scores are the Artificial Analysis Intelligence Index."
+}

runs/corpus1_2026-08-29/models.csv

@@ -0,0 +1,21 @@
+company_id,model_name,model_display_name,model_logo_url,api_type,context,parameters,model_version,release_date,thinking_mode,description
+2,Claude-sonnet-5,Claude Sonnet 5,/logos/models/claude.svg,OpenRouter,1000000,,,30/06/2026,1,"Sonnet 5 is Anthropic's most capable Sonnet-class model, with frontier performance across coding, agents, and professional work. It supports adaptive thinking with selectable reasoning effort levels (low, medium, high, max,..."
+30,Deepseek-v4-flash-0731,DeepSeek V4 Flash 0731,/logos/models/deepseek.svg,OpenRouter,1048576,,,31/07/2026,1,"DeepSeek V4 Flash 0731 is a sparse mixture-of-experts model from DeepSeek, with 13B active parameters out of 284B total. This re-post-trained revision is suited for coding, reasoning, and agent workflows...."
+22,GLM-5.2,GLM 5.2,/logos/models/glm.svg,OpenRouter,1048576,,,16/06/2026,1,"GLM 5.2 is a large-scale reasoning model from Z.ai. It supports text input and output with a 1M-token context window, and is suited for long-horizon agent workflows, project-level software engineering,..."
+22,GLM-5.3,GLM 5.3,/logos/models/glm.svg,OpenRouter,1048576,,,18/08/2026,1,"GLM-5.3 is a large-scale reasoning model from Z.ai, built for complex software engineering and long-horizon agent tasks. It supports text input and output with a 1M-token context window, and improves..."
+3,Gemini-3.7-flash,Gemini 3.7 Flash,/logos/models/gemini.svg,OpenRouter,1048576,,,13/08/2026,1,"Gemini 3.7 Flash is a multimodal model from Google for fast agentic workflows, coding, and complex multi-step reasoning. It is designed for tasks that require responsive performance and reliable multi-step..."
+3,Gemma-4-31b-it,Gemma 4 31B IT,/logos/models/gemini.svg,OpenRouter,262144,,,02/04/2026,0,"Gemma 4 31B Instruct is Google DeepMind's 30.7B dense multimodal model supporting text and image input with text output. Features a 256K token context window, configurable thinking/reasoning mode, native function..."
+1,Gpt-5.6-luna,GPT-5.6 Luna,/logos/models/openai.svg,OpenRouter,1050000,,,09/07/2026,1,"GPT-5.6 Luna is a fast, cost-efficient model in OpenAI's GPT-5.6 series. It is suited for high-volume, latency-sensitive tasks such as chat, classification, and lightweight agentic workflows, providing capable reasoning for..."
+1,Gpt-5.6-sol,GPT-5.6 Sol,/logos/models/openai.svg,OpenRouter,1050000,,,09/07/2026,1,"GPT-5.6 Sol is the flagship model in OpenAI's GPT-5.6 series. It is suited for complex reasoning, coding, and agentic workflows, and is particularly strong at command-line and multi-step coding tasks..."
+1,Gpt-oss-120b,Gpt oss 120b,/logos/models/openai.svg,OpenRouter,131000,,,05/08/2025,1,"gpt-oss-120b is an open-weight, 117B-parameter Mixture-of-Experts (MoE) language model from OpenAI designed for high-reasoning, agentic, and general-purpose production use cases. It activates 5.1B parameters per forward pass and is optimized..."
+23,Grok-4.3,Grok 4.3,/logos/models/grok.svg,OpenRouter,1000000,,,30/04/2026,1,"Grok 4.3 is a reasoning model from SpaceXAI. It accepts text and image inputs with text output, and is suited for agentic workflows, instruction-following tasks, and applications requiring high factual..."
+23,Grok-4.6,Grok 4.6,/logos/models/grok.svg,OpenRouter,500000,,,12/08/2026,1,"Grok 4.6 is SpaceXAI's smartest model with frontier performance on coding, knowledge work, and STEM."
+,Hy3,Hy3,/logos/companies/tencent.svg,OpenRouter,262144,,,06/07/2026,1,"Hy3 is a 295B-parameter Mixture-of-Experts model from Tencent (21B active, 192 experts with top-8 routing) built for reasoning, agentic workflows, and real-world production use. It supports a configurable reasoning effort:..."
+29,Kimi-k2.6,Kimi K2.6,/logos/models/kimi.svg,OpenRouter,262144,,,20/04/2026,1,"Kimi K2.6 is Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration. It handles complex end-to-end coding tasks across Python, Rust, and Go, and..."
+31,Minimax-m3,MiniMax M3,/logos/models/minimax.svg,OpenRouter,524288,,,31/05/2026,1,"MiniMax-M3 is a multimodal foundation model from MiniMax. It supports text, image, and video inputs with text output, a 1M-token context window, and is suited for long-horizon agentic work, coding,..."
+5,Muse-glimmer-30b,Muse Glimmer 30B,/logos/models/llama.svg,OpenRouter,131072,,,09/08/2026,1,"Muse Glimmer 30B is a dense, open-weight multimodal model from Meta Superintelligence Labs, distilled from Muse Spark and optimized for autonomous agents on consumer hardware. It is suited for long-horizon..."
+5,Muse-spark-1.2,Muse Spark 1.2,/logos/models/llama.svg,OpenRouter,1048576,,,05/08/2026,1,"Muse Spark 1.2 is a reasoning model from Meta, designed for complex agentic tasks. It accepts text, images, video, audio, and PDF documents, returns text, and offers a 1M-token context..."
+27,Nemotron-3-ultra-550b-a55b,Nemotron 3 Ultra,/logos/companies/nvidia.svg,OpenRouter,512288,,,04/06/2026,1,"NVIDIA Nemotron 3 Ultra is an open frontier-reasoning and orchestration model from NVIDIA, with 55B active parameters out of 550B total (MoE). Built on a hybrid Transformer-Mamba mixture-of-experts architecture, it..."
+27,Nemotron-3.5-lightning,Nemotron 3.5 Lightning,/logos/companies/nvidia.svg,OpenRouter,262144,,,11/08/2026,1,"NVIDIA Nemotron 3.5 Lightning is an open mixture-of-experts model from NVIDIA, with 3B active parameters out of 30B total. It is suited for high-throughput agentic workloads and specialized tasks that..."
+18,Qwen3.5-9b,Qwen3.5-9B,/logos/models/qwen.svg,OpenRouter,262144,,,10/03/2026,1,"Qwen3.5-9B is a multimodal foundation model from the Qwen3.5 family, designed to deliver strong reasoning, coding, and visual understanding in an efficient 9B-parameter architecture. It uses a unified vision-language design..."
+18,Qwen3.8-max,Qwen3.8 Max,/logos/models/qwen.svg,OpenRouter,1000000,,,03/08/2026,1,"Qwen3.8 Max is the flagship model in Alibaba's Qwen3.8 series, the general-availability successor to the Qwen3.8 Max Preview. It is a multimodal reasoning model intended for complex reasoning, visual understanding,..."

runs/corpus1_2026-08-29/p99_latency.csv

@@ -0,0 +1,21 @@
+model_name,computer_science,philosophy,social_science,language,science,technology,arts,literature,history,agentic,Average (All Topics)
+Claude-sonnet-5,381.2921,104.8469,150.7276,360.2342,382.0566,140.6637,95.9912,92.0707,60.9264,21.8604,381.4904
+Deepseek-v4-flash-0731,485.4297,216.7517,273.7694,328.3101,339.4819,268.1163,302.6529,103.0883,75.0265,54.4425,371.8184
+GLM-5.2,556.7430,87.8774,156.5257,464.4455,405.3727,168.3228,488.9067,73.2595,51.1162,71.5953,472.5025
+GLM-5.3,482.2769,65.5347,264.6983,186.9782,273.8038,40.7503,271.2004,226.0346,76.9326,26.5868,277.5305
+Gemini-3.7-flash,112.0019,44.4858,73.2451,103.8070,104.5300,106.8521,175.8386,46.7922,41.8933,19.7682,116.9328
+Gemma-4-31b-it,150.8002,19.3946,72.7363,38.9223,117.3473,43.9231,128.3358,71.1715,54.0312,11.0467,117.3068
+Gpt-5.6-luna,350.9477,160.1915,305.5635,244.0369,232.0352,259.3220,252.5188,212.7474,142.4966,70.1005,298.1938
+Gpt-5.6-sol,427.5868,158.6420,446.8503,369.6037,364.3317,270.7171,277.7541,340.1200,160.8888,58.0529,400.9731
+Gpt-oss-120b,119.3192,65.6491,121.1553,82.8434,148.7787,219.6003,197.8684,110.2657,68.6790,22.4842,145.7659
+Grok-4.3,139.0810,21.7913,32.9661,77.1721,190.1671,63.6058,34.0228,32.0545,21.1913,18.0779,142.9761
+Grok-4.6,516.0242,63.2208,173.8266,167.8891,394.9126,124.4178,216.7900,103.3527,86.5324,44.6373,237.7973
+Hy3,90.6812,72.4423,56.2769,85.1795,127.3992,102.8429,70.5779,33.5604,42.2783,25.8370,112.4103
+Kimi-k2.6,514.9422,185.8383,210.4866,559.3454,268.7651,378.1531,361.8981,167.3240,153.1104,171.7831,487.5059
+Minimax-m3,388.7669,154.5003,125.1672,231.2252,333.0503,183.6310,422.1926,110.4447,140.8245,32.4138,376.6069
+Muse-glimmer-30b,79.3627,47.8193,27.4107,51.0628,84.4844,59.7630,37.3880,24.9099,28.9925,15.2067,82.3923
+Muse-spark-1.2,328.2436,102.7985,124.0775,117.2623,256.5623,171.9972,58.2257,103.1936,114.0986,68.6671,193.8406
+Nemotron-3-ultra-550b-a55b,322.5315,148.4146,130.9003,354.5328,255.6433,453.0947,273.6677,60.6336,45.3766,97.2601,322.0949
+Nemotron-3.5-lightning,335.5982,47.9946,118.8067,279.2514,386.9197,304.9544,224.5446,37.2677,124.8642,136.1011,343.8833
+Qwen3.5-9b,285.1780,262.1037,332.5706,509.5801,555.6462,469.0424,355.0504,328.4950,362.3371,317.2151,488.2476
+Qwen3.8-max,256.7132,147.1793,158.0862,404.5026,300.0727,269.8637,452.3149,156.3527,123.5702,26.6810,408.5471

runs/corpus1_2026-08-29/summary_data.csv

@@ -0,0 +1,21 @@
+Model,Iterations,AutoBench,AAI Index,Costs (USD),Avg Answer Duration (sec),P99 Answer Duration (sec),Fail Rate %
+Claude-sonnet-5,243,4.732924,55,0.040498091,45.0655,381.4904,3.57%
+Deepseek-v4-flash-0731,245,4.587252,52,0.000517397,53.4702,371.8184,2.78%
+GLM-5.2,247,4.673865,53,0.005111099,52.3773,472.5025,1.98%
+GLM-5.3,243,4.650638,60,0.005702562,38.2621,277.5305,3.57%
+Gemini-3.7-flash,251,4.675363,56,0.010703945,21.5635,116.9328,0.40%
+Gemma-4-31b-it,252,4.345516,30,0.000230341,15.8917,117.3068,0.00%
+Gpt-5.6-luna,251,4.738622,52,0.004334039,56.5227,298.1938,0.40%
+Gpt-5.6-sol,248,4.783205,61,0.078654744,66.3198,400.9731,1.59%
+Gpt-oss-120b,252,4.123172,24,0.000219573,32.2931,145.7659,0.00%
+Grok-4.3,252,4.443302,38,0.005036845,19.3693,142.9761,0.00%
+Grok-4.6,247,4.595013,61,0.016396672,48.4345,237.7973,1.98%
+Hy3,252,4.608766,42,0.000942565,26.5821,112.4103,0.00%
+Kimi-k2.6,238,4.690759,45,0.014548843,93.0601,487.5059,5.56%
+Minimax-m3,244,4.651844,45,0.004374423,44.2021,376.6069,3.17%
+Muse-glimmer-30b,252,4.353040,35,0.001092813,17.0245,82.3923,0.00%
+Muse-spark-1.2,251,4.560261,57,0.011496686,33.3163,193.8406,0.40%
+Nemotron-3-ultra-550b-a55b,244,4.494319,38,0.006871648,38.0708,322.0949,3.17%
+Nemotron-3.5-lightning,252,4.358279,24,0.001497770,48.2569,343.8833,0.00%
+Qwen3.5-9b,223,4.154484,22,0.001026152,144.2203,488.2476,11.51%
+Qwen3.8-max,246,4.738829,58,0.016261683,54.8266,408.5471,2.38%