Merge pull request #33 from redhat-et/feat/add-qwen3-8

results.csv

@@ -1,74 +1,92 @@
-benchmark.name,benchmark.repo,benchmark.num_tasks,benchmark.url,harness.name,harness.skills,harness.is_oss,harness.url,model.name,model.repo,model.is_oss,model.num_params,model.precision,model.url,environment.name,environment.url,metrics.n_tasks,metrics.n_errors,metrics.score,metrics.n_input_tokens,metrics.n_cache_tokens,metrics.n_output_tokens,metrics.n_total_tokens,metrics.agent_time_seconds,metrics.total_time_seconds,metrics.cost_usd,metrics.mean_input_tokens_per_task,metrics.mean_cache_tokens_per_task,metrics.mean_output_tokens_per_task,metrics.mean_tokens_per_task,metrics.mean_cost_usd_per_task,metrics.mean_total_time_seconds_per_task,metrics.mean_agent_time_seconds_per_task
-ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,OpenCode,[],True,https://github.com/opencode-ai/opencode,GLM 5.3,zai-org/glm-5-3,False,1000,bf16,,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.377,377737743.0,369300480.0,2362160.0,749400383.0,45238.0,49636.0,8.28,9443443.0,9232512.0,59054.0,18735009.0,0.21,1240.0,1130.0
-ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Haiku 4.5,Haiku 4.5,False,1000,bf16,https://www.anthropic.com/claude/haiku,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.181,105433712.0,102933620.0,1004356.0,209371688.0,10285.0,14015.0,18.44,2635842.0,2573340.0,25108.0,5234292.0,0.46,350.0,257.0
-ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 5.5,Opus 5.5,False,1000,bf16,https://www.anthropic.com/claude/opus,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.354,31578057.0,29944414.0,318164.0,61840635.0,4413.0,8139.0,20.52,789451.0,748610.0,7954.0,1546015.0,0.51,203.0,110.0
-ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8 Flash Next,Inferact/Qwen3.8-Flash-Next-NVFP4,False,1000,bf16,,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.392,96024051.0,24637600.0,1398403.0,122060054.0,60360.0,63805.0,11.76,2400601.0,615940.0,34960.0,3051501.0,0.29,1595.0,1509.0
-ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 5,Sonnet 5,False,1000,bf16,https://www.anthropic.com/claude/sonnet,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.438,129942215.0,126175488.0,1064980.0,257182683.0,12915.0,16464.0,45.3,3248555.0,3154387.0,26624.0,6429567.0,1.13,411.0,322.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.6,Opus 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.633,438501505.0,419608736.0,2823496.0,860933737.0,126529.0,243310.0,434.54,1228295.0,1175374.0,7908.0,2411579.0,1.22,681.0,354.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.557,500304168.0,479035103.0,4937782.0,984277053.0,150652.0,269008.0,349.77,1401412.0,1341835.0,13831.0,2757078.0,0.98,753.0,421.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,31.0,0.317,1088634865.0,0.0,6200414.0,1094835279.0,398444.0,575536.0,15.05,3049397.0,0.0,17368.0,3066765.0,0.04,1612.0,1116.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.294,419671770.0,0.0,2277963.0,421949733.0,98909.0,169021.0,3.74,1175551.0,0.0,6380.0,1181932.0,0.01,473.0,277.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,20.0,0.23,419282595.0,0.0,1954817.0,421237412.0,100632.0,168697.0,3.8,1174461.0,0.0,5475.0,1179936.0,0.01,472.0,281.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,14.0,0.224,783670817.0,0.0,5510291.0,789181108.0,144352.0,355122.0,38.03,2195156.0,0.0,15434.0,2210591.0,0.11,994.0,404.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.308,839586046.0,0.0,5706139.0,845292185.0,94762.0,251102.0,37.44,2351781.0,0.0,15983.0,2367765.0,0.1,703.0,265.0
-RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,13.0,0.216,770574986.0,0.0,6519128.0,777094114.0,116348.0,360934.0,30.65,2158473.0,0.0,18260.0,2176734.0,0.09,1011.0,325.0
-RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,9.0,0.493,885869433.0,0.0,4763916.0,890633349.0,201374.0,401657.0,73.84,2481426.0,0.0,13344.0,2494771.0,0.21,1125.0,564.0
-RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,10.0,0.44,410001939.0,0.0,2851571.0,412853510.0,115135.0,315451.0,42.22,1148464.0,0.0,7987.0,1156452.0,0.12,883.0,322.0
-RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,1.0,0.468,560672462.0,0.0,3618673.0,564291135.0,114791.0,333385.0,42.09,1570511.0,0.0,10136.0,1580647.0,0.12,933.0,321.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,115.0,6.0,0.0,0.0,0.0,0.0,0.0,29895.0,36503.0,8.3,0.0,0.0,0.0,0.0,0.07,317.0,259.0
-Shellbench,ShellBench/public-tasks,109,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GLM-5.2-FP8,RedHatAI/GLM-5.2-FP8,True,743,fp8,https://huggingface.co/RedHatAI/GLM-5.2-FP8,harbor,https://github.com/harbor-framework/harbor,109.0,2.0,0.187,0.0,0.0,0.0,0.0,20060.0,28907.0,22.29,0.0,0.0,0.0,0.0,0.2,265.0,184.0
-Shellbench,ShellBench/public-tasks,109,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GLM-5.2,z-ai/glm-5.2,True,743,,https://openrouter.ai/z-ai/glm-5.2,harbor,https://github.com/harbor-framework/harbor,109.0,0.0,0.183,2176246.0,1912720.0,64113.0,4153079.0,16440.0,21509.0,0.0,19965.0,17547.0,588.0,38101.0,0.0,197.0,150.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.0,0.0,0.0,0.0,0.0,8170.0,13296.0,2.27,0.0,0.0,0.0,0.0,0.02,115.0,71.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,fp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,2.0,0.009,0.0,0.0,0.0,0.0,4846.0,9862.0,1.35,0.0,0.0,0.0,0.0,0.01,85.0,42.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,fp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.014,0.0,0.0,0.0,0.0,21575.0,26392.0,5.99,0.0,0.0,0.0,0.0,0.05,229.0,187.0
-Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.128,0.0,0.0,0.0,0.0,10812.0,16949.0,6.01,0.0,0.0,0.0,0.0,0.05,147.0,94.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.698,192346997.0,186506482.0,2179945.0,381033424.0,32745.0,39030.0,185.66155285,2003614.0,1942775.0,22707.0,3969098.0,1.93,406.0,341.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.781,187217712.0,187209844.0,1280944.0,375708500.0,30686.0,39352.0,151.4104807499999,1950184.0,1950102.0,13343.0,3913630.0,1.58,409.0,319.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.5,190672390.0,184409111.0,1593112.0,376674613.0,40527.0,49734.0,184.42824125000004,1986170.0,1920928.0,16594.0,3923693.0,1.92,518.0,422.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,16.0,0.417,122366824.0,0.0,1307027.0,123673851.0,138148.0,158812.0,54.82,1274654.0,0.0,13614.0,1288269.0,0.57,1654.0,1439.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,12.0,0.417,100068358.0,0.0,1261237.0,101329595.0,60747.0,77623.0,19.28,1042378.0,0.0,13137.0,1055516.0,0.2,808.0,632.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.469,78678324.0,0.0,1370430.0,80048754.0,51767.0,66845.0,16.43,819565.0,0.0,14275.0,833841.0,0.17,696.0,539.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.604,198924339.0,189578624.0,1560836.0,390063799.0,32914.0,39468.0,188.34296700000002,2072128.0,1974777.0,16258.0,4063164.0,1.96,411.0,342.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.375,347744043.0,0.0,1903023.0,349647066.0,59653.0,67586.0,2.25,3622333.0,0.0,19823.0,3642156.0,0.02,704.0,621.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.333,112041538.0,0.0,781173.0,112822711.0,25086.0,31180.0,0.95,1167099.0,0.0,8137.0,1175236.0,0.01,324.0,261.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,15.0,0.292,125247664.0,0.0,597435.0,125845099.0,75715.0,79741.0,2.86,1304663.0,0.0,6223.0,1310886.0,0.03,830.0,788.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.26,147452235.0,0.0,953241.0,148405476.0,33613.0,55241.0,5.84,1535960.0,0.0,9929.0,1545890.0,0.06,575.0,350.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.385,140362798.0,0.0,1230697.0,141593495.0,23543.0,49289.0,6.54,1462112.0,0.0,12819.0,1474932.0,0.07,513.0,245.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.333,104994429.0,0.0,1095179.0,106089608.0,15004.0,55176.0,4.17,1093691.0,0.0,11408.0,1105100.0,0.04,574.0,156.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.432,406026150.0,0.0,2839236.0,408865386.0,100355.0,115220.0,26.44,4229439.0,0.0,29575.0,4259014.0,0.28,1200.0,1045.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.323,686392098.0,0.0,2915876.0,689307974.0,86130.0,99705.0,19.45,7149917.0,0.0,30373.0,7180291.0,0.2,1038.0,897.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,14.0,0.375,1051387065.0,0.0,3651367.0,1055038432.0,113768.0,127326.0,25.69,10951948.0,0.0,38035.0,10989983.0,0.27,1326.0,1185.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,8.0,0.521,386929017.0,0.0,1847087.0,388776104.0,118693.0,137736.0,43.52,4030510.0,0.0,19240.0,4049751.0,0.45,1434.0,1236.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.573,344853232.0,0.0,1661197.0,346514429.0,118356.0,196235.0,43.4,3592221.0,0.0,17304.0,3609525.0,0.45,2044.0,1232.0
-SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.49,395189021.0,0.0,2206097.0,397395118.0,80305.0,91995.0,29.45,4116552.0,0.0,22980.0,4139532.0,0.31,958.0,836.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,6.0,0.458,367897697.0,0.0,1694885.0,369592582.0,39024.0,46758.0,9.64,3832267.0,0.0,17655.0,3849922.0,0.1,487.0,406.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.375,207164679.0,0.0,1598703.0,208763382.0,49450.0,57287.0,12.21,2157965.0,0.0,16653.0,2174618.0,0.13,596.0,515.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,5.0,0.406,0.0,0.0,0.0,0.0,38085.0,50779.0,9.4,0.0,0.0,0.0,0.0,0.1,528.0,396.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.479,742491363.0,0.0,2387609.0,744878972.0,54543.0,62422.0,13.47,7734285.0,0.0,24870.0,7759155.0,0.14,650.0,568.0
-SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,9.0,0.438,159198517.0,0.0,972133.0,160170650.0,33610.0,38272.0,9.34,1658317.0,0.0,10126.0,1668444.0,0.1,398.0,350.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.868,402225158.0,388781382.0,4588859.0,795595399.0,104467.0,159566.0,394.7823929,804450.0,777562.0,9177.0,1591190.0,0.79,319.0,208.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.834,347751942.0,347725246.0,3640204.0,699117392.0,114633.0,171540.0,319.53304049999974,695503.0,695450.0,7280.0,1398234.0,0.64,343.0,229.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,,,0.796,,,,,,,,,,,,,,
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,35.0,0.612,651306166.0,0.0,4591859.0,655898025.0,524857.0,669355.0,104.14,1302612.0,0.0,9183.0,1311796.0,0.21,1338.0,1049.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.606,525563275.0,0.0,4183675.0,529746950.0,166092.0,444527.0,52.73,1051126.0,0.0,8367.0,1059493.0,0.11,889.0,332.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.574,377479979.0,0.0,4177117.0,381657096.0,122274.0,385336.0,38.82,754959.0,0.0,8354.0,763314.0,0.08,770.0,244.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.798,415795756.0,386352384.0,3431224.0,805579364.0,92588.0,141637.0,443.32977200000016,831591.0,772704.0,6862.0,1611158.0,0.89,283.0,185.0
-SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,13.0,0.376,833651681.0,0.0,5656173.0,839307854.0,150981.0,200547.0,5.7,1667303.0,0.0,11312.0,1678615.0,0.01,401.0,301.0
-SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,33.0,0.326,202856940.0,0.0,1444371.0,204301311.0,35190.0,322482.0,3.32,405713.0,0.0,2888.0,408602.0,0.01,644.0,70.0
-SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,47.0,0.234,90107247.0,0.0,905031.0,91012278.0,146747.0,176174.0,13.86,180214.0,0.0,1810.0,182024.0,0.03,352.0,293.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,53.0,0.118,309857096.0,0.0,2416878.0,312273974.0,54338.0,555599.0,9.43,619714.0,0.0,4833.0,624547.0,0.02,1111.0,108.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.318,373713878.0,0.0,4051687.0,377765565.0,49103.0,543025.0,13.64,747427.0,0.0,8103.0,755531.0,0.03,1086.0,98.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.232,195246618.0,0.0,2726988.0,197973606.0,29236.0,458591.0,8.12,390493.0,0.0,5453.0,395947.0,0.02,917.0,58.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.472,882330271.0,0.0,6689285.0,889019556.0,174288.0,410828.0,45.91,1764660.0,0.0,13378.0,1778039.0,0.09,821.0,348.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.41,724984042.0,0.0,5941303.0,730925345.0,93088.0,518565.0,21.02,1449968.0,0.0,11882.0,1461850.0,0.04,1037.0,186.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.498,996839122.0,0.0,7196671.0,1004035793.0,144084.0,446845.0,32.53,1993678.0,0.0,14393.0,2008071.0,0.07,893.0,288.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.694,1005844088.0,0.0,6255800.0,1012099888.0,228858.0,411348.0,83.91,2011688.0,0.0,12511.0,2024199.0,0.17,822.0,457.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,8.0,0.642,395291929.0,0.0,3296668.0,398588597.0,92700.0,374913.0,33.99,790583.0,0.0,6593.0,797177.0,0.07,749.0,185.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,7.0,0.694,544070591.0,0.0,4965652.0,549036243.0,130591.0,408706.0,47.88,1088141.0,0.0,9931.0,1098072.0,0.1,817.0,261.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.632,1106618897.0,0.0,5733245.0,1112352142.0,122808.0,171897.0,34.11,2213237.0,0.0,11466.0,2224704.0,0.07,343.0,245.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.588,0.0,0.0,0.0,0.0,120399.0,200354.0,33.44,0.0,0.0,0.0,0.0,0.07,400.0,240.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.548,469806650.0,0.0,4937761.0,474744411.0,120473.0,185168.0,29.75,939613.0,0.0,9875.0,949488.0,0.06,370.0,240.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.65,791183735.0,0.0,6333798.0,797517533.0,154531.0,218988.0,38.16,1582367.0,0.0,12667.0,1595035.0,0.08,437.0,309.0
-SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.638,609589099.0,0.0,3964252.0,613553351.0,132273.0,178984.0,36.74,1219178.0,0.0,7928.0,1227106.0,0.07,357.0,264.0
-Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.303,47607780.0,0.0,1657188.0,49264968.0,46467.0,58507.0,11.47,534918.0,0.0,18620.0,553538.0,0.13,657.0,522.0
-Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.36,82108716.0,0.0,2056390.0,84165106.0,44991.0,64760.0,11.11,922569.0,0.0,23105.0,945675.0,0.12,727.0,505.0
+benchmark.name,benchmark.repo,benchmark.num_tasks,benchmark.url,harness.name,harness.skills,harness.is_oss,harness.url,model.name,model.repo,model.is_oss,model.num_params,model.precision,model.url,environment.name,environment.url,metrics.n_tasks,metrics.n_errors,metrics.score,metrics.n_input_tokens,metrics.n_cache_tokens,metrics.n_output_tokens,metrics.n_total_tokens,metrics.agent_time_seconds,metrics.total_time_seconds,metrics.cost_usd,metrics.mean_input_tokens_per_task,metrics.mean_cache_tokens_per_task,metrics.mean_output_tokens_per_task,metrics.mean_tokens_per_task,metrics.mean_cost_usd_per_task,metrics.mean_total_time_seconds_per_task,metrics.mean_agent_time_seconds_per_task,harness.version,metrics.cache_hit_rate
+ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,OpenCode,[],True,https://github.com/opencode-ai/opencode,GLM 5.3,zai-org/glm-5-3,False,1000,bf16,,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.377,377737743.0,369300480.0,2362160.0,749400383.0,45238.0,49636.0,8.28,9443443.0,9232512.0,59054.0,18735009.0,0.21,1240.0,1130.0,,
+ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Haiku 4.5,Haiku 4.5,False,1000,bf16,https://www.anthropic.com/claude/haiku,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.181,105433712.0,102933620.0,1004356.0,209371688.0,10285.0,14015.0,18.44,2635842.0,2573340.0,25108.0,5234292.0,0.46,350.0,257.0,,
+ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 5.5,Opus 5.5,False,1000,bf16,https://www.anthropic.com/claude/opus,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.354,31578057.0,29944414.0,318164.0,61840635.0,4413.0,8139.0,20.52,789451.0,748610.0,7954.0,1546015.0,0.51,203.0,110.0,,
+ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8 Flash Next,Inferact/Qwen3.8-Flash-Next-NVFP4,False,1000,bf16,,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.392,96024051.0,24637600.0,1398403.0,122060054.0,60360.0,63805.0,11.76,2400601.0,615940.0,34960.0,3051501.0,0.29,1595.0,1509.0,,
+ITBench AA,ArtificialAnalysis/ITBench-AA,40,https://huggingface.co/datasets/ArtificialAnalysis/ITBench-AA,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 5,Sonnet 5,False,1000,bf16,https://www.anthropic.com/claude/sonnet,harbor,https://github.com/harbor-framework/harbor,40.0,0.0,0.438,129942215.0,126175488.0,1064980.0,257182683.0,12915.0,16464.0,45.3,3248555.0,3154387.0,26624.0,6429567.0,1.13,411.0,322.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.6,Opus 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.633,438501505.0,419608736.0,2823496.0,860933737.0,126529.0,243310.0,434.54,1228295.0,1175374.0,7908.0,2411579.0,1.22,681.0,354.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,357.0,0.0,0.557,500304168.0,479035103.0,4937782.0,984277053.0,150652.0,269008.0,349.77,1401412.0,1341835.0,13831.0,2757078.0,0.98,753.0,421.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,31.0,0.317,1088634865.0,0.0,6200414.0,1094835279.0,398444.0,575536.0,15.05,3049397.0,0.0,17368.0,3066765.0,0.04,1612.0,1116.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.294,419671770.0,0.0,2277963.0,421949733.0,98909.0,169021.0,3.74,1175551.0,0.0,6380.0,1181932.0,0.01,473.0,277.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,357.0,20.0,0.23,419282595.0,0.0,1954817.0,421237412.0,100632.0,168697.0,3.8,1174461.0,0.0,5475.0,1179936.0,0.01,472.0,281.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,14.0,0.224,783670817.0,0.0,5510291.0,789181108.0,144352.0,355122.0,38.03,2195156.0,0.0,15434.0,2210591.0,0.11,994.0,404.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,7.0,0.308,839586046.0,0.0,5706139.0,845292185.0,94762.0,251102.0,37.44,2351781.0,0.0,15983.0,2367765.0,0.1,703.0,265.0,,
+RH SWE-Bench,rounakbende10/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,357.0,13.0,0.216,770574986.0,0.0,6519128.0,777094114.0,116348.0,360934.0,30.65,2158473.0,0.0,18260.0,2176734.0,0.09,1011.0,325.0,,
+RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,9.0,0.493,885869433.0,0.0,4763916.0,890633349.0,201374.0,401657.0,73.84,2481426.0,0.0,13344.0,2494771.0,0.21,1125.0,564.0,,
+RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,10.0,0.44,410001939.0,0.0,2851571.0,412853510.0,115135.0,315451.0,42.22,1148464.0,0.0,7987.0,1156452.0,0.12,883.0,322.0,,
+RH SWE-Bench,rounakbende/rh-swe-bench,357,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,357.0,1.0,0.468,560672462.0,0.0,3618673.0,564291135.0,114791.0,333385.0,42.09,1570511.0,0.0,10136.0,1580647.0,0.12,933.0,321.0,,
+RH SWE-Bench v1.1,rounakbende/rh-swe-bench,341,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,341.0,13.0,0.85,887790747.0,0.0,8417974.0,896208721.0,300687.0,454567.0,28.4,2603491.0,0.0,24686.0,2628178.0,0.08,1333.0,881.0,2.1.220,0.0
+RH SWE-Bench v1.1,rounakbende/rh-swe-bench,341,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,341.0,38.0,0.842,792804722.0,0.0,7969715.0,800774437.0,344045.0,416516.0,25.99,2324940.0,0.0,23371.0,2348312.0,0.08,1221.0,1008.0,2.1.220,0.0
+RH SWE-Bench v1.1,rounakbende/rh-swe-bench,341,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,341.0,11.0,0.871,738071568.0,0.0,7662094.0,745733662.0,109058.0,151123.0,8.24,2164432.0,0.0,22469.0,2186902.0,0.02,443.0,319.0,1.18.1,0.0
+RH SWE-Bench v1.1,rounakbende/rh-swe-bench,341,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,341.0,7.0,0.818,707447115.0,0.0,8943726.0,716390841.0,196440.0,260779.0,18.55,2074624.0,0.0,26227.0,2100852.0,0.05,764.0,576.0,0.73.1,0.0
+RH SWE-Bench v1.1,rounakbende/rh-swe-bench,341,https://huggingface.co/datasets/rounakbende/rh-swe-bench,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,341.0,7.0,0.891,749469406.0,0.0,7904331.0,757373737.0,184768.0,243195.0,17.45,2197857.0,0.0,23179.0,2221037.0,0.05,713.0,541.0,1.18.1,0.0
+RH SWE-Bench v1.1,rounakbende/rh-swe-bench,341,https://huggingface.co/datasets/rounakbende/rh-swe-bench,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,341.0,13.0,0.821,795776451.0,0.0,9637556.0,805414007.0,209274.0,266853.0,19.76,2333655.0,0.0,28262.0,2361917.0,0.06,782.0,613.0,0.73.1,0.0
+Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,115.0,6.0,0.0,0.0,0.0,0.0,0.0,29895.0,36503.0,8.3,0.0,0.0,0.0,0.0,0.07,317.0,259.0,,
+Shellbench,ShellBench/public-tasks,109,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GLM-5.2-FP8,RedHatAI/GLM-5.2-FP8,True,743,fp8,https://huggingface.co/RedHatAI/GLM-5.2-FP8,harbor,https://github.com/harbor-framework/harbor,109.0,2.0,0.187,0.0,0.0,0.0,0.0,20060.0,28907.0,22.29,0.0,0.0,0.0,0.0,0.2,265.0,184.0,,
+Shellbench,ShellBench/public-tasks,109,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GLM-5.2,z-ai/glm-5.2,True,743,,https://openrouter.ai/z-ai/glm-5.2,harbor,https://github.com/harbor-framework/harbor,109.0,0.0,0.183,2176246.0,1912720.0,64113.0,4153079.0,16440.0,21509.0,0.0,19965.0,17547.0,588.0,38101.0,0.0,197.0,150.0,,
+Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.0,0.0,0.0,0.0,0.0,8170.0,13296.0,2.27,0.0,0.0,0.0,0.0,0.02,115.0,71.0,,
+Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,fp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,2.0,0.009,0.0,0.0,0.0,0.0,4846.0,9862.0,1.35,0.0,0.0,0.0,0.0,0.01,85.0,42.0,,
+Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,fp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.014,0.0,0.0,0.0,0.0,21575.0,26392.0,5.99,0.0,0.0,0.0,0.0,0.05,229.0,187.0,,
+Shellbench,ShellBench/public-tasks,115,https://github.com/ShellBench/public-tasks,OpenClaw,[],True,https://github.com/OpenClaw/OpenClaw,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,115.0,1.0,0.128,0.0,0.0,0.0,0.0,10812.0,16949.0,6.01,0.0,0.0,0.0,0.0,0.05,147.0,94.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.698,192346997.0,186506482.0,2179945.0,381033424.0,32745.0,39030.0,185.66155285,2003614.0,1942775.0,22707.0,3969098.0,1.93,406.0,341.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.781,187217712.0,187209844.0,1280944.0,375708500.0,30686.0,39352.0,151.4104807499999,1950184.0,1950102.0,13343.0,3913630.0,1.58,409.0,319.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.5,190672390.0,184409111.0,1593112.0,376674613.0,40527.0,49734.0,184.42824125000004,1986170.0,1920928.0,16594.0,3923693.0,1.92,518.0,422.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,16.0,0.417,122366824.0,0.0,1307027.0,123673851.0,138148.0,158812.0,54.82,1274654.0,0.0,13614.0,1288269.0,0.57,1654.0,1439.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,12.0,0.417,100068358.0,0.0,1261237.0,101329595.0,60747.0,77623.0,19.28,1042378.0,0.0,13137.0,1055516.0,0.2,808.0,632.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.469,78678324.0,0.0,1370430.0,80048754.0,51767.0,66845.0,16.43,819565.0,0.0,14275.0,833841.0,0.17,696.0,539.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.604,198924339.0,189578624.0,1560836.0,390063799.0,32914.0,39468.0,188.34296700000002,2072128.0,1974777.0,16258.0,4063164.0,1.96,411.0,342.0,,
+SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.375,347744043.0,0.0,1903023.0,349647066.0,59653.0,67586.0,2.25,3622333.0,0.0,19823.0,3642156.0,0.02,704.0,621.0,,
+SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.333,112041538.0,0.0,781173.0,112822711.0,25086.0,31180.0,0.95,1167099.0,0.0,8137.0,1175236.0,0.01,324.0,261.0,,
+SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,96.0,15.0,0.292,125247664.0,0.0,597435.0,125845099.0,75715.0,79741.0,2.86,1304663.0,0.0,6223.0,1310886.0,0.03,830.0,788.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.26,147452235.0,0.0,953241.0,148405476.0,33613.0,55241.0,5.84,1535960.0,0.0,9929.0,1545890.0,0.06,575.0,350.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.385,140362798.0,0.0,1230697.0,141593495.0,23543.0,49289.0,6.54,1462112.0,0.0,12819.0,1474932.0,0.07,513.0,245.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.333,104994429.0,0.0,1095179.0,106089608.0,15004.0,55176.0,4.17,1093691.0,0.0,11408.0,1105100.0,0.04,574.0,156.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.432,406026150.0,0.0,2839236.0,408865386.0,100355.0,115220.0,26.44,4229439.0,0.0,29575.0,4259014.0,0.28,1200.0,1045.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.323,686392098.0,0.0,2915876.0,689307974.0,86130.0,99705.0,19.45,7149917.0,0.0,30373.0,7180291.0,0.2,1038.0,897.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,14.0,0.375,1051387065.0,0.0,3651367.0,1055038432.0,113768.0,127326.0,25.69,10951948.0,0.0,38035.0,10989983.0,0.27,1326.0,1185.0,,
+SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,8.0,0.521,386929017.0,0.0,1847087.0,388776104.0,118693.0,137736.0,43.52,4030510.0,0.0,19240.0,4049751.0,0.45,1434.0,1236.0,,
+SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.573,344853232.0,0.0,1661197.0,346514429.0,118356.0,196235.0,43.4,3592221.0,0.0,17304.0,3609525.0,0.45,2044.0,1232.0,,
+SWE-Bench Pro -- Ansible,scale-ai/swe-bench-pro,96,https://huggingface.co/datasets/scale-ai/swe-bench-pro,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,0.0,0.49,395189021.0,0.0,2206097.0,397395118.0,80305.0,91995.0,29.45,4116552.0,0.0,22980.0,4139532.0,0.31,958.0,836.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,6.0,0.458,367897697.0,0.0,1694885.0,369592582.0,39024.0,46758.0,9.64,3832267.0,0.0,17655.0,3849922.0,0.1,487.0,406.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.375,207164679.0,0.0,1598703.0,208763382.0,49450.0,57287.0,12.21,2157965.0,0.0,16653.0,2174618.0,0.13,596.0,515.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,5.0,0.406,0.0,0.0,0.0,0.0,38085.0,50779.0,9.4,0.0,0.0,0.0,0.0,0.1,528.0,396.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,1.0,0.479,742491363.0,0.0,2387609.0,744878972.0,54543.0,62422.0,13.47,7734285.0,0.0,24870.0,7759155.0,0.14,650.0,568.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,96.0,9.0,0.438,159198517.0,0.0,972133.0,160170650.0,33610.0,38272.0,9.34,1658317.0,0.0,10126.0,1668444.0,0.1,398.0,350.0,,
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,96.0,9.0,0.865,528943964.0,0.0,4599074.0,533543038.0,143977.0,150271.0,13.6,5509832.0,0.0,47907.0,5557739.0,0.14,1565.0,1499.0,2.1.220,0.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,20.0,0.823,432805899.0,0.0,4034016.0,436839915.0,201283.0,209615.0,15.21,4508394.0,0.0,42021.0,4550415.0,0.16,2183.0,2096.0,2.1.220,0.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.875,463743453.0,0.0,4163018.0,467906471.0,110158.0,114666.0,8.32,4830660.0,0.0,43364.0,4874025.0,0.09,1194.0,1147.0,1.18.1,0.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,96.0,8.0,0.76,390149875.0,0.0,4612564.0,394762439.0,119698.0,123975.0,9.04,4064061.0,0.0,48047.0,4112108.0,0.09,1291.0,1246.0,0.73.1,0.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,96.0,2.0,0.896,425282514.0,0.0,4060188.0,429342702.0,91941.0,95885.0,8.68,4430026.0,0.0,42293.0,4472319.0,0.09,998.0,957.0,1.18.1,0.0
+SWE-Bench Pro -- Ansible,ScaleAI/SWE-bench_Pro,96,https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,96.0,4.0,0.708,371935761.0,0.0,4594364.0,376530125.0,95294.0,101829.0,9.0,3874330.0,0.0,47857.0,3922188.0,0.09,1060.0,992.0,0.73.1,0.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.868,402225158.0,388781382.0,4588859.0,795595399.0,104467.0,159566.0,394.7823929,804450.0,777562.0,9177.0,1591190.0,0.79,319.0,208.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Opus 4.8,Opus 4.8,False,1000,bf16,https://www.anthropic.com/news/claude-opus-4-8,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.834,347751942.0,347725246.0,3640204.0,699117392.0,114633.0,171540.0,319.53304049999974,695503.0,695450.0,7280.0,1398234.0,0.64,343.0,229.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Sonnet 4.6,Sonnet 4.6,False,1000,bf16,https://www.anthropic.com/news/claude-sonnet-4-6,harbor,https://github.com/harbor-framework/harbor,,,0.796,,,,,,,,,,,,,,,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,35.0,0.612,651306166.0,0.0,4591859.0,655898025.0,524857.0,669355.0,104.14,1302612.0,0.0,9183.0,1311796.0,0.21,1338.0,1049.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.606,525563275.0,0.0,4183675.0,529746950.0,166092.0,444527.0,52.73,1051126.0,0.0,8367.0,1059493.0,0.11,889.0,332.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Gemma4-31B-FP8,RedHatAI/gemma-4-31B-it-FP8-block,True,31,fp8,https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.574,377479979.0,0.0,4177117.0,381657096.0,122274.0,385336.0,38.82,754959.0,0.0,8354.0,763314.0,0.08,770.0,244.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Codex,[],True,https://github.com/openai/codex,GPT 5.5 - high,GPT 5.5 - high,False,9700,bf16,https://openai.com/index/introducing-gpt-5-5,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.798,415795756.0,386352384.0,3431224.0,805579364.0,92588.0,141637.0,443.32977200000016,831591.0,772704.0,6862.0,1611158.0,0.89,283.0,185.0,,
+SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Claude Code,[],False,https://docs.anthropic.com/en/docs/claude-code,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,13.0,0.376,833651681.0,0.0,5656173.0,839307854.0,150981.0,200547.0,5.7,1667303.0,0.0,11312.0,1678615.0,0.01,401.0,301.0,,
+SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,33.0,0.326,202856940.0,0.0,1444371.0,204301311.0,35190.0,322482.0,3.32,405713.0,0.0,2888.0,408602.0,0.01,644.0,70.0,,
+SWE-Bench Verified,swe-bench/swe-bench-verified,500,https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,GPT-OSS-120B,RedHatAI/gpt-oss-120b,True,120,mxfp4,https://huggingface.co/RedHatAI/gpt-oss-120b,harbor,https://github.com/harbor-framework/harbor,500.0,47.0,0.234,90107247.0,0.0,905031.0,91012278.0,146747.0,176174.0,13.86,180214.0,0.0,1810.0,182024.0,0.03,352.0,293.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,53.0,0.118,309857096.0,0.0,2416878.0,312273974.0,54338.0,555599.0,9.43,619714.0,0.0,4833.0,624547.0,0.02,1111.0,108.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.318,373713878.0,0.0,4051687.0,377765565.0,49103.0,543025.0,13.64,747427.0,0.0,8103.0,755531.0,0.03,1086.0,98.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Mistral-Small-4-119B-2603-NVFP4,RedHatAI/Mistral-Small-4-119B-2603-NVFP4,True,119,nvfp4,https://huggingface.co/RedHatAI/Mistral-Small-4-119B-2603-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.232,195246618.0,0.0,2726988.0,197973606.0,29236.0,458591.0,8.12,390493.0,0.0,5453.0,395947.0,0.02,917.0,58.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,0.0,0.472,882330271.0,0.0,6689285.0,889019556.0,174288.0,410828.0,45.91,1764660.0,0.0,13378.0,1778039.0,0.09,821.0,348.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.41,724984042.0,0.0,5941303.0,730925345.0,93088.0,518565.0,21.02,1449968.0,0.0,11882.0,1461850.0,0.04,1037.0,186.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Nemotron-3-Super-120B-NVFP4,RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,True,120,nvfp4,https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.498,996839122.0,0.0,7196671.0,1004035793.0,144084.0,446845.0,32.53,1993678.0,0.0,14393.0,2008071.0,0.07,893.0,288.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.694,1005844088.0,0.0,6255800.0,1012099888.0,228858.0,411348.0,83.91,2011688.0,0.0,12511.0,2024199.0,0.17,822.0,457.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/opencode-ai/opencode,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,8.0,0.642,395291929.0,0.0,3296668.0,398588597.0,92700.0,374913.0,33.99,790583.0,0.0,6593.0,797177.0,0.07,749.0,185.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.6-27B-FP8,RedHatAI/Qwen3.6-27B-FP8,True,27,fp8,https://huggingface.co/RedHatAI/Qwen3.6-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,7.0,0.694,544070591.0,0.0,4965652.0,549036243.0,130591.0,408706.0,47.88,1088141.0,0.0,9931.0,1098072.0,0.1,817.0,261.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,1.0,0.632,1106618897.0,0.0,5733245.0,1112352142.0,122808.0,171897.0,34.11,2213237.0,0.0,11466.0,2224704.0,0.07,343.0,245.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenClaw,[],True,https://github.com/openclaw/openclaw,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.588,0.0,0.0,0.0,0.0,120399.0,200354.0,33.44,0.0,0.0,0.0,0.0,0.07,400.0,240.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.548,469806650.0,0.0,4937761.0,474744411.0,120473.0,185168.0,29.75,939613.0,0.0,9875.0,949488.0,0.06,370.0,240.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,6.0,0.65,791183735.0,0.0,6333798.0,797517533.0,154531.0,218988.0,38.16,1582367.0,0.0,12667.0,1595035.0,0.08,437.0,309.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Qwen Code,[],True,https://github.com/QwenLM/qwen-code,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,500.0,3.0,0.638,609589099.0,0.0,3964252.0,613553351.0,132273.0,178984.0,36.74,1219178.0,0.0,7928.0,1227106.0,0.07,357.0,264.0,,
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,500.0,12.0,0.868,1669922350.0,0.0,20004121.0,1689926471.0,556023.0,596836.0,52.51,3339844.0,0.0,40008.0,3379852.0,0.11,1193.0,1112.0,2.1.220,0.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Claude Code,[],False,https://github.com/anthropics/claude-code,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,73.0,0.822,1512203005.0,0.0,19765110.0,1531968115.0,780181.0,829940.0,58.95,3024406.0,0.0,39530.0,3063936.0,0.12,1659.0,1560.0,2.1.220,0.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.936,1121951449.0,0.0,13182636.0,1135134085.0,319883.0,353489.0,24.17,2243902.0,0.0,26365.0,2270268.0,0.05,706.0,639.0,1.18.1,0.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.8-27B-FP8,Qwen/Qwen3.8-27B-FP8,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B-FP8,harbor,https://github.com/harbor-framework/harbor,500.0,10.0,0.66,935430893.0,0.0,17724428.0,953155321.0,391922.0,424230.0,29.61,1870861.0,0.0,35448.0,1906310.0,0.06,848.0,783.0,0.73.1,0.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,500.0,4.0,0.94,1210189326.0,0.0,15000753.0,1225190079.0,308680.0,334780.0,29.15,2420378.0,0.0,30001.0,2450380.0,0.06,669.0,617.0,1.18.1,0.0
+SWE-Bench Verified,SWE-bench/SWE-bench_Verified,500,https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified,Pi,[],True,https://github.com/plandex-ai/plandex,Qwen3.8-27B,Qwen/Qwen3.8-27B,True,27,bf16,https://huggingface.co/Qwen/Qwen3.8-27B,harbor,https://github.com/harbor-framework/harbor,500.0,11.0,0.632,912012141.0,0.0,17006338.0,929018479.0,618114.0,674123.0,58.38,1824024.0,0.0,34012.0,1858036.0,0.12,1348.0,1236.0,0.73.1,0.0
+Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,OpenCode,[],True,https://github.com/anomalyco/opencode,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.303,47607780.0,0.0,1657188.0,49264968.0,46467.0,58507.0,11.47,534918.0,0.0,18620.0,553538.0,0.13,657.0,522.0,,
+Terminal Bench 2.0,terminal-bench/terminal-bench-2,89,https://www.tbench.ai/benchmarks/terminal-bench-2,Pi,[],True,https://github.com/earendil-works/pi/tree/main,Qwen3.6-35B-A3B-NVFP4,RedHatAI/Qwen3.6-35B-A3B-NVFP4,True,35,nvfp4,https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4,harbor,https://github.com/harbor-framework/harbor,89.0,5.0,0.36,82108716.0,0.0,2056390.0,84165106.0,44991.0,64760.0,11.11,922569.0,0.0,23105.0,945675.0,0.12,727.0,505.0,,

results/rh-swe-bench-qwen3-8-27b-claude-code.json

@@ -0,0 +1,51 @@
+{
+    "benchmark": {
+        "name": "RH SWE-Bench v1.1",
+        "repo": "rounakbende/rh-swe-bench",
+        "num_tasks": 341,
+        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
+    },
+    "harness": {
+        "name": "Claude Code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code",
+        "version": "2.1.220"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "rounakbende10/rh-swe-bench",
+            "ref": "sha256:96005bdeb80c90e6a487df8cb893dc2856b2bec88d122a14580abcc58481ef75"
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 341,
+        "n_errors": 13,
+        "score": 0.85,
+        "n_input_tokens": 887790747,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 8417974,
+        "n_total_tokens": 896208721,
+        "agent_time_seconds": 300687,
+        "total_time_seconds": 454567,
+        "cost_usd": 28.4,
+        "mean_input_tokens_per_task": 2603491,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 24686,
+        "mean_tokens_per_task": 2628178,
+        "mean_cost_usd_per_task": 0.08,
+        "mean_total_time_seconds_per_task": 1333,
+        "mean_agent_time_seconds_per_task": 881,
+        "cache_hit_rate": 0.0
+    }
+}

results/rh-swe-bench-qwen3-8-27b-fp8-claude-code.json

@@ -0,0 +1,51 @@
+{
+    "benchmark": {
+        "name": "RH SWE-Bench v1.1",
+        "repo": "rounakbende/rh-swe-bench",
+        "num_tasks": 341,
+        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
+    },
+    "harness": {
+        "name": "Claude Code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code",
+        "version": "2.1.220"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "rounakbende10/rh-swe-bench",
+            "ref": "sha256:96005bdeb80c90e6a487df8cb893dc2856b2bec88d122a14580abcc58481ef75"
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 341,
+        "n_errors": 38,
+        "score": 0.842,
+        "n_input_tokens": 792804722,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 7969715,
+        "n_total_tokens": 800774437,
+        "agent_time_seconds": 344045,
+        "total_time_seconds": 416516,
+        "cost_usd": 25.99,
+        "mean_input_tokens_per_task": 2324940,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 23371,
+        "mean_tokens_per_task": 2348312,
+        "mean_cost_usd_per_task": 0.08,
+        "mean_total_time_seconds_per_task": 1221,
+        "mean_agent_time_seconds_per_task": 1008,
+        "cache_hit_rate": 0.0
+    }
+}

results/rh-swe-bench-qwen3-8-27b-fp8-opencode.json

@@ -0,0 +1,51 @@
+{
+    "benchmark": {
+        "name": "RH SWE-Bench v1.1",
+        "repo": "rounakbende/rh-swe-bench",
+        "num_tasks": 341,
+        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
+    },
+    "harness": {
+        "name": "OpenCode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode",
+        "version": "1.18.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "rounakbende10/rh-swe-bench",
+            "ref": "sha256:96005bdeb80c90e6a487df8cb893dc2856b2bec88d122a14580abcc58481ef75"
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 341,
+        "n_errors": 11,
+        "score": 0.871,
+        "n_input_tokens": 738071568,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 7662094,
+        "n_total_tokens": 745733662,
+        "agent_time_seconds": 109058,
+        "total_time_seconds": 151123,
+        "cost_usd": 8.24,
+        "mean_input_tokens_per_task": 2164432,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 22469,
+        "mean_tokens_per_task": 2186902,
+        "mean_cost_usd_per_task": 0.02,
+        "mean_total_time_seconds_per_task": 443,
+        "mean_agent_time_seconds_per_task": 319,
+        "cache_hit_rate": 0.0
+    }
+}

results/rh-swe-bench-qwen3-8-27b-fp8-pi.json

@@ -0,0 +1,51 @@
+{
+    "benchmark": {
+        "name": "RH SWE-Bench v1.1",
+        "repo": "rounakbende/rh-swe-bench",
+        "num_tasks": 341,
+        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
+    },
+    "harness": {
+        "name": "Pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/plandex-ai/plandex",
+        "version": "0.73.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "rounakbende10/rh-swe-bench",
+            "ref": "sha256:96005bdeb80c90e6a487df8cb893dc2856b2bec88d122a14580abcc58481ef75"
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 341,
+        "n_errors": 7,
+        "score": 0.818,
+        "n_input_tokens": 707447115,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 8943726,
+        "n_total_tokens": 716390841,
+        "agent_time_seconds": 196440,
+        "total_time_seconds": 260779,
+        "cost_usd": 18.55,
+        "mean_input_tokens_per_task": 2074624,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 26227,
+        "mean_tokens_per_task": 2100852,
+        "mean_cost_usd_per_task": 0.05,
+        "mean_total_time_seconds_per_task": 764,
+        "mean_agent_time_seconds_per_task": 576,
+        "cache_hit_rate": 0.0
+    }
+}

results/rh-swe-bench-qwen3-8-27b-opencode.json

@@ -0,0 +1,51 @@
+{
+    "benchmark": {
+        "name": "RH SWE-Bench v1.1",
+        "repo": "rounakbende/rh-swe-bench",
+        "num_tasks": 341,
+        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
+    },
+    "harness": {
+        "name": "OpenCode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode",
+        "version": "1.18.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "rounakbende10/rh-swe-bench",
+            "ref": "sha256:96005bdeb80c90e6a487df8cb893dc2856b2bec88d122a14580abcc58481ef75"
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 341,
+        "n_errors": 7,
+        "score": 0.891,
+        "n_input_tokens": 749469406,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 7904331,
+        "n_total_tokens": 757373737,
+        "agent_time_seconds": 184768,
+        "total_time_seconds": 243195,
+        "cost_usd": 17.45,
+        "mean_input_tokens_per_task": 2197857,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 23179,
+        "mean_tokens_per_task": 2221037,
+        "mean_cost_usd_per_task": 0.05,
+        "mean_total_time_seconds_per_task": 713,
+        "mean_agent_time_seconds_per_task": 541,
+        "cache_hit_rate": 0.0
+    }
+}

results/rh-swe-bench-qwen3-8-27b-pi.json

@@ -0,0 +1,51 @@
+{
+    "benchmark": {
+        "name": "RH SWE-Bench v1.1",
+        "repo": "rounakbende/rh-swe-bench",
+        "num_tasks": 341,
+        "url": "https://huggingface.co/datasets/rounakbende/rh-swe-bench"
+    },
+    "harness": {
+        "name": "Pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/plandex-ai/plandex",
+        "version": "0.73.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "rounakbende10/rh-swe-bench",
+            "ref": "sha256:96005bdeb80c90e6a487df8cb893dc2856b2bec88d122a14580abcc58481ef75"
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 341,
+        "n_errors": 13,
+        "score": 0.821,
+        "n_input_tokens": 795776451,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 9637556,
+        "n_total_tokens": 805414007,
+        "agent_time_seconds": 209274,
+        "total_time_seconds": 266853,
+        "cost_usd": 19.76,
+        "mean_input_tokens_per_task": 2333655,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 28262,
+        "mean_tokens_per_task": 2361917,
+        "mean_cost_usd_per_task": 0.06,
+        "mean_total_time_seconds_per_task": 782,
+        "mean_agent_time_seconds_per_task": 613,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-pro--ansible-qwen3-8-27b-claude-code.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Pro -- Ansible",
+        "repo": "ScaleAI/SWE-bench_Pro",
+        "num_tasks": 96,
+        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
+    },
+    "harness": {
+        "name": "Claude Code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code",
+        "version": "2.1.220"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 96,
+        "n_errors": 9,
+        "score": 0.865,
+        "n_input_tokens": 528943964,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4599074,
+        "n_total_tokens": 533543038,
+        "agent_time_seconds": 143977,
+        "total_time_seconds": 150271,
+        "cost_usd": 13.6,
+        "mean_input_tokens_per_task": 5509832,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 47907,
+        "mean_tokens_per_task": 5557739,
+        "mean_cost_usd_per_task": 0.14,
+        "mean_total_time_seconds_per_task": 1565,
+        "mean_agent_time_seconds_per_task": 1499,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-pro--ansible-qwen3-8-27b-fp8-claude-code.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Pro -- Ansible",
+        "repo": "ScaleAI/SWE-bench_Pro",
+        "num_tasks": 96,
+        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
+    },
+    "harness": {
+        "name": "Claude Code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code",
+        "version": "2.1.220"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 96,
+        "n_errors": 20,
+        "score": 0.823,
+        "n_input_tokens": 432805899,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4034016,
+        "n_total_tokens": 436839915,
+        "agent_time_seconds": 201283,
+        "total_time_seconds": 209615,
+        "cost_usd": 15.21,
+        "mean_input_tokens_per_task": 4508394,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 42021,
+        "mean_tokens_per_task": 4550415,
+        "mean_cost_usd_per_task": 0.16,
+        "mean_total_time_seconds_per_task": 2183,
+        "mean_agent_time_seconds_per_task": 2096,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-pro--ansible-qwen3-8-27b-fp8-opencode.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Pro -- Ansible",
+        "repo": "ScaleAI/SWE-bench_Pro",
+        "num_tasks": 96,
+        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
+    },
+    "harness": {
+        "name": "OpenCode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode",
+        "version": "1.18.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 96,
+        "n_errors": 2,
+        "score": 0.875,
+        "n_input_tokens": 463743453,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4163018,
+        "n_total_tokens": 467906471,
+        "agent_time_seconds": 110158,
+        "total_time_seconds": 114666,
+        "cost_usd": 8.32,
+        "mean_input_tokens_per_task": 4830660,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 43364,
+        "mean_tokens_per_task": 4874025,
+        "mean_cost_usd_per_task": 0.09,
+        "mean_total_time_seconds_per_task": 1194,
+        "mean_agent_time_seconds_per_task": 1147,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-pro--ansible-qwen3-8-27b-fp8-pi.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Pro -- Ansible",
+        "repo": "ScaleAI/SWE-bench_Pro",
+        "num_tasks": 96,
+        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
+    },
+    "harness": {
+        "name": "Pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/plandex-ai/plandex",
+        "version": "0.73.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 96,
+        "n_errors": 8,
+        "score": 0.76,
+        "n_input_tokens": 390149875,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4612564,
+        "n_total_tokens": 394762439,
+        "agent_time_seconds": 119698,
+        "total_time_seconds": 123975,
+        "cost_usd": 9.04,
+        "mean_input_tokens_per_task": 4064061,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 48047,
+        "mean_tokens_per_task": 4112108,
+        "mean_cost_usd_per_task": 0.09,
+        "mean_total_time_seconds_per_task": 1291,
+        "mean_agent_time_seconds_per_task": 1246,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-pro--ansible-qwen3-8-27b-opencode.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Pro -- Ansible",
+        "repo": "ScaleAI/SWE-bench_Pro",
+        "num_tasks": 96,
+        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
+    },
+    "harness": {
+        "name": "OpenCode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode",
+        "version": "1.18.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 96,
+        "n_errors": 2,
+        "score": 0.896,
+        "n_input_tokens": 425282514,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4060188,
+        "n_total_tokens": 429342702,
+        "agent_time_seconds": 91941,
+        "total_time_seconds": 95885,
+        "cost_usd": 8.68,
+        "mean_input_tokens_per_task": 4430026,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 42293,
+        "mean_tokens_per_task": 4472319,
+        "mean_cost_usd_per_task": 0.09,
+        "mean_total_time_seconds_per_task": 998,
+        "mean_agent_time_seconds_per_task": 957,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-pro--ansible-qwen3-8-27b-pi.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Pro -- Ansible",
+        "repo": "ScaleAI/SWE-bench_Pro",
+        "num_tasks": 96,
+        "url": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"
+    },
+    "harness": {
+        "name": "Pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/plandex-ai/plandex",
+        "version": "0.73.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 96,
+        "n_errors": 4,
+        "score": 0.708,
+        "n_input_tokens": 371935761,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 4594364,
+        "n_total_tokens": 376530125,
+        "agent_time_seconds": 95294,
+        "total_time_seconds": 101829,
+        "cost_usd": 9.0,
+        "mean_input_tokens_per_task": 3874330,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 47857,
+        "mean_tokens_per_task": 3922188,
+        "mean_cost_usd_per_task": 0.09,
+        "mean_total_time_seconds_per_task": 1060,
+        "mean_agent_time_seconds_per_task": 992,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-verified-qwen3-8-27b-claude-code.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "Claude Code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code",
+        "version": "2.1.220"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 500,
+        "n_errors": 12,
+        "score": 0.868,
+        "n_input_tokens": 1669922350,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 20004121,
+        "n_total_tokens": 1689926471,
+        "agent_time_seconds": 556023,
+        "total_time_seconds": 596836,
+        "cost_usd": 52.51,
+        "mean_input_tokens_per_task": 3339844,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 40008,
+        "mean_tokens_per_task": 3379852,
+        "mean_cost_usd_per_task": 0.11,
+        "mean_total_time_seconds_per_task": 1193,
+        "mean_agent_time_seconds_per_task": 1112,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-verified-qwen3-8-27b-fp8-claude-code.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "Claude Code",
+        "skills": [],
+        "is_oss": false,
+        "url": "https://github.com/anthropics/claude-code",
+        "version": "2.1.220"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 500,
+        "n_errors": 73,
+        "score": 0.822,
+        "n_input_tokens": 1512203005,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 19765110,
+        "n_total_tokens": 1531968115,
+        "agent_time_seconds": 780181,
+        "total_time_seconds": 829940,
+        "cost_usd": 58.95,
+        "mean_input_tokens_per_task": 3024406,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 39530,
+        "mean_tokens_per_task": 3063936,
+        "mean_cost_usd_per_task": 0.12,
+        "mean_total_time_seconds_per_task": 1659,
+        "mean_agent_time_seconds_per_task": 1560,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-verified-qwen3-8-27b-fp8-opencode.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "OpenCode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode",
+        "version": "1.18.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 500,
+        "n_errors": 4,
+        "score": 0.936,
+        "n_input_tokens": 1121951449,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 13182636,
+        "n_total_tokens": 1135134085,
+        "agent_time_seconds": 319883,
+        "total_time_seconds": 353489,
+        "cost_usd": 24.17,
+        "mean_input_tokens_per_task": 2243902,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 26365,
+        "mean_tokens_per_task": 2270268,
+        "mean_cost_usd_per_task": 0.05,
+        "mean_total_time_seconds_per_task": 706,
+        "mean_agent_time_seconds_per_task": 639,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-verified-qwen3-8-27b-fp8-pi.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "Pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/plandex-ai/plandex",
+        "version": "0.73.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B-FP8",
+        "repo": "Qwen/Qwen3.8-27B-FP8",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B-FP8"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 500,
+        "n_errors": 10,
+        "score": 0.66,
+        "n_input_tokens": 935430893,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 17724428,
+        "n_total_tokens": 953155321,
+        "agent_time_seconds": 391922,
+        "total_time_seconds": 424230,
+        "cost_usd": 29.61,
+        "mean_input_tokens_per_task": 1870861,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 35448,
+        "mean_tokens_per_task": 1906310,
+        "mean_cost_usd_per_task": 0.06,
+        "mean_total_time_seconds_per_task": 848,
+        "mean_agent_time_seconds_per_task": 783,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-verified-qwen3-8-27b-opencode.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "OpenCode",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/anomalyco/opencode",
+        "version": "1.18.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 500,
+        "n_errors": 4,
+        "score": 0.94,
+        "n_input_tokens": 1210189326,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 15000753,
+        "n_total_tokens": 1225190079,
+        "agent_time_seconds": 308680,
+        "total_time_seconds": 334780,
+        "cost_usd": 29.15,
+        "mean_input_tokens_per_task": 2420378,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 30001,
+        "mean_tokens_per_task": 2450380,
+        "mean_cost_usd_per_task": 0.06,
+        "mean_total_time_seconds_per_task": 669,
+        "mean_agent_time_seconds_per_task": 617,
+        "cache_hit_rate": 0.0
+    }
+}

results/swe-bench-verified-qwen3-8-27b-pi.json

@@ -0,0 +1,54 @@
+{
+    "benchmark": {
+        "name": "SWE-Bench Verified",
+        "repo": "SWE-bench/SWE-bench_Verified",
+        "num_tasks": 500,
+        "url": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified"
+    },
+    "harness": {
+        "name": "Pi",
+        "skills": [],
+        "is_oss": true,
+        "url": "https://github.com/plandex-ai/plandex",
+        "version": "0.73.1"
+    },
+    "model": {
+        "name": "Qwen3.8-27B",
+        "repo": "Qwen/Qwen3.8-27B",
+        "is_oss": true,
+        "num_params": 27,
+        "precision": "bf16",
+        "url": "https://huggingface.co/Qwen/Qwen3.8-27B"
+    },
+    "environment": {
+        "name": "harbor",
+        "config": {
+            "name": "scale-ai/swe-bench-pro",
+            "ref": "sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+            "task_names": [
+                "*ansible*"
+            ]
+        },
+        "url": "https://github.com/harbor-framework/harbor"
+    },
+    "metrics": {
+        "n_tasks": 500,
+        "n_errors": 11,
+        "score": 0.632,
+        "n_input_tokens": 912012141,
+        "n_cache_tokens": 0,
+        "n_output_tokens": 17006338,
+        "n_total_tokens": 929018479,
+        "agent_time_seconds": 618114,
+        "total_time_seconds": 674123,
+        "cost_usd": 58.38,
+        "mean_input_tokens_per_task": 1824024,
+        "mean_cache_tokens_per_task": 0,
+        "mean_output_tokens_per_task": 34012,
+        "mean_tokens_per_task": 1858036,
+        "mean_cost_usd_per_task": 0.12,
+        "mean_total_time_seconds_per_task": 1348,
+        "mean_agent_time_seconds_per_task": 1236,
+        "cache_hit_rate": 0.0
+    }
+}