{
  "schema_version": "0.2.2",
  "evaluation_id": "swebench_lite_dev/RedHatAI/Kimi-K2.6-NVFP4/1784247001.8595154",
  "retrieved_timestamp": "1784247001.8595154",
  "source_metadata": {
    "source_name": "SWE-bench harness",
    "source_type": "evaluation_run",
    "source_organization_name": "RedHatAI",
    "evaluator_relationship": "first_party",
    "additional_details": {
      "note": "predictions generated with mini-swe-agent, resolved-rate scored with the official SWE-bench harness (swebench.harness.run_evaluation)"
    }
  },
  "eval_library": {
    "name": "swebench",
    "version": "4.1.0"
  },
  "model_info": {
    "name": "RedHatAI/Kimi-K2.6-NVFP4",
    "id": "RedHatAI/Kimi-K2.6-NVFP4",
    "developer": "RedHatAI",
    "inference_engine": {
      "name": "vllm",
      "version": "0.24.0"
    },
    "additional_details": {
      "profile": "kimi_nvfp4",
      "subset": "lite",
      "split": "dev"
    }
  },
  "evaluation_results": [
    {
      "evaluation_name": "swebench_lite_dev/resolved_rate",
      "source_data": {
        "dataset_name": "SWE-bench Lite dev",
        "source_type": "hf_dataset",
        "hf_repo": "princeton-nlp/SWE-Bench_Lite",
        "hf_split": "dev",
        "samples_number": 23
      },
      "metric_config": {
        "evaluation_description": "Resolve rate: fraction of benchmark instances where the submitted patch passes verification (0.0-1.0), scored with the official SWE-bench harness.",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 0.21739130434782608,
        "details": {
          "total_instances": "23",
          "resolved_instances": "5",
          "submitted_instances": "23",
          "completed_instances": "19",
          "unresolved_instances": "14",
          "empty_patch_instances": "4",
          "error_instances": "0"
        },
        "completed_ids": [
          "marshmallow-code__marshmallow-1343",
          "marshmallow-code__marshmallow-1359",
          "pvlib__pvlib-python-1072",
          "pvlib__pvlib-python-1154",
          "pvlib__pvlib-python-1606",
          "pvlib__pvlib-python-1707",
          "pvlib__pvlib-python-1854",
          "pydicom__pydicom-1139",
          "pydicom__pydicom-1256",
          "pydicom__pydicom-1413",
          "pydicom__pydicom-1694",
          "pydicom__pydicom-901",
          "pylint-dev__astroid-1196",
          "pylint-dev__astroid-1268",
          "pylint-dev__astroid-1866",
          "pylint-dev__astroid-1978",
          "pyvista__pyvista-4315",
          "sqlfluff__sqlfluff-1517",
          "sqlfluff__sqlfluff-2419"
        ],
        "submitted_ids": [
          "marshmallow-code__marshmallow-1343",
          "marshmallow-code__marshmallow-1359",
          "pvlib__pvlib-python-1072",
          "pvlib__pvlib-python-1154",
          "pvlib__pvlib-python-1606",
          "pvlib__pvlib-python-1707",
          "pvlib__pvlib-python-1854",
          "pydicom__pydicom-1139",
          "pydicom__pydicom-1256",
          "pydicom__pydicom-1413",
          "pydicom__pydicom-1694",
          "pydicom__pydicom-901",
          "pylint-dev__astroid-1196",
          "pylint-dev__astroid-1268",
          "pylint-dev__astroid-1333",
          "pylint-dev__astroid-1866",
          "pylint-dev__astroid-1978",
          "pyvista__pyvista-4315",
          "sqlfluff__sqlfluff-1517",
          "sqlfluff__sqlfluff-1625",
          "sqlfluff__sqlfluff-1733",
          "sqlfluff__sqlfluff-1763",
          "sqlfluff__sqlfluff-2419"
        ],
        "resolved_ids": [
          "marshmallow-code__marshmallow-1343",
          "pydicom__pydicom-1256",
          "pydicom__pydicom-1694",
          "pylint-dev__astroid-1268",
          "sqlfluff__sqlfluff-2419"
        ]
      },
      "generation_config": {
        "generation_args": {
          "agentic_eval_config": {
            "available_tools": [
              {
                "name": "bash"
              }
            ]
          },
          "max_attempts": 1
        }
      }
    },
    {
      "evaluation_name": "swebench_lite_dev/submission_rate",
      "source_data": {
        "dataset_name": "SWE-bench Lite dev",
        "source_type": "hf_dataset",
        "hf_repo": "princeton-nlp/SWE-Bench_Lite",
        "hf_split": "dev",
        "samples_number": 23
      },
      "metric_config": {
        "evaluation_description": "Submission rate (Submitted / total instances).",
        "lower_is_better": false,
        "score_type": "continuous",
        "min_score": 0.0,
        "max_score": 1.0
      },
      "score_details": {
        "score": 1.0,
        "details": {
          "total_instances": "23",
          "submitted_instances": "23",
          "unsubmitted_instances": "0"
        },
        "submitted_ids": [
          "marshmallow-code__marshmallow-1343",
          "marshmallow-code__marshmallow-1359",
          "pvlib__pvlib-python-1072",
          "pvlib__pvlib-python-1154",
          "pvlib__pvlib-python-1606",
          "pvlib__pvlib-python-1707",
          "pvlib__pvlib-python-1854",
          "pydicom__pydicom-1139",
          "pydicom__pydicom-1256",
          "pydicom__pydicom-1413",
          "pydicom__pydicom-1694",
          "pydicom__pydicom-901",
          "pylint-dev__astroid-1196",
          "pylint-dev__astroid-1268",
          "pylint-dev__astroid-1333",
          "pylint-dev__astroid-1866",
          "pylint-dev__astroid-1978",
          "pyvista__pyvista-4315",
          "sqlfluff__sqlfluff-1517",
          "sqlfluff__sqlfluff-1625",
          "sqlfluff__sqlfluff-1733",
          "sqlfluff__sqlfluff-1763",
          "sqlfluff__sqlfluff-2419"
        ]
      },
      "generation_config": {
        "generation_args": {
          "agentic_eval_config": {
            "available_tools": [
              {
                "name": "bash"
              }
            ]
          },
          "max_attempts": 1
        }
      }
    }
  ]
}