{ "status": "passed", "release": "v0.0.1-alpha", "base_model": "Qwen/Qwen3-4B", "base_model_revision": "1cfa9a7208912126459214e8b04321603b3df60c", "gpu": "Tesla T4", "dataset": { "name": "nimora-agent-alpha-v0.0.1", "license": "Apache-2.0", "provenance": "deterministic synthetic templates authored for Nimora", "supervision": "prefix_to_next_decision", "limitations": [ "Template-generated data does not establish real-world coding quality.", "No private repositories, user logs, credentials, or scraped code are included.", "Provider names appear in tasks, but actions use provider-neutral runtime tools." ], "families": [ "create_and_test", "edit_and_test", "git_to_change", "inspect", "permission_recovery", "review_approve", "review_block", "review_merge", "stale_write_recovery", "test_recovery" ], "counts": { "train": 400, "validation": 40, "eval": 40 }, "split_index_ranges": { "train": [ 0, 39 ], "validation": [ 10000, 10003 ], "eval": [ 20000, 20003 ] } }, "training": { "train": { "train_runtime": 805.2917, "train_samples_per_second": 0.497, "train_steps_per_second": 0.497, "total_flos": 3575252515292160.0, "train_loss": 0.3089988502621418, "epoch": 1.0 }, "evaluation": { "eval_loss": 0.004357926081866026, "eval_runtime": 23.0059, "eval_samples_per_second": 1.739, "eval_steps_per_second": 1.739, "epoch": 1.0 }, "adapter_dir": "/content/nimora-agent-alpha/run/final-adapter" }, "base_evaluation": { "count": 40, "valid_rate": 1.0, "semantic_rate": 0.275, "arguments_exact_rate": 0.2, "cases": [ { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Find the parser implementation file\",\"action\":{\"name\":\"find_file\",\"arguments\":{\"filename\":\"parser.py\"}}}", "actual": { "plan": "Find the parser implementation file", "action": { "name": "find_file", "arguments": { "filename": "parser.py" } }, "result": null }, "family": "inspect", "expected": { "action": { "arguments": { "path": "src", "query": "def parser" }, "name": "workspace.search" }, "plan": "Search narrowly before reading files." } }, { "valid": true, "semantic": true, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/router.py\",\"query\":\"def router\"},\"name\":\"workspace.read\"},\"plan\":\"Read the file to find the relevant entry point.\"}", "actual": { "plan": "Read the file to find the relevant entry point.", "action": { "name": "workspace.read", "arguments": { "path": "src/router.py", "query": "def router" } }, "result": null }, "family": "inspect", "expected": { "action": { "arguments": { "end_line": 90, "path": "src/router.py", "start_line": 1 }, "name": "workspace.read" }, "plan": "Read the matching implementation." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"The serializer implementation is located in src/serializer.py. The relevant entry point is the serializer function, which returns the normalized value of the input.\"}", "actual": { "plan": null, "action": null, "result": "The serializer implementation is located in src/serializer.py. The relevant entry point is the serializer function, which returns the normalized value of the input." }, "family": "inspect", "expected": { "result": "Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Find the scheduler implementation file\",\"action\":{\"name\":\"find_file\",\"arguments\":{\"filename\":\"scheduler.py\"}}}", "actual": { "plan": "Find the scheduler implementation file", "action": { "name": "find_file", "arguments": { "filename": "scheduler.py" } }, "result": null }, "family": "inspect", "expected": { "action": { "arguments": { "path": "src", "query": "def scheduler" }, "name": "workspace.search" }, "plan": "Search narrowly before reading files." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/parser.py\",\"line_number\":1,\"content\":\"def handle(items):\\n return items[0]\\n\"},\"name\":\"workspace.write\"},\"plan\":\"Modify the parser to handle empty input by returning an empty list.\"}", "actual": { "plan": "Modify the parser to handle empty input by returning an empty list.", "action": { "name": "workspace.write", "arguments": { "path": "src/parser.py", "line_number": 1, "content": "def handle(items):\n return items[0]\n" } }, "result": null }, "family": "edit_and_test", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779", "new_text": "def handle(items):\n if not items:\n return None\n return items[0]", "old_text": "def handle(items):\n return items[0]", "path": "src/parser.py" }, "name": "workspace.replace" }, "plan": "Apply the smallest concurrency-safe replacement." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config.py\",\"sha256\":\"e71ba961d8f5e8191e177c3a6280684464be7b4cba26445f75574f28aceb5347\"},\"name\":\"workspace.read\"},\"plan\":\"Verify the replacement and capture the new content hash.\"}", "actual": { "plan": "Verify the replacement and capture the new content hash.", "action": { "name": "workspace.read", "arguments": { "path": "src/config.py", "sha256": "e71ba961d8f5e8191e177c3a6280684464be7b4cba26445f75574f28aceb5347" } }, "result": null }, "family": "edit_and_test", "expected": { "action": { "arguments": { "argv": [ "pytest", "-q", "tests/test_config.py" ], "cwd": "." }, "name": "shell.run" }, "plan": "Run only the relevant test module first." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"The empty-input regression in the client has been fixed and the focused tests have passed successfully.\"}", "actual": { "plan": null, "action": null, "result": "The empty-input regression in the client has been fixed and the focused tests have passed successfully." }, "family": "edit_and_test", "expected": { "result": "Fixed empty-input handling in `src/client.py`; all 6 focused tests pass." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Check for the presence of the empty-input regression in the cache module\",\"action\":{\"name\":\"check_regression\",\"arguments\":{\"module\":\"cache\",\"regression\":\"empty-input\"}}}", "actual": { "plan": "Check for the presence of the empty-input regression in the cache module", "action": { "name": "check_regression", "arguments": { "module": "cache", "regression": "empty-input" } }, "result": null }, "family": "edit_and_test", "expected": { "action": { "arguments": { "path": "src/cache.py" }, "name": "workspace.read" }, "plan": "Read the current implementation and capture its content hash." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/health.py\",\"source\":\"def health():\\n return \\\"ok\\\"\\n\"},\"name\":\"workspace.read\"},\"plan\":\"Verify the module's content.\"}", "actual": { "plan": "Verify the module's content.", "action": { "name": "workspace.read", "arguments": { "path": "src/health.py", "source": "def health():\n return \"ok\"\n" } }, "result": null }, "family": "create_and_test", "expected": { "action": { "arguments": { "argv": [ "pytest", "-q", "tests/test_health.py" ], "cwd": "." }, "name": "shell.run" }, "plan": "Verify the new behavior with its focused test." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"The version module was successfully created and its focused test passed.\"}", "actual": { "plan": null, "action": null, "result": "The version module was successfully created and its focused test passed." }, "family": "create_and_test", "expected": { "result": "Added `src/version.py` and verified its 2 focused tests pass." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Add a diagnostics module and verify its test\",\"action\":{\"name\":\"add_file\",\"arguments\":{\"filename\":\"diagnostics.py\",\"content\":\"def check_system_status():\\n return \\\"System status: OK\\\"\\n\\n# Test the diagnostics module\\nif __name__ == \\\"__main__\\\":\\n print(check_system_status())\"}}}", "actual": { "plan": "Add a diagnostics module and verify its test", "action": { "name": "add_file", "arguments": { "filename": "diagnostics.py", "content": "def check_system_status():\n return \"System status: OK\"\n\n# Test the diagnostics module\nif __name__ == \"__main__\":\n print(check_system_status())" } }, "result": null }, "family": "create_and_test", "expected": { "action": { "arguments": { "limit": 100, "path": "src" }, "name": "workspace.list" }, "plan": "List the package before creating a new module." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"name\":\"metadata\",\"path\":\"src\"},\"name\":\"workspace.create_module\"},\"plan\":\"Create a new module named 'metadata' in the 'src' directory.\"}", "actual": { "plan": "Create a new module named 'metadata' in the 'src' directory.", "action": { "name": "workspace.create_module", "arguments": { "name": "metadata", "path": "src" } }, "result": null }, "family": "create_and_test", "expected": { "action": { "arguments": { "content": "def metadata():\n return \"ok\"\n", "expected_sha256": null, "path": "src/metadata.py" }, "name": "workspace.write" }, "plan": "Create only the requested module." } }, { "valid": true, "semantic": true, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"message\":\"Fix issue 20100\",\"path\":\"src/issue_20100.py\",\"staged\":true},\"name\":\"git.commit\"},\"plan\":\"Commit the changes for issue 20100.\"}", "actual": { "plan": "Commit the changes for issue 20100.", "action": { "name": "git.commit", "arguments": { "message": "Fix issue 20100", "path": "src/issue_20100.py", "staged": true } }, "result": null }, "family": "git_to_change", "expected": { "action": { "arguments": { "expected_head": "d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845", "message": "Fix issue 20100", "paths": [ "src/issue_20100.py", "tests/test_issue_20100.py" ] }, "name": "git.commit" }, "plan": "Commit only the requested source and test files." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"name\":\"fix/issue-20101\",\"head\":\"d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3\",\"title\":\"Fix issue 20101\",\"description\":\"Fixes the issue 20101.\"},\"name\":\"gitlab.create_merge_request\"},\"plan\":\"Create a merge request on GitLab for the fix.\"}", "actual": { "plan": "Create a merge request on GitLab for the fix.", "action": { "name": "gitlab.create_merge_request", "arguments": { "name": "fix/issue-20101", "head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", "title": "Fix issue 20101", "description": "Fixes the issue 20101." } }, "result": null }, "family": "git_to_change", "expected": { "action": { "arguments": { "branch": "fix/issue-20101", "expected_head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", "remote": "origin" }, "name": "git.push" }, "plan": "Push the exact commit on the focused branch." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"branch\":\"fix/issue-20102\",\"token\":\"gitea_token\"},\"name\":\"gitea.create_pr\"},\"plan\":\"Create a pull request on Gitea for the focused branch.\"}", "actual": { "plan": "Create a pull request on Gitea for the focused branch.", "action": { "name": "gitea.create_pr", "arguments": { "branch": "fix/issue-20102", "token": "gitea_token" } }, "result": null }, "family": "git_to_change", "expected": { "action": { "arguments": { "base_branch": "main", "body": "Includes the minimal implementation and focused tests.", "head_branch": "fix/issue-20102", "title": "Fix issue 20102" }, "name": "provider.open_change" }, "plan": "Open the already-pushed branch as a Gitea change." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"The fix for issue 20103 has been prepared, committed, and opened on Forgejo. The pull request number is 20103.\"}", "actual": { "plan": null, "action": null, "result": "The fix for issue 20103 has been prepared, committed, and opened on Forgejo. The pull request number is 20103." }, "family": "git_to_change", "expected": { "result": "Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"Change 20020 has been approved with current-revision evidence.\"}", "actual": { "plan": null, "action": null, "result": "Change 20020 has been approved with current-revision evidence." }, "family": "review_approve", "expected": { "result": "Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Check the current-revision evidence for GitLab change 20021\",\"action\":{\"name\":\"get_gitlab_change_revision\",\"arguments\":{\"change_id\":20021}}}", "actual": { "plan": "Check the current-revision evidence for GitLab change 20021", "action": { "name": "get_gitlab_change_revision", "arguments": { "change_id": 20021 } }, "result": null }, "family": "review_approve", "expected": { "action": { "arguments": { "number": 20021 }, "name": "provider.get_change" }, "plan": "Fetch the change and bind review to its current head." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20022,\"sha\":\"8be1c0da0cf192d5742d40cb5e6a27ff82b4f9480a91572df36674bf6fb26a0a\",\"approval\":\"approved\"},\"name\":\"provider.review\"},\"plan\":\"Approve the change with current-revision evidence.\"}", "actual": { "plan": "Approve the change with current-revision evidence.", "action": { "name": "provider.review", "arguments": { "number": 20022, "sha": "8be1c0da0cf192d5742d40cb5e6a27ff82b4f9480a91572df36674bf6fb26a0a", "approval": "approved" } }, "result": null }, "family": "review_approve", "expected": { "action": { "arguments": { "number": 20022 }, "name": "provider.get_diff" }, "plan": "Inspect the diff for that exact revision." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20023,\"head_sha\":\"53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381\"},\"name\":\"provider.approve\"},\"plan\":\"Approve the change with the current-revision evidence.\"}", "actual": { "plan": "Approve the change with the current-revision evidence.", "action": { "name": "provider.approve", "arguments": { "number": 20023, "head_sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" } }, "result": null }, "family": "review_approve", "expected": { "action": { "arguments": { "sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" }, "name": "provider.get_checks" }, "plan": "Check CI for the same commit SHA." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20070,\"head_sha\":\"76c464eb4bfccce7c4b6d2fe3323268568f14d4e42334c495a47b617e439b035\"},\"name\":\"provider.get_check\"},\"plan\":\"Verify that all checks for the change have passed.\"}", "actual": { "plan": "Verify that all checks for the change have passed.", "action": { "name": "provider.get_check", "arguments": { "number": 20070, "head_sha": "76c464eb4bfccce7c4b6d2fe3323268568f14d4e42334c495a47b617e439b035" } }, "result": null }, "family": "review_block", "expected": { "action": { "arguments": { "number": 20070 }, "name": "provider.get_diff" }, "plan": "Inspect the exact diff before reaching a conclusion." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20071,\"head_sha\":\"9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921\"},\"name\":\"provider.get_check\"},\"plan\":\"Verify if all checks have passed for the change.\"}", "actual": { "plan": "Verify if all checks have passed for the change.", "action": { "name": "provider.get_check", "arguments": { "number": 20071, "head_sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" } }, "result": null }, "family": "review_block", "expected": { "action": { "arguments": { "sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" }, "name": "provider.get_checks" }, "plan": "Read checks for the reviewed SHA." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"change_20072_not_approved\"}", "actual": { "plan": null, "action": null, "result": "change_20072_not_approved" }, "family": "review_block", "expected": { "result": "Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Check if change 20073 has passed all necessary reviews.\",\"action\":{\"name\":\"get_change_reviews\",\"arguments\":{\"change_id\":20073}}}", "actual": { "plan": "Check if change 20073 has passed all necessary reviews.", "action": { "name": "get_change_reviews", "arguments": { "change_id": 20073 } }, "result": null }, "family": "review_block", "expected": { "action": { "arguments": { "number": 20073 }, "name": "provider.get_change" }, "plan": "Fetch the current change revision." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20150,\"head_sha\":\"1849708cc7094b7bc1b97ddc52582e7223c485948abaed97ce0c76b3f4b82d39\"},\"name\":\"provider.rebase\"},\"plan\":\"Rebase the change with the exact revision.\"}", "actual": { "plan": "Rebase the change with the exact revision.", "action": { "name": "provider.rebase", "arguments": { "number": 20150, "head_sha": "1849708cc7094b7bc1b97ddc52582e7223c485948abaed97ce0c76b3f4b82d39" } }, "result": null }, "family": "review_merge", "expected": { "action": { "arguments": { "number": 20150 }, "name": "provider.get_diff" }, "plan": "Review the diff at the bound revision." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"head_sha\":\"f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7\",\"number\":20151},\"name\":\"provider.merge\"},\"plan\":\"Merge the change with squash after verification.\"}", "actual": { "plan": "Merge the change with squash after verification.", "action": { "name": "provider.merge", "arguments": { "head_sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7", "number": 20151 } }, "result": null }, "family": "review_merge", "expected": { "action": { "arguments": { "sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7" }, "name": "provider.get_checks" }, "plan": "Verify required checks on the same SHA." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"sha\":\"d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d\",\"number\":20152},\"name\":\"provider.merge\"},\"plan\":\"Merge the change after verifying its exact revision.\"}", "actual": { "plan": "Merge the change after verifying its exact revision.", "action": { "name": "provider.merge", "arguments": { "sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", "number": 20152 } }, "result": null }, "family": "review_merge", "expected": { "action": { "arguments": { "expected_sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", "method": "merge", "number": 20152 }, "name": "provider.merge_change" }, "plan": "Merge only the reviewed, passing revision." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\": \"Change 20153 has been successfully merged with a rebase after verifying its exact revision.\"}", "actual": { "plan": null, "action": null, "result": "Change 20153 has been successfully merged with a rebase after verifying its exact revision." }, "family": "review_merge", "expected": { "result": "Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"grep\",\"\" ,\"tests/test_parser.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Locate the test function causing the failure.\"}", "actual": { "plan": "Locate the test function causing the failure.", "action": { "name": "shell.run", "arguments": { "argv": [ "grep", "", "tests/test_parser.py" ], "cwd": "." } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "path": "src", "query": "def handle" }, "name": "workspace.search" }, "plan": "Locate the failing implementation named in the test." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/router.py\",\"line\":12},\"name\":\"workspace.open\"},\"plan\":\"Inspect the implementation to find the issue.\"}", "actual": { "plan": "Inspect the implementation to find the issue.", "action": { "name": "workspace.open", "arguments": { "path": "src/router.py", "line": 12 } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "path": "src/router.py" }, "name": "workspace.read" }, "plan": "Read the implementation before changing it." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/worker.py\",\"query\":\"def handle\"},\"name\":\"workspace.search\"},\"plan\":\"Find the function that returns values[0] and is called in the test.\"}", "actual": { "plan": "Find the function that returns values[0] and is called in the test.", "action": { "name": "workspace.search", "arguments": { "path": "src/worker.py", "query": "def handle" } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e", "new_text": "return values[0] if values else None", "old_text": "return values[0]", "path": "src/worker.py" }, "name": "workspace.replace" }, "plan": "Guard the observed empty-input access." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_client.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Verify the fix by re-running the test.\"}", "actual": { "plan": "Verify the fix by re-running the test.", "action": { "name": "shell.run", "arguments": { "argv": [ "pytest", "-q", "tests/test_client.py" ], "cwd": "." } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "argv": [ "pytest", "-q", "tests/test_client.py" ], "cwd": "." }, "name": "shell.run" }, "plan": "Rerun the previously failing test." } }, { "valid": true, "semantic": true, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519\",\"new_text\":\"MODE='new'\\nTIMEOUT=30\",\"old_text\":\"MODE='old'\\nTIMEOUT=30\",\"path\":\"src/config_1.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Replace the content with the new value while preserving the existing content.\"}", "actual": { "plan": "Replace the content with the new value while preserving the existing content.", "action": { "name": "workspace.replace", "arguments": { "expected_occurrences": 1, "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", "new_text": "MODE='new'\nTIMEOUT=30", "old_text": "MODE='old'\nTIMEOUT=30", "path": "src/config_1.py" } }, "result": null }, "family": "stale_write_recovery", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", "new_text": "MODE='new'", "old_text": "MODE='old'", "path": "src/config_1.py" }, "name": "workspace.replace" }, "plan": "Preserve the concurrent addition and replace only the requested value." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config_2.py\"},\"name\":\"workspace.read\"},\"plan\":\"Verify the change and capture the new hash for confirmation.\"}", "actual": { "plan": "Verify the change and capture the new hash for confirmation.", "action": { "name": "workspace.read", "arguments": { "path": "src/config_2.py" } }, "result": null }, "family": "stale_write_recovery", "expected": { "result": "Applied the requested rename while preserving the concurrent timeout change." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Check for concurrent changes\",\"action\":{\"name\":\"check_concurrent_changes\",\"arguments\":{}}}", "actual": { "plan": "Check for concurrent changes", "action": { "name": "check_concurrent_changes", "arguments": {} }, "result": null }, "family": "stale_write_recovery", "expected": { "action": { "arguments": { "path": "src/config_3.py" }, "name": "workspace.read" }, "plan": "Read the current file and capture its hash." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config_4.py\",\"sha256\":\"c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9\"},\"name\":\"workspace.apply\"},\"plan\":\"Apply the current file without overwriting concurrent changes.\"}", "actual": { "plan": "Apply the current file without overwriting concurrent changes.", "action": { "name": "workspace.apply", "arguments": { "path": "src/config_4.py", "sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9" } }, "result": null }, "family": "stale_write_recovery", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9", "new_text": "MODE='new'", "old_text": "MODE='old'", "path": "src/config_4.py" }, "name": "workspace.replace" }, "plan": "Attempt the exact replacement against the observed hash." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"go\",\"test\",\"./...\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", "actual": { "plan": "Attempt the requested verification through the policy-checked shell tool.", "action": { "name": "shell.run", "arguments": { "argv": [ "go", "test", "./..." ], "cwd": "." } }, "result": null }, "family": "permission_recovery", "expected": { "result": "Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Run the project verification command\",\"action\":{\"name\":\"run_command\",\"arguments\":{\"command\":\"npm test\"}}}", "actual": { "plan": "Run the project verification command", "action": { "name": "run_command", "arguments": { "command": "npm test" } }, "result": null }, "family": "permission_recovery", "expected": { "action": { "arguments": { "argv": [ "npm", "test" ], "cwd": "." }, "name": "shell.run" }, "plan": "Attempt the requested verification through the policy-checked shell tool." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"cargo\",\"test\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", "actual": { "plan": "Attempt the requested verification through the policy-checked shell tool.", "action": { "name": "shell.run", "arguments": { "argv": [ "cargo", "test" ], "cwd": "." } }, "result": null }, "family": "permission_recovery", "expected": { "result": "Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." } }, { "valid": true, "semantic": false, "arguments_exact": false, "error": null, "raw": "{\"plan\":\"Run the project verification command\",\"action\":{\"name\":\"run_command\",\"arguments\":{\"command\":\"go test ./...\"}}}", "actual": { "plan": "Run the project verification command", "action": { "name": "run_command", "arguments": { "command": "go test ./..." } }, "result": null }, "family": "permission_recovery", "expected": { "action": { "arguments": { "argv": [ "go", "test", "./..." ], "cwd": "." }, "name": "shell.run" }, "plan": "Attempt the requested verification through the policy-checked shell tool." } } ] }, "adapter_evaluation": { "count": 40, "valid_rate": 1.0, "semantic_rate": 1.0, "arguments_exact_rate": 0.975, "cases": [ { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src\",\"query\":\"def parser\"},\"name\":\"workspace.search\"},\"plan\":\"Search narrowly before reading files.\"}", "actual": { "plan": "Search narrowly before reading files.", "action": { "name": "workspace.search", "arguments": { "path": "src", "query": "def parser" } }, "result": null }, "family": "inspect", "expected": { "action": { "arguments": { "path": "src", "query": "def parser" }, "name": "workspace.search" }, "plan": "Search narrowly before reading files." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"end_line\":90,\"path\":\"src/router.py\",\"start_line\":1},\"name\":\"workspace.read\"},\"plan\":\"Read the matching implementation.\"}", "actual": { "plan": "Read the matching implementation.", "action": { "name": "workspace.read", "arguments": { "end_line": 90, "path": "src/router.py", "start_line": 1 } }, "result": null }, "family": "inspect", "expected": { "action": { "arguments": { "end_line": 90, "path": "src/router.py", "start_line": 1 }, "name": "workspace.read" }, "plan": "Read the matching implementation." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation.\"}", "actual": { "plan": null, "action": null, "result": "Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation." }, "family": "inspect", "expected": { "result": "Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src\",\"query\":\"def scheduler\"},\"name\":\"workspace.search\"},\"plan\":\"Search narrowly before reading files.\"}", "actual": { "plan": "Search narrowly before reading files.", "action": { "name": "workspace.search", "arguments": { "path": "src", "query": "def scheduler" } }, "result": null }, "family": "inspect", "expected": { "action": { "arguments": { "path": "src", "query": "def scheduler" }, "name": "workspace.search" }, "plan": "Search narrowly before reading files." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779\",\"new_text\":\"def handle(items):\\n if not items:\\n return None\\n return items[0]\",\"old_text\":\"def handle(items):\\n return items[0]\",\"path\":\"src/parser.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Apply the smallest concurrency-safe replacement.\"}", "actual": { "plan": "Apply the smallest concurrency-safe replacement.", "action": { "name": "workspace.replace", "arguments": { "expected_occurrences": 1, "expected_sha256": "58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779", "new_text": "def handle(items):\n if not items:\n return None\n return items[0]", "old_text": "def handle(items):\n return items[0]", "path": "src/parser.py" } }, "result": null }, "family": "edit_and_test", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779", "new_text": "def handle(items):\n if not items:\n return None\n return items[0]", "old_text": "def handle(items):\n return items[0]", "path": "src/parser.py" }, "name": "workspace.replace" }, "plan": "Apply the smallest concurrency-safe replacement." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_config.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Run only the relevant test module first.\"}", "actual": { "plan": "Run only the relevant test module first.", "action": { "name": "shell.run", "arguments": { "argv": [ "pytest", "-q", "tests/test_config.py" ], "cwd": "." } }, "result": null }, "family": "edit_and_test", "expected": { "action": { "arguments": { "argv": [ "pytest", "-q", "tests/test_config.py" ], "cwd": "." }, "name": "shell.run" }, "plan": "Run only the relevant test module first." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Fixed empty-input handling in `src/client.py`; all 6 focused tests pass.\"}", "actual": { "plan": null, "action": null, "result": "Fixed empty-input handling in `src/client.py`; all 6 focused tests pass." }, "family": "edit_and_test", "expected": { "result": "Fixed empty-input handling in `src/client.py`; all 6 focused tests pass." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/cache.py\"},\"name\":\"workspace.read\"},\"plan\":\"Read the current implementation and capture its content hash.\"}", "actual": { "plan": "Read the current implementation and capture its content hash.", "action": { "name": "workspace.read", "arguments": { "path": "src/cache.py" } }, "result": null }, "family": "edit_and_test", "expected": { "action": { "arguments": { "path": "src/cache.py" }, "name": "workspace.read" }, "plan": "Read the current implementation and capture its content hash." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_health.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Verify the new behavior with its focused test.\"}", "actual": { "plan": "Verify the new behavior with its focused test.", "action": { "name": "shell.run", "arguments": { "argv": [ "pytest", "-q", "tests/test_health.py" ], "cwd": "." } }, "result": null }, "family": "create_and_test", "expected": { "action": { "arguments": { "argv": [ "pytest", "-q", "tests/test_health.py" ], "cwd": "." }, "name": "shell.run" }, "plan": "Verify the new behavior with its focused test." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Added `src/version.py` and verified its 2 focused tests pass.\"}", "actual": { "plan": null, "action": null, "result": "Added `src/version.py` and verified its 2 focused tests pass." }, "family": "create_and_test", "expected": { "result": "Added `src/version.py` and verified its 2 focused tests pass." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"limit\":100,\"path\":\"src\"},\"name\":\"workspace.list\"},\"plan\":\"List the package before creating a new module.\"}", "actual": { "plan": "List the package before creating a new module.", "action": { "name": "workspace.list", "arguments": { "limit": 100, "path": "src" } }, "result": null }, "family": "create_and_test", "expected": { "action": { "arguments": { "limit": 100, "path": "src" }, "name": "workspace.list" }, "plan": "List the package before creating a new module." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"content\":\"def metadata():\\n return \\\"ok\\\"\\n\",\"expected_sha256\":null,\"path\":\"src/metadata.py\"},\"name\":\"workspace.write\"},\"plan\":\"Create only the requested module.\"}", "actual": { "plan": "Create only the requested module.", "action": { "name": "workspace.write", "arguments": { "content": "def metadata():\n return \"ok\"\n", "expected_sha256": null, "path": "src/metadata.py" } }, "result": null }, "family": "create_and_test", "expected": { "action": { "arguments": { "content": "def metadata():\n return \"ok\"\n", "expected_sha256": null, "path": "src/metadata.py" }, "name": "workspace.write" }, "plan": "Create only the requested module." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_head\":\"d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845\",\"message\":\"Fix issue 20100\",\"paths\":[\"src/issue_20100.py\",\"tests/test_issue_20100.py\"]},\"name\":\"git.commit\"},\"plan\":\"Commit only the requested source and test files.\"}", "actual": { "plan": "Commit only the requested source and test files.", "action": { "name": "git.commit", "arguments": { "expected_head": "d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845", "message": "Fix issue 20100", "paths": [ "src/issue_20100.py", "tests/test_issue_20100.py" ] } }, "result": null }, "family": "git_to_change", "expected": { "action": { "arguments": { "expected_head": "d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845", "message": "Fix issue 20100", "paths": [ "src/issue_20100.py", "tests/test_issue_20100.py" ] }, "name": "git.commit" }, "plan": "Commit only the requested source and test files." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"branch\":\"fix/issue-20101\",\"expected_head\":\"d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3\",\"remote\":\"origin\"},\"name\":\"git.push\"},\"plan\":\"Push the exact commit on the focused branch.\"}", "actual": { "plan": "Push the exact commit on the focused branch.", "action": { "name": "git.push", "arguments": { "branch": "fix/issue-20101", "expected_head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", "remote": "origin" } }, "result": null }, "family": "git_to_change", "expected": { "action": { "arguments": { "branch": "fix/issue-20101", "expected_head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", "remote": "origin" }, "name": "git.push" }, "plan": "Push the exact commit on the focused branch." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"base_branch\":\"main\",\"body\":\"Includes the minimal implementation and focused tests.\",\"head_branch\":\"fix/issue-20102\",\"title\":\"Fix issue 20102\"},\"name\":\"provider.open_change\"},\"plan\":\"Open the already-pushed branch as a Gitea change.\"}", "actual": { "plan": "Open the already-pushed branch as a Gitea change.", "action": { "name": "provider.open_change", "arguments": { "base_branch": "main", "body": "Includes the minimal implementation and focused tests.", "head_branch": "fix/issue-20102", "title": "Fix issue 20102" } }, "result": null }, "family": "git_to_change", "expected": { "action": { "arguments": { "base_branch": "main", "body": "Includes the minimal implementation and focused tests.", "head_branch": "fix/issue-20102", "title": "Fix issue 20102" }, "name": "provider.open_change" }, "plan": "Open the already-pushed branch as a Gitea change." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`.\"}", "actual": { "plan": null, "action": null, "result": "Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`." }, "family": "git_to_change", "expected": { "result": "Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`.\"}", "actual": { "plan": null, "action": null, "result": "Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`." }, "family": "review_approve", "expected": { "result": "Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20021},\"name\":\"provider.get_change\"},\"plan\":\"Fetch the change and bind review to its current head.\"}", "actual": { "plan": "Fetch the change and bind review to its current head.", "action": { "name": "provider.get_change", "arguments": { "number": 20021 } }, "result": null }, "family": "review_approve", "expected": { "action": { "arguments": { "number": 20021 }, "name": "provider.get_change" }, "plan": "Fetch the change and bind review to its current head." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20022},\"name\":\"provider.get_diff\"},\"plan\":\"Inspect the diff for that exact revision.\"}", "actual": { "plan": "Inspect the diff for that exact revision.", "action": { "name": "provider.get_diff", "arguments": { "number": 20022 } }, "result": null }, "family": "review_approve", "expected": { "action": { "arguments": { "number": 20022 }, "name": "provider.get_diff" }, "plan": "Inspect the diff for that exact revision." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"sha\":\"53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381\"},\"name\":\"provider.get_checks\"},\"plan\":\"Check CI for the same commit SHA.\"}", "actual": { "plan": "Check CI for the same commit SHA.", "action": { "name": "provider.get_checks", "arguments": { "sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" } }, "result": null }, "family": "review_approve", "expected": { "action": { "arguments": { "sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" }, "name": "provider.get_checks" }, "plan": "Check CI for the same commit SHA." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20070},\"name\":\"provider.get_diff\"},\"plan\":\"Inspect the exact diff before reaching a conclusion.\"}", "actual": { "plan": "Inspect the exact diff before reaching a conclusion.", "action": { "name": "provider.get_diff", "arguments": { "number": 20070 } }, "result": null }, "family": "review_block", "expected": { "action": { "arguments": { "number": 20070 }, "name": "provider.get_diff" }, "plan": "Inspect the exact diff before reaching a conclusion." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"sha\":\"9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921\"},\"name\":\"provider.get_checks\"},\"plan\":\"Read checks for the reviewed SHA.\"}", "actual": { "plan": "Read checks for the reviewed SHA.", "action": { "name": "provider.get_checks", "arguments": { "sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" } }, "result": null }, "family": "review_block", "expected": { "action": { "arguments": { "sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" }, "name": "provider.get_checks" }, "plan": "Read checks for the reviewed SHA." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`.\"}", "actual": { "plan": null, "action": null, "result": "Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`." }, "family": "review_block", "expected": { "result": "Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20073},\"name\":\"provider.get_change\"},\"plan\":\"Fetch the current change revision.\"}", "actual": { "plan": "Fetch the current change revision.", "action": { "name": "provider.get_change", "arguments": { "number": 20073 } }, "result": null }, "family": "review_block", "expected": { "action": { "arguments": { "number": 20073 }, "name": "provider.get_change" }, "plan": "Fetch the current change revision." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"number\":20150},\"name\":\"provider.get_diff\"},\"plan\":\"Review the diff at the bound revision.\"}", "actual": { "plan": "Review the diff at the bound revision.", "action": { "name": "provider.get_diff", "arguments": { "number": 20150 } }, "result": null }, "family": "review_merge", "expected": { "action": { "arguments": { "number": 20150 }, "name": "provider.get_diff" }, "plan": "Review the diff at the bound revision." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"sha\":\"f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7\"},\"name\":\"provider.get_checks\"},\"plan\":\"Verify required checks on the same SHA.\"}", "actual": { "plan": "Verify required checks on the same SHA.", "action": { "name": "provider.get_checks", "arguments": { "sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7" } }, "result": null }, "family": "review_merge", "expected": { "action": { "arguments": { "sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7" }, "name": "provider.get_checks" }, "plan": "Verify required checks on the same SHA." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_sha\":\"d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d\",\"method\":\"merge\",\"number\":20152},\"name\":\"provider.merge_change\"},\"plan\":\"Merge only the reviewed, passing revision.\"}", "actual": { "plan": "Merge only the reviewed, passing revision.", "action": { "name": "provider.merge_change", "arguments": { "expected_sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", "method": "merge", "number": 20152 } }, "result": null }, "family": "review_merge", "expected": { "action": { "arguments": { "expected_sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", "method": "merge", "number": 20152 }, "name": "provider.merge_change" }, "plan": "Merge only the reviewed, passing revision." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase.\"}", "actual": { "plan": null, "action": null, "result": "Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase." }, "family": "review_merge", "expected": { "result": "Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src\",\"query\":\"def handle\"},\"name\":\"workspace.search\"},\"plan\":\"Locate the failing implementation named in the test.\"}", "actual": { "plan": "Locate the failing implementation named in the test.", "action": { "name": "workspace.search", "arguments": { "path": "src", "query": "def handle" } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "path": "src", "query": "def handle" }, "name": "workspace.search" }, "plan": "Locate the failing implementation named in the test." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/router.py\"},\"name\":\"workspace.read\"},\"plan\":\"Read the implementation before changing it.\"}", "actual": { "plan": "Read the implementation before changing it.", "action": { "name": "workspace.read", "arguments": { "path": "src/router.py" } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "path": "src/router.py" }, "name": "workspace.read" }, "plan": "Read the implementation before changing it." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e\",\"new_text\":\"return values[0] if values else None\",\"old_text\":\"return values[0]\",\"path\":\"src/worker.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Guard the observed empty-input access.\"}", "actual": { "plan": "Guard the observed empty-input access.", "action": { "name": "workspace.replace", "arguments": { "expected_occurrences": 1, "expected_sha256": "76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e", "new_text": "return values[0] if values else None", "old_text": "return values[0]", "path": "src/worker.py" } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e", "new_text": "return values[0] if values else None", "old_text": "return values[0]", "path": "src/worker.py" }, "name": "workspace.replace" }, "plan": "Guard the observed empty-input access." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_client.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Rerun the previously failing test.\"}", "actual": { "plan": "Rerun the previously failing test.", "action": { "name": "shell.run", "arguments": { "argv": [ "pytest", "-q", "tests/test_client.py" ], "cwd": "." } }, "result": null }, "family": "test_recovery", "expected": { "action": { "arguments": { "argv": [ "pytest", "-q", "tests/test_client.py" ], "cwd": "." }, "name": "shell.run" }, "plan": "Rerun the previously failing test." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519\",\"new_text\":\"MODE='new'\",\"old_text\":\"MODE='old'\",\"path\":\"src/config_1.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Preserve the concurrent addition and replace only the requested value.\"}", "actual": { "plan": "Preserve the concurrent addition and replace only the requested value.", "action": { "name": "workspace.replace", "arguments": { "expected_occurrences": 1, "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", "new_text": "MODE='new'", "old_text": "MODE='old'", "path": "src/config_1.py" } }, "result": null }, "family": "stale_write_recovery", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", "new_text": "MODE='new'", "old_text": "MODE='old'", "path": "src/config_1.py" }, "name": "workspace.replace" }, "plan": "Preserve the concurrent addition and replace only the requested value." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Applied the requested rename while preserving the concurrent timeout change.\"}", "actual": { "plan": null, "action": null, "result": "Applied the requested rename while preserving the concurrent timeout change." }, "family": "stale_write_recovery", "expected": { "result": "Applied the requested rename while preserving the concurrent timeout change." } }, { "valid": true, "semantic": true, "arguments_exact": false, "error": null, "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config_1.py\"},\"name\":\"workspace.read\"},\"plan\":\"Read the current file and capture its hash.\"}", "actual": { "plan": "Read the current file and capture its hash.", "action": { "name": "workspace.read", "arguments": { "path": "src/config_1.py" } }, "result": null }, "family": "stale_write_recovery", "expected": { "action": { "arguments": { "path": "src/config_3.py" }, "name": "workspace.read" }, "plan": "Read the current file and capture its hash." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9\",\"new_text\":\"MODE='new'\",\"old_text\":\"MODE='old'\",\"path\":\"src/config_4.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Attempt the exact replacement against the observed hash.\"}", "actual": { "plan": "Attempt the exact replacement against the observed hash.", "action": { "name": "workspace.replace", "arguments": { "expected_occurrences": 1, "expected_sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9", "new_text": "MODE='new'", "old_text": "MODE='old'", "path": "src/config_4.py" } }, "result": null }, "family": "stale_write_recovery", "expected": { "action": { "arguments": { "expected_occurrences": 1, "expected_sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9", "new_text": "MODE='new'", "old_text": "MODE='old'", "path": "src/config_4.py" }, "name": "workspace.replace" }, "plan": "Attempt the exact replacement against the observed hash." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command.\"}", "actual": { "plan": null, "action": null, "result": "Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." }, "family": "permission_recovery", "expected": { "result": "Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"npm\",\"test\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", "actual": { "plan": "Attempt the requested verification through the policy-checked shell tool.", "action": { "name": "shell.run", "arguments": { "argv": [ "npm", "test" ], "cwd": "." } }, "result": null }, "family": "permission_recovery", "expected": { "action": { "arguments": { "argv": [ "npm", "test" ], "cwd": "." }, "name": "shell.run" }, "plan": "Attempt the requested verification through the policy-checked shell tool." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"result\":\"Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command.\"}", "actual": { "plan": null, "action": null, "result": "Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." }, "family": "permission_recovery", "expected": { "result": "Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." } }, { "valid": true, "semantic": true, "arguments_exact": true, "error": null, "raw": "{\"action\":{\"arguments\":{\"argv\":[\"go\",\"test\",\"./...\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", "actual": { "plan": "Attempt the requested verification through the policy-checked shell tool.", "action": { "name": "shell.run", "arguments": { "argv": [ "go", "test", "./..." ], "cwd": "." } }, "result": null }, "family": "permission_recovery", "expected": { "action": { "arguments": { "argv": [ "go", "test", "./..." ], "cwd": "." }, "name": "shell.run" }, "plan": "Attempt the requested verification through the policy-checked shell tool." } } ] }, "publication_gate": { "valid_rate_at_least_0_90": true, "semantic_rate_at_least_0_80": true, "not_materially_worse_than_base": true }, "versions": { "python": "3.13.15", "torch": "2.11.0+cu128", "transformers": "4.57.6", "peft": "0.20.0", "accelerate": "1.14.0", "bitsandbytes": "0.50.1" }, "wheel_sha256": "57a015816506665d75eaaedb557b4f53883258e69094723ac145340097c857c6", "data_archive_sha256": "2bb44ab5a80d6e3bd455d490403464d083ca6a5d5fe5b4a9b4fc3769d0bd7dc5", "elapsed_seconds": 1526.45 }