| { |
| "status": "passed", |
| "release": "v0.0.1-alpha", |
| "base_model": "Qwen/Qwen3-4B", |
| "base_model_revision": "1cfa9a7208912126459214e8b04321603b3df60c", |
| "gpu": "Tesla T4", |
| "dataset": { |
| "name": "nimora-agent-alpha-v0.0.1", |
| "license": "Apache-2.0", |
| "provenance": "deterministic synthetic templates authored for Nimora", |
| "supervision": "prefix_to_next_decision", |
| "limitations": [ |
| "Template-generated data does not establish real-world coding quality.", |
| "No private repositories, user logs, credentials, or scraped code are included.", |
| "Provider names appear in tasks, but actions use provider-neutral runtime tools." |
| ], |
| "families": [ |
| "create_and_test", |
| "edit_and_test", |
| "git_to_change", |
| "inspect", |
| "permission_recovery", |
| "review_approve", |
| "review_block", |
| "review_merge", |
| "stale_write_recovery", |
| "test_recovery" |
| ], |
| "counts": { |
| "train": 400, |
| "validation": 40, |
| "eval": 40 |
| }, |
| "split_index_ranges": { |
| "train": [ |
| 0, |
| 39 |
| ], |
| "validation": [ |
| 10000, |
| 10003 |
| ], |
| "eval": [ |
| 20000, |
| 20003 |
| ] |
| } |
| }, |
| "training": { |
| "train": { |
| "train_runtime": 805.2917, |
| "train_samples_per_second": 0.497, |
| "train_steps_per_second": 0.497, |
| "total_flos": 3575252515292160.0, |
| "train_loss": 0.3089988502621418, |
| "epoch": 1.0 |
| }, |
| "evaluation": { |
| "eval_loss": 0.004357926081866026, |
| "eval_runtime": 23.0059, |
| "eval_samples_per_second": 1.739, |
| "eval_steps_per_second": 1.739, |
| "epoch": 1.0 |
| }, |
| "adapter_dir": "/content/nimora-agent-alpha/run/final-adapter" |
| }, |
| "base_evaluation": { |
| "count": 40, |
| "valid_rate": 1.0, |
| "semantic_rate": 0.275, |
| "arguments_exact_rate": 0.2, |
| "cases": [ |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Find the parser implementation file\",\"action\":{\"name\":\"find_file\",\"arguments\":{\"filename\":\"parser.py\"}}}", |
| "actual": { |
| "plan": "Find the parser implementation file", |
| "action": { |
| "name": "find_file", |
| "arguments": { |
| "filename": "parser.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "inspect", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src", |
| "query": "def parser" |
| }, |
| "name": "workspace.search" |
| }, |
| "plan": "Search narrowly before reading files." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/router.py\",\"query\":\"def router\"},\"name\":\"workspace.read\"},\"plan\":\"Read the file to find the relevant entry point.\"}", |
| "actual": { |
| "plan": "Read the file to find the relevant entry point.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/router.py", |
| "query": "def router" |
| } |
| }, |
| "result": null |
| }, |
| "family": "inspect", |
| "expected": { |
| "action": { |
| "arguments": { |
| "end_line": 90, |
| "path": "src/router.py", |
| "start_line": 1 |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the matching implementation." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"The serializer implementation is located in src/serializer.py. The relevant entry point is the serializer function, which returns the normalized value of the input.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "The serializer implementation is located in src/serializer.py. The relevant entry point is the serializer function, which returns the normalized value of the input." |
| }, |
| "family": "inspect", |
| "expected": { |
| "result": "Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Find the scheduler implementation file\",\"action\":{\"name\":\"find_file\",\"arguments\":{\"filename\":\"scheduler.py\"}}}", |
| "actual": { |
| "plan": "Find the scheduler implementation file", |
| "action": { |
| "name": "find_file", |
| "arguments": { |
| "filename": "scheduler.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "inspect", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src", |
| "query": "def scheduler" |
| }, |
| "name": "workspace.search" |
| }, |
| "plan": "Search narrowly before reading files." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/parser.py\",\"line_number\":1,\"content\":\"def handle(items):\\n return items[0]\\n\"},\"name\":\"workspace.write\"},\"plan\":\"Modify the parser to handle empty input by returning an empty list.\"}", |
| "actual": { |
| "plan": "Modify the parser to handle empty input by returning an empty list.", |
| "action": { |
| "name": "workspace.write", |
| "arguments": { |
| "path": "src/parser.py", |
| "line_number": 1, |
| "content": "def handle(items):\n return items[0]\n" |
| } |
| }, |
| "result": null |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779", |
| "new_text": "def handle(items):\n if not items:\n return None\n return items[0]", |
| "old_text": "def handle(items):\n return items[0]", |
| "path": "src/parser.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Apply the smallest concurrency-safe replacement." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config.py\",\"sha256\":\"e71ba961d8f5e8191e177c3a6280684464be7b4cba26445f75574f28aceb5347\"},\"name\":\"workspace.read\"},\"plan\":\"Verify the replacement and capture the new content hash.\"}", |
| "actual": { |
| "plan": "Verify the replacement and capture the new content hash.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/config.py", |
| "sha256": "e71ba961d8f5e8191e177c3a6280684464be7b4cba26445f75574f28aceb5347" |
| } |
| }, |
| "result": null |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_config.py" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Run only the relevant test module first." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"The empty-input regression in the client has been fixed and the focused tests have passed successfully.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "The empty-input regression in the client has been fixed and the focused tests have passed successfully." |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "result": "Fixed empty-input handling in `src/client.py`; all 6 focused tests pass." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Check for the presence of the empty-input regression in the cache module\",\"action\":{\"name\":\"check_regression\",\"arguments\":{\"module\":\"cache\",\"regression\":\"empty-input\"}}}", |
| "actual": { |
| "plan": "Check for the presence of the empty-input regression in the cache module", |
| "action": { |
| "name": "check_regression", |
| "arguments": { |
| "module": "cache", |
| "regression": "empty-input" |
| } |
| }, |
| "result": null |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src/cache.py" |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the current implementation and capture its content hash." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/health.py\",\"source\":\"def health():\\n return \\\"ok\\\"\\n\"},\"name\":\"workspace.read\"},\"plan\":\"Verify the module's content.\"}", |
| "actual": { |
| "plan": "Verify the module's content.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/health.py", |
| "source": "def health():\n return \"ok\"\n" |
| } |
| }, |
| "result": null |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_health.py" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Verify the new behavior with its focused test." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"The version module was successfully created and its focused test passed.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "The version module was successfully created and its focused test passed." |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "result": "Added `src/version.py` and verified its 2 focused tests pass." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Add a diagnostics module and verify its test\",\"action\":{\"name\":\"add_file\",\"arguments\":{\"filename\":\"diagnostics.py\",\"content\":\"def check_system_status():\\n return \\\"System status: OK\\\"\\n\\n# Test the diagnostics module\\nif __name__ == \\\"__main__\\\":\\n print(check_system_status())\"}}}", |
| "actual": { |
| "plan": "Add a diagnostics module and verify its test", |
| "action": { |
| "name": "add_file", |
| "arguments": { |
| "filename": "diagnostics.py", |
| "content": "def check_system_status():\n return \"System status: OK\"\n\n# Test the diagnostics module\nif __name__ == \"__main__\":\n print(check_system_status())" |
| } |
| }, |
| "result": null |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "limit": 100, |
| "path": "src" |
| }, |
| "name": "workspace.list" |
| }, |
| "plan": "List the package before creating a new module." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"name\":\"metadata\",\"path\":\"src\"},\"name\":\"workspace.create_module\"},\"plan\":\"Create a new module named 'metadata' in the 'src' directory.\"}", |
| "actual": { |
| "plan": "Create a new module named 'metadata' in the 'src' directory.", |
| "action": { |
| "name": "workspace.create_module", |
| "arguments": { |
| "name": "metadata", |
| "path": "src" |
| } |
| }, |
| "result": null |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "content": "def metadata():\n return \"ok\"\n", |
| "expected_sha256": null, |
| "path": "src/metadata.py" |
| }, |
| "name": "workspace.write" |
| }, |
| "plan": "Create only the requested module." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"message\":\"Fix issue 20100\",\"path\":\"src/issue_20100.py\",\"staged\":true},\"name\":\"git.commit\"},\"plan\":\"Commit the changes for issue 20100.\"}", |
| "actual": { |
| "plan": "Commit the changes for issue 20100.", |
| "action": { |
| "name": "git.commit", |
| "arguments": { |
| "message": "Fix issue 20100", |
| "path": "src/issue_20100.py", |
| "staged": true |
| } |
| }, |
| "result": null |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_head": "d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845", |
| "message": "Fix issue 20100", |
| "paths": [ |
| "src/issue_20100.py", |
| "tests/test_issue_20100.py" |
| ] |
| }, |
| "name": "git.commit" |
| }, |
| "plan": "Commit only the requested source and test files." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"name\":\"fix/issue-20101\",\"head\":\"d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3\",\"title\":\"Fix issue 20101\",\"description\":\"Fixes the issue 20101.\"},\"name\":\"gitlab.create_merge_request\"},\"plan\":\"Create a merge request on GitLab for the fix.\"}", |
| "actual": { |
| "plan": "Create a merge request on GitLab for the fix.", |
| "action": { |
| "name": "gitlab.create_merge_request", |
| "arguments": { |
| "name": "fix/issue-20101", |
| "head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", |
| "title": "Fix issue 20101", |
| "description": "Fixes the issue 20101." |
| } |
| }, |
| "result": null |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "action": { |
| "arguments": { |
| "branch": "fix/issue-20101", |
| "expected_head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", |
| "remote": "origin" |
| }, |
| "name": "git.push" |
| }, |
| "plan": "Push the exact commit on the focused branch." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"branch\":\"fix/issue-20102\",\"token\":\"gitea_token\"},\"name\":\"gitea.create_pr\"},\"plan\":\"Create a pull request on Gitea for the focused branch.\"}", |
| "actual": { |
| "plan": "Create a pull request on Gitea for the focused branch.", |
| "action": { |
| "name": "gitea.create_pr", |
| "arguments": { |
| "branch": "fix/issue-20102", |
| "token": "gitea_token" |
| } |
| }, |
| "result": null |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "action": { |
| "arguments": { |
| "base_branch": "main", |
| "body": "Includes the minimal implementation and focused tests.", |
| "head_branch": "fix/issue-20102", |
| "title": "Fix issue 20102" |
| }, |
| "name": "provider.open_change" |
| }, |
| "plan": "Open the already-pushed branch as a Gitea change." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"The fix for issue 20103 has been prepared, committed, and opened on Forgejo. The pull request number is 20103.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "The fix for issue 20103 has been prepared, committed, and opened on Forgejo. The pull request number is 20103." |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "result": "Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"Change 20020 has been approved with current-revision evidence.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Change 20020 has been approved with current-revision evidence." |
| }, |
| "family": "review_approve", |
| "expected": { |
| "result": "Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Check the current-revision evidence for GitLab change 20021\",\"action\":{\"name\":\"get_gitlab_change_revision\",\"arguments\":{\"change_id\":20021}}}", |
| "actual": { |
| "plan": "Check the current-revision evidence for GitLab change 20021", |
| "action": { |
| "name": "get_gitlab_change_revision", |
| "arguments": { |
| "change_id": 20021 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_approve", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20021 |
| }, |
| "name": "provider.get_change" |
| }, |
| "plan": "Fetch the change and bind review to its current head." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20022,\"sha\":\"8be1c0da0cf192d5742d40cb5e6a27ff82b4f9480a91572df36674bf6fb26a0a\",\"approval\":\"approved\"},\"name\":\"provider.review\"},\"plan\":\"Approve the change with current-revision evidence.\"}", |
| "actual": { |
| "plan": "Approve the change with current-revision evidence.", |
| "action": { |
| "name": "provider.review", |
| "arguments": { |
| "number": 20022, |
| "sha": "8be1c0da0cf192d5742d40cb5e6a27ff82b4f9480a91572df36674bf6fb26a0a", |
| "approval": "approved" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_approve", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20022 |
| }, |
| "name": "provider.get_diff" |
| }, |
| "plan": "Inspect the diff for that exact revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20023,\"head_sha\":\"53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381\"},\"name\":\"provider.approve\"},\"plan\":\"Approve the change with the current-revision evidence.\"}", |
| "actual": { |
| "plan": "Approve the change with the current-revision evidence.", |
| "action": { |
| "name": "provider.approve", |
| "arguments": { |
| "number": 20023, |
| "head_sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_approve", |
| "expected": { |
| "action": { |
| "arguments": { |
| "sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" |
| }, |
| "name": "provider.get_checks" |
| }, |
| "plan": "Check CI for the same commit SHA." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20070,\"head_sha\":\"76c464eb4bfccce7c4b6d2fe3323268568f14d4e42334c495a47b617e439b035\"},\"name\":\"provider.get_check\"},\"plan\":\"Verify that all checks for the change have passed.\"}", |
| "actual": { |
| "plan": "Verify that all checks for the change have passed.", |
| "action": { |
| "name": "provider.get_check", |
| "arguments": { |
| "number": 20070, |
| "head_sha": "76c464eb4bfccce7c4b6d2fe3323268568f14d4e42334c495a47b617e439b035" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_block", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20070 |
| }, |
| "name": "provider.get_diff" |
| }, |
| "plan": "Inspect the exact diff before reaching a conclusion." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20071,\"head_sha\":\"9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921\"},\"name\":\"provider.get_check\"},\"plan\":\"Verify if all checks have passed for the change.\"}", |
| "actual": { |
| "plan": "Verify if all checks have passed for the change.", |
| "action": { |
| "name": "provider.get_check", |
| "arguments": { |
| "number": 20071, |
| "head_sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_block", |
| "expected": { |
| "action": { |
| "arguments": { |
| "sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" |
| }, |
| "name": "provider.get_checks" |
| }, |
| "plan": "Read checks for the reviewed SHA." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"change_20072_not_approved\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "change_20072_not_approved" |
| }, |
| "family": "review_block", |
| "expected": { |
| "result": "Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Check if change 20073 has passed all necessary reviews.\",\"action\":{\"name\":\"get_change_reviews\",\"arguments\":{\"change_id\":20073}}}", |
| "actual": { |
| "plan": "Check if change 20073 has passed all necessary reviews.", |
| "action": { |
| "name": "get_change_reviews", |
| "arguments": { |
| "change_id": 20073 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_block", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20073 |
| }, |
| "name": "provider.get_change" |
| }, |
| "plan": "Fetch the current change revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20150,\"head_sha\":\"1849708cc7094b7bc1b97ddc52582e7223c485948abaed97ce0c76b3f4b82d39\"},\"name\":\"provider.rebase\"},\"plan\":\"Rebase the change with the exact revision.\"}", |
| "actual": { |
| "plan": "Rebase the change with the exact revision.", |
| "action": { |
| "name": "provider.rebase", |
| "arguments": { |
| "number": 20150, |
| "head_sha": "1849708cc7094b7bc1b97ddc52582e7223c485948abaed97ce0c76b3f4b82d39" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_merge", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20150 |
| }, |
| "name": "provider.get_diff" |
| }, |
| "plan": "Review the diff at the bound revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"head_sha\":\"f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7\",\"number\":20151},\"name\":\"provider.merge\"},\"plan\":\"Merge the change with squash after verification.\"}", |
| "actual": { |
| "plan": "Merge the change with squash after verification.", |
| "action": { |
| "name": "provider.merge", |
| "arguments": { |
| "head_sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7", |
| "number": 20151 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_merge", |
| "expected": { |
| "action": { |
| "arguments": { |
| "sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7" |
| }, |
| "name": "provider.get_checks" |
| }, |
| "plan": "Verify required checks on the same SHA." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"sha\":\"d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d\",\"number\":20152},\"name\":\"provider.merge\"},\"plan\":\"Merge the change after verifying its exact revision.\"}", |
| "actual": { |
| "plan": "Merge the change after verifying its exact revision.", |
| "action": { |
| "name": "provider.merge", |
| "arguments": { |
| "sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", |
| "number": 20152 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_merge", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", |
| "method": "merge", |
| "number": 20152 |
| }, |
| "name": "provider.merge_change" |
| }, |
| "plan": "Merge only the reviewed, passing revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\": \"Change 20153 has been successfully merged with a rebase after verifying its exact revision.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Change 20153 has been successfully merged with a rebase after verifying its exact revision." |
| }, |
| "family": "review_merge", |
| "expected": { |
| "result": "Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"grep\",\"\" ,\"tests/test_parser.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Locate the test function causing the failure.\"}", |
| "actual": { |
| "plan": "Locate the test function causing the failure.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "grep", |
| "", |
| "tests/test_parser.py" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src", |
| "query": "def handle" |
| }, |
| "name": "workspace.search" |
| }, |
| "plan": "Locate the failing implementation named in the test." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/router.py\",\"line\":12},\"name\":\"workspace.open\"},\"plan\":\"Inspect the implementation to find the issue.\"}", |
| "actual": { |
| "plan": "Inspect the implementation to find the issue.", |
| "action": { |
| "name": "workspace.open", |
| "arguments": { |
| "path": "src/router.py", |
| "line": 12 |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src/router.py" |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the implementation before changing it." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/worker.py\",\"query\":\"def handle\"},\"name\":\"workspace.search\"},\"plan\":\"Find the function that returns values[0] and is called in the test.\"}", |
| "actual": { |
| "plan": "Find the function that returns values[0] and is called in the test.", |
| "action": { |
| "name": "workspace.search", |
| "arguments": { |
| "path": "src/worker.py", |
| "query": "def handle" |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e", |
| "new_text": "return values[0] if values else None", |
| "old_text": "return values[0]", |
| "path": "src/worker.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Guard the observed empty-input access." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_client.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Verify the fix by re-running the test.\"}", |
| "actual": { |
| "plan": "Verify the fix by re-running the test.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_client.py" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_client.py" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Rerun the previously failing test." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519\",\"new_text\":\"MODE='new'\\nTIMEOUT=30\",\"old_text\":\"MODE='old'\\nTIMEOUT=30\",\"path\":\"src/config_1.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Replace the content with the new value while preserving the existing content.\"}", |
| "actual": { |
| "plan": "Replace the content with the new value while preserving the existing content.", |
| "action": { |
| "name": "workspace.replace", |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", |
| "new_text": "MODE='new'\nTIMEOUT=30", |
| "old_text": "MODE='old'\nTIMEOUT=30", |
| "path": "src/config_1.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", |
| "new_text": "MODE='new'", |
| "old_text": "MODE='old'", |
| "path": "src/config_1.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Preserve the concurrent addition and replace only the requested value." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config_2.py\"},\"name\":\"workspace.read\"},\"plan\":\"Verify the change and capture the new hash for confirmation.\"}", |
| "actual": { |
| "plan": "Verify the change and capture the new hash for confirmation.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/config_2.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "result": "Applied the requested rename while preserving the concurrent timeout change." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Check for concurrent changes\",\"action\":{\"name\":\"check_concurrent_changes\",\"arguments\":{}}}", |
| "actual": { |
| "plan": "Check for concurrent changes", |
| "action": { |
| "name": "check_concurrent_changes", |
| "arguments": {} |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src/config_3.py" |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the current file and capture its hash." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config_4.py\",\"sha256\":\"c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9\"},\"name\":\"workspace.apply\"},\"plan\":\"Apply the current file without overwriting concurrent changes.\"}", |
| "actual": { |
| "plan": "Apply the current file without overwriting concurrent changes.", |
| "action": { |
| "name": "workspace.apply", |
| "arguments": { |
| "path": "src/config_4.py", |
| "sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9" |
| } |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9", |
| "new_text": "MODE='new'", |
| "old_text": "MODE='old'", |
| "path": "src/config_4.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Attempt the exact replacement against the observed hash." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"go\",\"test\",\"./...\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", |
| "actual": { |
| "plan": "Attempt the requested verification through the policy-checked shell tool.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "go", |
| "test", |
| "./..." |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "result": "Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Run the project verification command\",\"action\":{\"name\":\"run_command\",\"arguments\":{\"command\":\"npm test\"}}}", |
| "actual": { |
| "plan": "Run the project verification command", |
| "action": { |
| "name": "run_command", |
| "arguments": { |
| "command": "npm test" |
| } |
| }, |
| "result": null |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "npm", |
| "test" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Attempt the requested verification through the policy-checked shell tool." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"cargo\",\"test\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", |
| "actual": { |
| "plan": "Attempt the requested verification through the policy-checked shell tool.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "cargo", |
| "test" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "result": "Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": false, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"plan\":\"Run the project verification command\",\"action\":{\"name\":\"run_command\",\"arguments\":{\"command\":\"go test ./...\"}}}", |
| "actual": { |
| "plan": "Run the project verification command", |
| "action": { |
| "name": "run_command", |
| "arguments": { |
| "command": "go test ./..." |
| } |
| }, |
| "result": null |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "go", |
| "test", |
| "./..." |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Attempt the requested verification through the policy-checked shell tool." |
| } |
| } |
| ] |
| }, |
| "adapter_evaluation": { |
| "count": 40, |
| "valid_rate": 1.0, |
| "semantic_rate": 1.0, |
| "arguments_exact_rate": 0.975, |
| "cases": [ |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src\",\"query\":\"def parser\"},\"name\":\"workspace.search\"},\"plan\":\"Search narrowly before reading files.\"}", |
| "actual": { |
| "plan": "Search narrowly before reading files.", |
| "action": { |
| "name": "workspace.search", |
| "arguments": { |
| "path": "src", |
| "query": "def parser" |
| } |
| }, |
| "result": null |
| }, |
| "family": "inspect", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src", |
| "query": "def parser" |
| }, |
| "name": "workspace.search" |
| }, |
| "plan": "Search narrowly before reading files." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"end_line\":90,\"path\":\"src/router.py\",\"start_line\":1},\"name\":\"workspace.read\"},\"plan\":\"Read the matching implementation.\"}", |
| "actual": { |
| "plan": "Read the matching implementation.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "end_line": 90, |
| "path": "src/router.py", |
| "start_line": 1 |
| } |
| }, |
| "result": null |
| }, |
| "family": "inspect", |
| "expected": { |
| "action": { |
| "arguments": { |
| "end_line": 90, |
| "path": "src/router.py", |
| "start_line": 1 |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the matching implementation." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation." |
| }, |
| "family": "inspect", |
| "expected": { |
| "result": "Found `serializer` in `src/serializer.py` at line 17 and inspected its implementation." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src\",\"query\":\"def scheduler\"},\"name\":\"workspace.search\"},\"plan\":\"Search narrowly before reading files.\"}", |
| "actual": { |
| "plan": "Search narrowly before reading files.", |
| "action": { |
| "name": "workspace.search", |
| "arguments": { |
| "path": "src", |
| "query": "def scheduler" |
| } |
| }, |
| "result": null |
| }, |
| "family": "inspect", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src", |
| "query": "def scheduler" |
| }, |
| "name": "workspace.search" |
| }, |
| "plan": "Search narrowly before reading files." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779\",\"new_text\":\"def handle(items):\\n if not items:\\n return None\\n return items[0]\",\"old_text\":\"def handle(items):\\n return items[0]\",\"path\":\"src/parser.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Apply the smallest concurrency-safe replacement.\"}", |
| "actual": { |
| "plan": "Apply the smallest concurrency-safe replacement.", |
| "action": { |
| "name": "workspace.replace", |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779", |
| "new_text": "def handle(items):\n if not items:\n return None\n return items[0]", |
| "old_text": "def handle(items):\n return items[0]", |
| "path": "src/parser.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "58febb3133e74760e613269ec2e92f1c8861f4ca4b2fafe350bdb4eca1475779", |
| "new_text": "def handle(items):\n if not items:\n return None\n return items[0]", |
| "old_text": "def handle(items):\n return items[0]", |
| "path": "src/parser.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Apply the smallest concurrency-safe replacement." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_config.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Run only the relevant test module first.\"}", |
| "actual": { |
| "plan": "Run only the relevant test module first.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_config.py" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_config.py" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Run only the relevant test module first." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Fixed empty-input handling in `src/client.py`; all 6 focused tests pass.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Fixed empty-input handling in `src/client.py`; all 6 focused tests pass." |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "result": "Fixed empty-input handling in `src/client.py`; all 6 focused tests pass." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/cache.py\"},\"name\":\"workspace.read\"},\"plan\":\"Read the current implementation and capture its content hash.\"}", |
| "actual": { |
| "plan": "Read the current implementation and capture its content hash.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/cache.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "edit_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src/cache.py" |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the current implementation and capture its content hash." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_health.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Verify the new behavior with its focused test.\"}", |
| "actual": { |
| "plan": "Verify the new behavior with its focused test.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_health.py" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_health.py" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Verify the new behavior with its focused test." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Added `src/version.py` and verified its 2 focused tests pass.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Added `src/version.py` and verified its 2 focused tests pass." |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "result": "Added `src/version.py` and verified its 2 focused tests pass." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"limit\":100,\"path\":\"src\"},\"name\":\"workspace.list\"},\"plan\":\"List the package before creating a new module.\"}", |
| "actual": { |
| "plan": "List the package before creating a new module.", |
| "action": { |
| "name": "workspace.list", |
| "arguments": { |
| "limit": 100, |
| "path": "src" |
| } |
| }, |
| "result": null |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "limit": 100, |
| "path": "src" |
| }, |
| "name": "workspace.list" |
| }, |
| "plan": "List the package before creating a new module." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"content\":\"def metadata():\\n return \\\"ok\\\"\\n\",\"expected_sha256\":null,\"path\":\"src/metadata.py\"},\"name\":\"workspace.write\"},\"plan\":\"Create only the requested module.\"}", |
| "actual": { |
| "plan": "Create only the requested module.", |
| "action": { |
| "name": "workspace.write", |
| "arguments": { |
| "content": "def metadata():\n return \"ok\"\n", |
| "expected_sha256": null, |
| "path": "src/metadata.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "create_and_test", |
| "expected": { |
| "action": { |
| "arguments": { |
| "content": "def metadata():\n return \"ok\"\n", |
| "expected_sha256": null, |
| "path": "src/metadata.py" |
| }, |
| "name": "workspace.write" |
| }, |
| "plan": "Create only the requested module." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_head\":\"d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845\",\"message\":\"Fix issue 20100\",\"paths\":[\"src/issue_20100.py\",\"tests/test_issue_20100.py\"]},\"name\":\"git.commit\"},\"plan\":\"Commit only the requested source and test files.\"}", |
| "actual": { |
| "plan": "Commit only the requested source and test files.", |
| "action": { |
| "name": "git.commit", |
| "arguments": { |
| "expected_head": "d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845", |
| "message": "Fix issue 20100", |
| "paths": [ |
| "src/issue_20100.py", |
| "tests/test_issue_20100.py" |
| ] |
| } |
| }, |
| "result": null |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_head": "d5af24fbbf21e7893bea19d858428447c72c6bbe25b1b76344f09b99b3c87845", |
| "message": "Fix issue 20100", |
| "paths": [ |
| "src/issue_20100.py", |
| "tests/test_issue_20100.py" |
| ] |
| }, |
| "name": "git.commit" |
| }, |
| "plan": "Commit only the requested source and test files." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"branch\":\"fix/issue-20101\",\"expected_head\":\"d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3\",\"remote\":\"origin\"},\"name\":\"git.push\"},\"plan\":\"Push the exact commit on the focused branch.\"}", |
| "actual": { |
| "plan": "Push the exact commit on the focused branch.", |
| "action": { |
| "name": "git.push", |
| "arguments": { |
| "branch": "fix/issue-20101", |
| "expected_head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", |
| "remote": "origin" |
| } |
| }, |
| "result": null |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "action": { |
| "arguments": { |
| "branch": "fix/issue-20101", |
| "expected_head": "d256cce9835cecbb4881661161b01c78f7dbe7fc22b7b8b1a43523e0e9d5f9d3", |
| "remote": "origin" |
| }, |
| "name": "git.push" |
| }, |
| "plan": "Push the exact commit on the focused branch." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"base_branch\":\"main\",\"body\":\"Includes the minimal implementation and focused tests.\",\"head_branch\":\"fix/issue-20102\",\"title\":\"Fix issue 20102\"},\"name\":\"provider.open_change\"},\"plan\":\"Open the already-pushed branch as a Gitea change.\"}", |
| "actual": { |
| "plan": "Open the already-pushed branch as a Gitea change.", |
| "action": { |
| "name": "provider.open_change", |
| "arguments": { |
| "base_branch": "main", |
| "body": "Includes the minimal implementation and focused tests.", |
| "head_branch": "fix/issue-20102", |
| "title": "Fix issue 20102" |
| } |
| }, |
| "result": null |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "action": { |
| "arguments": { |
| "base_branch": "main", |
| "body": "Includes the minimal implementation and focused tests.", |
| "head_branch": "fix/issue-20102", |
| "title": "Fix issue 20102" |
| }, |
| "name": "provider.open_change" |
| }, |
| "plan": "Open the already-pushed branch as a Gitea change." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`." |
| }, |
| "family": "git_to_change", |
| "expected": { |
| "result": "Opened Forgejo change 20103 for `fix/issue-20103` at `48129e00357197548fdafe28042290aee84203362b4186b66a546729dd047b03`." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`." |
| }, |
| "family": "review_approve", |
| "expected": { |
| "result": "Approved change 20020 at exact revision `9f5e401f6663311689526f08efe6449c5684a219d1d0e7490537bcd8d1830fa9`." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20021},\"name\":\"provider.get_change\"},\"plan\":\"Fetch the change and bind review to its current head.\"}", |
| "actual": { |
| "plan": "Fetch the change and bind review to its current head.", |
| "action": { |
| "name": "provider.get_change", |
| "arguments": { |
| "number": 20021 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_approve", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20021 |
| }, |
| "name": "provider.get_change" |
| }, |
| "plan": "Fetch the change and bind review to its current head." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20022},\"name\":\"provider.get_diff\"},\"plan\":\"Inspect the diff for that exact revision.\"}", |
| "actual": { |
| "plan": "Inspect the diff for that exact revision.", |
| "action": { |
| "name": "provider.get_diff", |
| "arguments": { |
| "number": 20022 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_approve", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20022 |
| }, |
| "name": "provider.get_diff" |
| }, |
| "plan": "Inspect the diff for that exact revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"sha\":\"53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381\"},\"name\":\"provider.get_checks\"},\"plan\":\"Check CI for the same commit SHA.\"}", |
| "actual": { |
| "plan": "Check CI for the same commit SHA.", |
| "action": { |
| "name": "provider.get_checks", |
| "arguments": { |
| "sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_approve", |
| "expected": { |
| "action": { |
| "arguments": { |
| "sha": "53f79821d8640743877a35554e0bae3c3243b59cdd7461d79f4ac2b69fbe8381" |
| }, |
| "name": "provider.get_checks" |
| }, |
| "plan": "Check CI for the same commit SHA." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20070},\"name\":\"provider.get_diff\"},\"plan\":\"Inspect the exact diff before reaching a conclusion.\"}", |
| "actual": { |
| "plan": "Inspect the exact diff before reaching a conclusion.", |
| "action": { |
| "name": "provider.get_diff", |
| "arguments": { |
| "number": 20070 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_block", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20070 |
| }, |
| "name": "provider.get_diff" |
| }, |
| "plan": "Inspect the exact diff before reaching a conclusion." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"sha\":\"9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921\"},\"name\":\"provider.get_checks\"},\"plan\":\"Read checks for the reviewed SHA.\"}", |
| "actual": { |
| "plan": "Read checks for the reviewed SHA.", |
| "action": { |
| "name": "provider.get_checks", |
| "arguments": { |
| "sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_block", |
| "expected": { |
| "action": { |
| "arguments": { |
| "sha": "9e2e60af0f84d475571455a80aa0b0f2eecf413611abe2db14f6c8366d2fd921" |
| }, |
| "name": "provider.get_checks" |
| }, |
| "plan": "Read checks for the reviewed SHA." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`." |
| }, |
| "family": "review_block", |
| "expected": { |
| "result": "Did not approve change 20072: required tests fail on `b330f4b4fd4dd0ae1dad0b47a8c7e835c03cd8dbce16d43021132128522b7a4a`." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20073},\"name\":\"provider.get_change\"},\"plan\":\"Fetch the current change revision.\"}", |
| "actual": { |
| "plan": "Fetch the current change revision.", |
| "action": { |
| "name": "provider.get_change", |
| "arguments": { |
| "number": 20073 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_block", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20073 |
| }, |
| "name": "provider.get_change" |
| }, |
| "plan": "Fetch the current change revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"number\":20150},\"name\":\"provider.get_diff\"},\"plan\":\"Review the diff at the bound revision.\"}", |
| "actual": { |
| "plan": "Review the diff at the bound revision.", |
| "action": { |
| "name": "provider.get_diff", |
| "arguments": { |
| "number": 20150 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_merge", |
| "expected": { |
| "action": { |
| "arguments": { |
| "number": 20150 |
| }, |
| "name": "provider.get_diff" |
| }, |
| "plan": "Review the diff at the bound revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"sha\":\"f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7\"},\"name\":\"provider.get_checks\"},\"plan\":\"Verify required checks on the same SHA.\"}", |
| "actual": { |
| "plan": "Verify required checks on the same SHA.", |
| "action": { |
| "name": "provider.get_checks", |
| "arguments": { |
| "sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7" |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_merge", |
| "expected": { |
| "action": { |
| "arguments": { |
| "sha": "f0ec4f1cedc970a3cae4f059ea2e537fabf7271a93173ec56350b92f011387e7" |
| }, |
| "name": "provider.get_checks" |
| }, |
| "plan": "Verify required checks on the same SHA." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_sha\":\"d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d\",\"method\":\"merge\",\"number\":20152},\"name\":\"provider.merge_change\"},\"plan\":\"Merge only the reviewed, passing revision.\"}", |
| "actual": { |
| "plan": "Merge only the reviewed, passing revision.", |
| "action": { |
| "name": "provider.merge_change", |
| "arguments": { |
| "expected_sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", |
| "method": "merge", |
| "number": 20152 |
| } |
| }, |
| "result": null |
| }, |
| "family": "review_merge", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_sha": "d49a6148616b6108f3db866a9b3128d95a7eafd8039ea0c27a1d9351d0e2086d", |
| "method": "merge", |
| "number": 20152 |
| }, |
| "name": "provider.merge_change" |
| }, |
| "plan": "Merge only the reviewed, passing revision." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase." |
| }, |
| "family": "review_merge", |
| "expected": { |
| "result": "Merged change 20153 at `0b23ed71c765205fd7341e8aab7e00290216a956dd17a7482ba3bc852667ca75` using rebase." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src\",\"query\":\"def handle\"},\"name\":\"workspace.search\"},\"plan\":\"Locate the failing implementation named in the test.\"}", |
| "actual": { |
| "plan": "Locate the failing implementation named in the test.", |
| "action": { |
| "name": "workspace.search", |
| "arguments": { |
| "path": "src", |
| "query": "def handle" |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src", |
| "query": "def handle" |
| }, |
| "name": "workspace.search" |
| }, |
| "plan": "Locate the failing implementation named in the test." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/router.py\"},\"name\":\"workspace.read\"},\"plan\":\"Read the implementation before changing it.\"}", |
| "actual": { |
| "plan": "Read the implementation before changing it.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/router.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src/router.py" |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the implementation before changing it." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e\",\"new_text\":\"return values[0] if values else None\",\"old_text\":\"return values[0]\",\"path\":\"src/worker.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Guard the observed empty-input access.\"}", |
| "actual": { |
| "plan": "Guard the observed empty-input access.", |
| "action": { |
| "name": "workspace.replace", |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e", |
| "new_text": "return values[0] if values else None", |
| "old_text": "return values[0]", |
| "path": "src/worker.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "76f726b7987c93d815f8ecc19abd415131e30ac0da260e9e9e7092388e09b33e", |
| "new_text": "return values[0] if values else None", |
| "old_text": "return values[0]", |
| "path": "src/worker.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Guard the observed empty-input access." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"pytest\",\"-q\",\"tests/test_client.py\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Rerun the previously failing test.\"}", |
| "actual": { |
| "plan": "Rerun the previously failing test.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_client.py" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "test_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "pytest", |
| "-q", |
| "tests/test_client.py" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Rerun the previously failing test." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519\",\"new_text\":\"MODE='new'\",\"old_text\":\"MODE='old'\",\"path\":\"src/config_1.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Preserve the concurrent addition and replace only the requested value.\"}", |
| "actual": { |
| "plan": "Preserve the concurrent addition and replace only the requested value.", |
| "action": { |
| "name": "workspace.replace", |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", |
| "new_text": "MODE='new'", |
| "old_text": "MODE='old'", |
| "path": "src/config_1.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "60191e1caf070507af2e7e67d03e4354ff5417a7166dfce552ff864e3cf59519", |
| "new_text": "MODE='new'", |
| "old_text": "MODE='old'", |
| "path": "src/config_1.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Preserve the concurrent addition and replace only the requested value." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Applied the requested rename while preserving the concurrent timeout change.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Applied the requested rename while preserving the concurrent timeout change." |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "result": "Applied the requested rename while preserving the concurrent timeout change." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": false, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"path\":\"src/config_1.py\"},\"name\":\"workspace.read\"},\"plan\":\"Read the current file and capture its hash.\"}", |
| "actual": { |
| "plan": "Read the current file and capture its hash.", |
| "action": { |
| "name": "workspace.read", |
| "arguments": { |
| "path": "src/config_1.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "path": "src/config_3.py" |
| }, |
| "name": "workspace.read" |
| }, |
| "plan": "Read the current file and capture its hash." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"expected_occurrences\":1,\"expected_sha256\":\"c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9\",\"new_text\":\"MODE='new'\",\"old_text\":\"MODE='old'\",\"path\":\"src/config_4.py\"},\"name\":\"workspace.replace\"},\"plan\":\"Attempt the exact replacement against the observed hash.\"}", |
| "actual": { |
| "plan": "Attempt the exact replacement against the observed hash.", |
| "action": { |
| "name": "workspace.replace", |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9", |
| "new_text": "MODE='new'", |
| "old_text": "MODE='old'", |
| "path": "src/config_4.py" |
| } |
| }, |
| "result": null |
| }, |
| "family": "stale_write_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "expected_occurrences": 1, |
| "expected_sha256": "c9d084c19f44101fd6296b21b2306c0ec7db0561bcf055c4378cb946975a4be9", |
| "new_text": "MODE='new'", |
| "old_text": "MODE='old'", |
| "path": "src/config_4.py" |
| }, |
| "name": "workspace.replace" |
| }, |
| "plan": "Attempt the exact replacement against the observed hash." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "result": "Could not run `go test ./...` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"npm\",\"test\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", |
| "actual": { |
| "plan": "Attempt the requested verification through the policy-checked shell tool.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "npm", |
| "test" |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "npm", |
| "test" |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Attempt the requested verification through the policy-checked shell tool." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"result\":\"Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command.\"}", |
| "actual": { |
| "plan": null, |
| "action": null, |
| "result": "Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "result": "Could not run `cargo test` because runtime policy denied it. No verification result is claimed; an operator must allowlist the command." |
| } |
| }, |
| { |
| "valid": true, |
| "semantic": true, |
| "arguments_exact": true, |
| "error": null, |
| "raw": "{\"action\":{\"arguments\":{\"argv\":[\"go\",\"test\",\"./...\"],\"cwd\":\".\"},\"name\":\"shell.run\"},\"plan\":\"Attempt the requested verification through the policy-checked shell tool.\"}", |
| "actual": { |
| "plan": "Attempt the requested verification through the policy-checked shell tool.", |
| "action": { |
| "name": "shell.run", |
| "arguments": { |
| "argv": [ |
| "go", |
| "test", |
| "./..." |
| ], |
| "cwd": "." |
| } |
| }, |
| "result": null |
| }, |
| "family": "permission_recovery", |
| "expected": { |
| "action": { |
| "arguments": { |
| "argv": [ |
| "go", |
| "test", |
| "./..." |
| ], |
| "cwd": "." |
| }, |
| "name": "shell.run" |
| }, |
| "plan": "Attempt the requested verification through the policy-checked shell tool." |
| } |
| } |
| ] |
| }, |
| "publication_gate": { |
| "valid_rate_at_least_0_90": true, |
| "semantic_rate_at_least_0_80": true, |
| "not_materially_worse_than_base": true |
| }, |
| "versions": { |
| "python": "3.13.15", |
| "torch": "2.11.0+cu128", |
| "transformers": "4.57.6", |
| "peft": "0.20.0", |
| "accelerate": "1.14.0", |
| "bitsandbytes": "0.50.1" |
| }, |
| "wheel_sha256": "57a015816506665d75eaaedb557b4f53883258e69094723ac145340097c857c6", |
| "data_archive_sha256": "2bb44ab5a80d6e3bd455d490403464d083ca6a5d5fe5b4a9b4fc3769d0bd7dc5", |
| "elapsed_seconds": 1526.45 |
| } |
|
|
