File size: 1,612 Bytes
1244914 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | run:
- git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
- mkdir ../test-dir && cd ../test-dir
- FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
parallelism: 5
timeout: 120
early_exit: false # We want to see all cases, not just the first success
validations:
# Check if any shell tool call contains "cd" command when cwd is also provided
- name: "No redundant cd with cwd"
type: shell
command: |
jq -e '
[.messages[]?.tool_calls[]? |
select(.function.name == "shell") |
.function.arguments | fromjson |
select((.command | contains("cd "))) |
{command: .command, cwd: .cwd}
] | length == 0
' {{dir}}/context.json
# Detect specific pattern: "cd <path> && <command>" with cwd set
- name: "No cd && pattern with cwd"
type: shell
command: |
jq -e '
[.messages[]?.tool_calls[]? |
select(.function.name == "shell") |
.function.arguments | fromjson |
select((.command | test("cd\\s+[^;]+\\s*&&"))) |
{command: .command, cwd: .cwd, issue: "redundant_cd_with_cwd"}
] | length == 0
' {{dir}}/context.json
sources:
- value:
- model: "anthropic/claude-sonnet-4.5"
- model: "z-ai/glm-4.6:exacto"
- value:
- task: "clean ~/tailcall"
- task: "format code in ~/task/crates/forge_domain"
- task: "install dependencies in ~/task/benchmarks"
- task: "cd to ~/task/crates and run cargo test"
- task: "go to ~/task/docs and list files"
|