File size: 2,166 Bytes
1244914 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | run:
- git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
- FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
parallelism: 10
timeout: 180
early_exit: true
validations:
# Primary: Model should NOT use shell find command for searching code
- name: "No shell find command usage"
type: shell
command: |
jq -e '
[.messages[]?.tool_calls[]? |
select(.function.name == "shell") |
.function.arguments | fromjson |
select(.command | test("\\bfind\\s+"))
] | length == 0
' {{dir}}/context.json
# Model SHOULD use builtin search tools instead
- name: "Uses builtin search tools"
type: shell
command: |
jq -e '
[.messages[]?.tool_calls[]? |
select(.function.name == "search" or
.function.name == "fs_search" or
.function.name == "sem_search")
] | length > 0
' {{dir}}/context.json
# Detect specific anti-pattern: find with grep pipeline
- name: "No find + grep anti-pattern"
type: shell
command: |
jq -e '
[.messages[]?.tool_calls[]? |
select(.function.name == "shell") |
.function.arguments | fromjson |
select(.command | test("find.*\\|.*grep"))
] | length == 0
' {{dir}}/context.json
sources:
- value:
- model: "anthropic/claude-sonnet-4.5"
- model: "z-ai/glm-4.6:exacto"
- value:
# File discovery tasks - should use fs_search, NOT find command
- task: "I need to merge service and app crates. Tell me what files exist in the crates/ directory."
- task: "Find all Rust source files in the crates/service directory."
- task: "List all Rust files in crates/app to understand the structure."
- task: "Show me all .rs files in the project so I can plan refactoring."
# Structure understanding - should use fs_search or sem_search
- task: "Find all source files under crates/service and crates/app for analysis."
- task: "List all Rust source files in the workspace for documentation purposes."
|