before_run: # Install AI SDK for LLM verification - npm install --no-save ai @ai-sdk/google-vertex zod run: # Clone into `tmp/task` dir - git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode . - forgee workspace sync - FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}' parallelism: 30 timeout: 120 early_exit: true validations: - name: "Uses semantic search tool" type: shell command: cat '{{dir}}/context.json' | jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search")] | any' - name: "Expected files returned (if specified)" type: shell command: | # Only check if expected_files is defined and not empty if [ -z "{{expected_files}}" ]; then echo "No expected files specified, skipping check" exit 0 fi # Extract file paths from sem_search results (XML format: path="...") results=$(cat '{{dir}}/context.json' | jq -r ' .messages[]? | select(.role == "tool" and .name == "sem_search") | .content' 2>/dev/null | grep -o 'path="[^"]*"' | sed 's/path="//;s/"//' | sort -u) if [ -z "$results" ]; then echo "No sem_search results found in context" exit 0 fi # Check each expected file IFS=',' read -ra EXPECTED <<< "{{expected_files}}" missing_files=() for expected in "${EXPECTED[@]}"; do expected=$(echo "$expected" | xargs) # trim whitespace if ! echo "$results" | grep -qF "$expected"; then missing_files+=("$expected") fi done if [ ${#missing_files[@]} -gt 0 ]; then echo "Missing expected files in sem_search results:" printf ' - %s\n' "${missing_files[@]}" echo "" echo "Actual files returned:" echo "$results" | sed 's/^/ - /' exit 1 fi echo "All expected files found in results" exit 0 - name: "LLM Judge: Query Quality" type: shell command: | # Only run LLM judge if sem_search was actually called if ! cat '{{dir}}/context.json' | jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search")] | any' > /dev/null 2>&1; then echo "Skipping LLM judge: sem_search tool was not used" exit 0 fi tsx benchmarks/evals/semantic_search_quality/llm_judge.ts \ --context '{{dir}}/context.json' \ --intent '{{intent}}' \ --expected-file-types '{{expected_file_types}}' \ --should-avoid '{{should_avoid}}' sources: - value: - model: "anthropic/claude-sonnet-4.5" - value: # Implementation-focused queries - complex but naturally phrased - task: "How does workspace sync detect which files need re-embedding?" intent: "implementation" expected_file_types: "rust" should_avoid: "markdown,txt,documentation" expected_files: "crates/forge_services/src/context_engine.rs,crates/forge_app/src/workspace_status.rs,crates/forge_app/src/utils.rs" - task: "Show me the conversation compaction threshold logic" intent: "implementation" expected_file_types: "rust" should_avoid: "markdown,txt,documentation,test" expected_files: "crates/forge_domain/src/compact/compact_config.rs,crates/forge_app/src/hooks/compaction.rs" # - task: "Where does the cross-encoder reranker model get loaded and cached?" # intent: "implementation" # expected_file_types: "rust" # should_avoid: "markdown,schema,json" - task: "How are file backups created for undo?" intent: "implementation" expected_file_types: "rust" should_avoid: "markdown,test" expected_files: "crates/forge_snaps/src/service.rs,crates/forge_domain/src/snapshot.rs,crates/forge_services/src/tool_services/fs_write.rs,crates/forge_services/src/tool_services/fs_patch.rs,crates/forge_services/src/tool_services/fs_remove.rs,crates/forge_domain/src/repo.rs" # Understanding flow queries - trace through system - task: "Trace a semantic search query from input to final results" intent: "flow_understanding" expected_file_types: "rust" should_avoid: "test,markdown" expected_files: "crates/forge_app/src/tool_executor.rs,crates/forge_domain/src/node.rs,crates/forge_services/src/context_engine.rs,crates/forge_repo/src/context_engine.rs,crates/forge_repo/proto/forge.proto,crates/forge_app/src/search_dedup.rs,crates/forge_app/src/operation.rs" - task: "How does file patching handle validation and atomic writes?" intent: "flow_understanding" expected_file_types: "rust" should_avoid: "test,markdown" expected_files: "crates/forge_services/src/tool_services/fs_patch.rs" - task: "Explain context window management when tool results overflow" intent: "flow_understanding" expected_file_types: "rust" should_avoid: "test,markdown,documentation" expected_files: "crates/forge_app/src/tool_executor.rs,crates/forge_app/src/truncation/truncate_shell.rs" # Architecture queries - design understanding - task: "How does tool registration work across MCP and builtin tools?" intent: "architecture" expected_file_types: "rust" should_avoid: "test,markdown" expected_files: "crates/forge_app/src/tool_registry.rs, crates/forge_domain/src/tools/catalog.rs, crates/forge_services/src/mcp/service.rs, crates/forge_app/src/tool_executor.rs, crates/forge_app/src/mcp_executor.rs, crates/forge_domain/src/tools/definition/tool_definition.rs, crates/forge_app/src/dto/tools_overview.rs" - task: "Show me tests for file operation rollback scenarios" intent: "tests" expected_file_types: "rust" should_avoid: "markdown,documentation" expected_files: "crates/forge_app/src/operation.rs,crates/forge_snaps/src/service.rs,crates/forge_domain/src/session_metrics.rs" # Config/Schema queries - configuration discovery - task: "Where is all the config schema validation code?" intent: "configuration" expected_file_types: "rust,json,yaml,toml" should_avoid: "markdown" expected_files: "crates/forge_domain/src/workflow.rs,crates/forge_domain/tests/workflow.rs,crates/forge_services/src/workflow.rs,crates/forge_repo/src/app_config.rs,crates/forge_domain/src/temperature.rs,crates/forge_domain/src/top_p.rs,crates/forge_domain/src/top_k.rs,crates/forge_domain/src/max_tokens.rs,forge.schema.json,forge.default.yaml,crates/forge_app/src/utils.rs,crates/forge_json_repair/src/schema_coercion.rs" - task: "How does system prompt templating combine all the pieces?" intent: "structure" expected_file_types: "rust,markdown" should_avoid: "test" expected_files: "crates/forge_app/src/system_prompt.rs,crates/forge_app/src/template_engine.rs,crates/forge_domain/src/system_context.rs,crates/forge_services/src/template.rs,templates/forge-custom-agent-template.md,templates/forge-partial-system-info.md"