Add files using upload-large-folder tool

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,117 @@
+name: 🐛 Bug Report
+description: Report a bug or unexpected behavior in Forge
+title: "[Bug]: "
+labels: ["type: bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug! Please fill out the sections below to help us reproduce and fix the issue.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Bug Description
+      description: A clear and concise description of what the bug is.
+      placeholder: Describe the bug...
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to Reproduce
+      description: Steps to reproduce the behavior
+      placeholder: |
+        1. Run command '...'
+        2. Enter prompt '...'
+        3. See error
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+      placeholder: Describe what should happen...
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+      placeholder: Describe what actually happened...
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: Forge Version
+      description: What version of Forge are you running? (`forge --version`)
+      placeholder: e.g., 0.1.0
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating System & Version
+      description: What operating system and version are you using?
+      placeholder: e.g., macOS 14.1, Ubuntu 22.04, Windows 11
+    validations:
+      required: false
+
+  - type: dropdown
+    id: provider
+    attributes:
+      label: AI Provider
+      description: Which AI provider are you using?
+      options:
+        - OpenAI
+        - Anthropic
+        - OpenRouter
+        - Google Vertex AI
+        - Cerebras
+        - x-ai
+        - z.ai
+        - Requesty
+        - IO Intelligence
+        - Groq
+        - Amazon Bedrock
+        - OpenAI-compatible
+        - Other
+    validations:
+      required: false
+
+  - type: input
+    id: model
+    attributes:
+      label: Model
+      description: Which model are you using?
+      placeholder: e.g., claude-3.7-sonnet, gpt-4, o3-mini-high
+
+  - type: dropdown
+    id: installation
+    attributes:
+      label: Installation Method
+      description: How did you install Forge?
+      options:
+        - npx forgecode@latest
+        - npm install -g forgecode
+        - curl -fsSL https://forgecode.dev/cli | sh
+        - Built from source
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: config
+    attributes:
+      label: Configuration
+      description: |
+        Please share relevant parts of your `forge.yaml` configuration (remove sensitive data like API keys).
+      render: yaml

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 💬 Discord Community
+    url: https://discord.gg/kRZBPpkgwq
+    about: Join our Discord server for questions, discussions, and community support
+  - name: 📖 Documentation
+    url: https://forgecode.dev/docs/
+    about: Check out our comprehensive documentation
+  - name: ❓ Discussions
+    url: https://github.com/antinomyhq/forge/discussions
+    about: Ask questions and discuss ideas with the community

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,100 @@
+name: 📚 Documentation Issue
+description: Report problems with documentation or suggest improvements
+title: "[Docs]: "
+labels: ["type: docs"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for helping improve our documentation! Clear docs help everyone.
+
+  - type: dropdown
+    id: issue-type
+    attributes:
+      label: Documentation Issue Type
+      description: What type of documentation issue is this?
+      options:
+        - Missing documentation
+        - Incorrect/outdated information
+        - Unclear explanation
+        - Broken link
+        - Typo/grammar
+        - Code example doesn't work
+        - Need more examples
+        - Need better organization
+        - Other
+    validations:
+      required: true
+
+  - type: input
+    id: location
+    attributes:
+      label: Documentation Location
+      description: Where is the documentation issue? (URL, file path, or section name)
+      placeholder: https://github.com/antinomyhq/forge/blob/main/docs/...
+    validations:
+      required: true
+
+  - type: textarea
+    id: current-content
+    attributes:
+      label: Current Content
+      description: What does the current documentation say? (Quote or describe)
+      placeholder: Currently, the documentation says...
+
+  - type: textarea
+    id: issue-description
+    attributes:
+      label: Issue Description
+      description: What is wrong or confusing about the current documentation?
+      placeholder: The issue is that...
+    validations:
+      required: true
+
+  - type: textarea
+    id: suggested-improvement
+    attributes:
+      label: Suggested Improvement
+      description: How would you improve this documentation?
+      placeholder: It would be better if...
+    validations:
+      required: true
+
+  - type: dropdown
+    id: audience
+    attributes:
+      label: Target Audience
+      description: Who would benefit from this documentation improvement?
+      options:
+        - New users/beginners
+        - Experienced users
+        - Contributors/developers
+        - All users
+      validations:
+        required: true
+
+  - type: textarea
+    id: examples
+    attributes:
+      label: Example or Reference
+      description: Provide examples or references that could help improve the documentation
+      placeholder: |
+        For example:
+        ```yaml
+        # forge.yaml example
+        model: claude-3.7-sonnet
+        ```
+
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Contribution
+      description: Can you help improve this documentation?
+      options:
+        - label: I'm willing to submit a PR with the documentation improvements
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional Context
+      description: Any other information that would help improve the documentation

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,155 @@
+name: ✨ Feature Request
+description: Suggest a new feature or enhancement for Forge
+title: "[Feature]: "
+labels: ["type: feature"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for suggesting a new feature! 
+        
+        **Before you continue, please ensure you can clearly answer:**
+        1. **Why do you NEED this feature?** (Not just "nice to have")
+        2. **What is NOT possible right now?** (Current limitations)
+        3. **What WILL be possible with this feature?** (Concrete outcomes)
+        
+        Vague or incomplete requests will be closed. Take your time to write a compelling case.
+
+  - type: textarea
+    id: why-needed
+    attributes:
+      label: "1. Why Do You NEED This Feature?"
+      description: |
+        **Be specific.** Explain the real-world problem or blocker you're facing.
+        Not "it would be cool if..." but "I cannot accomplish X because..."
+        Why is this important to your workflow? What is the impact of NOT having this?
+      placeholder: |
+        I NEED this feature because:
+        - Current workflow requires me to...
+        - This costs me X hours per week...
+        - Without this, I cannot accomplish...
+        - The business impact is...
+    validations:
+      required: true
+
+  - type: textarea
+    id: current-limitation
+    attributes:
+      label: "2. What Is NOT Possible Right Now?"
+      description: |
+        **Describe the current limitations in detail.**
+        What have you tried? What doesn't work? Why can't you work around it?
+        Include specific examples, commands, or workflows that fail.
+      placeholder: |
+        Currently, I cannot:
+        - [Specific action/workflow that fails]
+        - [What happens when you try]
+        - [Why existing features don't solve this]
+        
+        I've tried:
+        - [Workaround 1 and why it doesn't work]
+        - [Workaround 2 and why it doesn't work]
+    validations:
+      required: true
+
+  - type: textarea
+    id: future-possibilities
+    attributes:
+      label: "3. What WILL Be Possible With This Feature?"
+      description: |
+        **Describe concrete outcomes and capabilities.**
+        What specific tasks/workflows will this enable?
+        How will your work change? What new things can you build/do?
+      placeholder: |
+        With this feature, I will be able to:
+        - [Specific new capability 1]
+        - [Specific new capability 2]
+        - [Measurable improvement: time saved, errors reduced, etc.]
+        
+        Example workflow:
+        ```
+        # Show the exact commands or steps you'll use
+        forge new-feature --example
+        ```
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed Solution (User Experience)
+      description: |
+        **Focus on the user experience, not implementation details.**
+        How would a user interact with this feature? What would they see/do?
+        Focus on what the user experiences, not how it would be built.
+      placeholder: |
+        From a user's perspective, the feature should work like this:
+        - When I [action], I would see/get [result]
+        - The interaction would be: [step-by-step user flow]
+        - Example: "When I run `forge search docs`, I'd see matching results from Google Workspace"
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives Considered
+      description: Have you considered any alternative solutions or workarounds?
+      placeholder: I've also thought about...
+
+  - type: dropdown
+    id: category
+    attributes:
+      label: Feature Category
+      description: Which area does this feature relate to?
+      options:
+        - AI/LLM Integration
+        - Code Generation/Editing
+        - File Operations
+        - Configuration
+        - CLI/UX
+        - Documentation
+        - Performance
+        - Security
+        - MCP Integration
+        - Multi-Agent Workflows
+        - Shell Plugin
+        - Testing
+        - Other
+    validations:
+      required: true
+
+  - type: dropdown
+    id: priority
+    attributes:
+      label: Priority/Impact
+      description: How important is this feature to your workflow?
+      options:
+        - Critical - Blocking my work
+        - High - Would significantly improve my workflow
+        - Medium - Would be nice to have
+        - Low - Minor improvement
+    validations:
+      required: true
+
+  - type: textarea
+    id: examples
+    attributes:
+      label: Examples from Other Tools
+      description: Are there similar features in other tools that could serve as inspiration?
+      placeholder: Tool X does this by...
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional Context
+      description: Add any other context, screenshots, or examples about the feature request.
+  
+  - type: checkboxes
+    id: requirements-check
+    attributes:
+      label: "Pre-submission Checklist"
+      options:
+        - label: "I have searched existing issues and confirmed this is not a duplicate"
+          required: true
+        - label: "I am willing to submit a PR to implement this feature"

.github/ISSUE_TEMPLATE/performance_issue.yml

@@ -0,0 +1,150 @@
+name: ⚡ Performance Issue
+description: Report performance problems, slowness, or resource usage issues
+title: "[Performance]: "
+labels: ["type: performance"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for reporting a performance issue! Please provide detailed information to help us investigate.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Performance Issue Description
+      description: Describe the performance problem you're experiencing
+      placeholder: Forge becomes slow when...
+    validations:
+      required: true
+
+  - type: dropdown
+    id: issue-type
+    attributes:
+      label: Issue Type
+      description: What type of performance issue are you experiencing?
+      options:
+        - Slow response time
+        - High CPU usage
+        - High memory usage
+        - Network latency
+        - File I/O slowness
+        - Startup time
+        - Tool execution time
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to Reproduce
+      description: What actions lead to the performance issue?
+      placeholder: |
+        1. Run command '...'
+        2. Process large file/directory
+        3. Notice slowdown after...
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-performance
+    attributes:
+      label: Expected Performance
+      description: What performance did you expect?
+      placeholder: I expected the operation to complete in...
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual-performance
+    attributes:
+      label: Actual Performance
+      description: What performance are you experiencing?
+      placeholder: The operation takes... and uses...
+    validations:
+      required: true
+
+  - type: textarea
+    id: measurements
+    attributes:
+      label: Performance Measurements
+      description: |
+        If possible, provide specific measurements (time, memory, CPU usage, etc.)
+        You can use tools like `time`, `htop`, Activity Monitor, or Task Manager
+      placeholder: |
+        - Execution time: 45 seconds
+        - Memory usage: 2.5 GB
+        - CPU usage: 180%
+      render: markdown
+
+  - type: input
+    id: project-size
+    attributes:
+      label: Project Size
+      description: Approximate size of your project (number of files, lines of code, etc.)
+      placeholder: e.g., 500 files, ~50k lines of code
+
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment Details
+      description: |
+        Provide details about your environment:
+        - Forge version (`forge --version`)
+        - Operating system and version
+        - CPU, RAM
+        - AI provider and model
+        - Shell
+      placeholder: |
+        - Forge: 0.1.0
+        - OS: macOS 14.1
+        - CPU: M2 Pro
+        - RAM: 16GB
+        - Provider: Anthropic
+        - Model: claude-3.7-sonnet
+        - Shell: zsh
+    validations:
+      required: true
+
+  - type: textarea
+    id: config
+    attributes:
+      label: Configuration
+      description: Share relevant parts of your `forge.yaml` (remove sensitive data)
+      render: yaml
+
+  - type: textarea
+    id: profiling
+    attributes:
+      label: Profiling Data
+      description: |
+        If you have any profiling data or verbose logs, please share them here.
+        Run with `--verbose` flag for detailed output.
+      render: shell
+
+  - type: dropdown
+    id: frequency
+    attributes:
+      label: Frequency
+      description: How often does this performance issue occur?
+      options:
+        - Always
+        - Often (>50% of the time)
+        - Sometimes (25-50% of the time)
+        - Rarely (<25% of the time)
+        - Only in specific scenarios
+    validations:
+      required: true
+
+  - type: textarea
+    id: workarounds
+    attributes:
+      label: Workarounds
+      description: Have you found any workarounds for this issue?
+      placeholder: I can work around this by...
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional Context
+      description: Any other relevant information about the performance issue

.github/ISSUE_TEMPLATE/provider_integration.yml

@@ -0,0 +1,129 @@
+name: 🔌 Provider Integration Request
+description: Request support for a new AI provider
+title: "[Provider]: "
+labels: ["type: feature"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for suggesting a new AI provider! Please provide detailed information about the provider.
+
+  - type: input
+    id: provider-name
+    attributes:
+      label: Provider Name
+      description: What is the name of the AI provider?
+      placeholder: e.g., Cohere, Mistral AI, Together AI
+    validations:
+      required: true
+
+  - type: input
+    id: website
+    attributes:
+      label: Provider Website
+      description: Link to the provider's website
+      placeholder: https://...
+    validations:
+      required: true
+
+  - type: input
+    id: api-docs
+    attributes:
+      label: API Documentation
+      description: Link to the provider's API documentation
+      placeholder: https://docs...
+
+  - type: dropdown
+    id: api-type
+    attributes:
+      label: API Compatibility
+      description: What type of API does this provider offer?
+      options:
+        - OpenAI-compatible
+        - Anthropic-compatible
+        - Custom API
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: models
+    attributes:
+      label: Available Models
+      description: List the models this provider offers that would be useful in Forge
+      placeholder: |
+        - model-1 (description)
+        - model-2 (description)
+    validations:
+      required: true
+
+  - type: textarea
+    id: features
+    attributes:
+      label: Key Features
+      description: What are the key features or advantages of this provider?
+      placeholder: |
+        - Fast inference
+        - Specialized models
+        - Cost-effective
+        - Open source models
+    validations:
+      required: true
+
+  - type: dropdown
+    id: authentication
+    attributes:
+      label: Authentication Method
+      description: How does authentication work for this provider?
+      options:
+        - API Key
+        - OAuth
+        - Bearer Token
+        - Basic Auth
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: pricing
+    attributes:
+      label: Pricing Information
+      description: Link to pricing information or describe the pricing model
+      placeholder: https://provider.com/pricing or "Free tier available, paid plans start at..."
+
+  - type: dropdown
+    id: priority
+    attributes:
+      label: Community Interest
+      description: How much demand do you think there is for this provider?
+      options:
+        - High - Many users would benefit
+        - Medium - Some users would benefit
+        - Low - Niche use case
+    validations:
+      required: true
+
+  - type: textarea
+    id: use-case
+    attributes:
+      label: Your Use Case
+      description: Why do you want to use this provider with Forge?
+      placeholder: I would use this provider because...
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: contribution
+    attributes:
+      label: Contribution
+      description: Are you willing to help with this integration?
+      options:
+        - label: I'm willing to submit a PR to implement this integration
+        - label: I can help with testing
+        - label: I have access to this provider's API for testing
+
+  - type: textarea
+    id: additional
+    attributes:
+      label: Additional Context
+      description: Any other information about this provider that would be helpful

.github/contribution.md

@@ -0,0 +1,69 @@
+# Code-Forge 🛠️
+
+A sophisticated AI-powered coding assistant platform built in Rust, designed to provide intelligent code generation, manipulation, and analysis capabilities through a modular and extensible architecture.
+
+## ✨ Features
+
+- 🤖 **AI-Powered Code Generation** - Advanced code generation and manipulation using modern AI models
+- 🔍 **Smart Code Analysis** - Language-aware parsing and analysis for multiple programming languages
+- 🛠️ **Extensive Tool System** - Rich set of development tools including file operations, shell commands, and code outline generation
+- 💾 **Persistent Conversations** - Maintain context and history across coding sessions  
+- 🔒 **Secure Operations** - Built-in security measures for file system and shell operations
+- 🔌 **Extensible Architecture** - Modular design supporting easy addition of new features and languages
+
+## 🚀 Setup
+
+### Prerequisites
+
+- Rust toolchain (1.75+)
+- SQLite
+- Tree-sitter (for code analysis)
+
+### Installation
+
+```bash
+# Build the project
+cargo build --release
+
+# Run the server
+cargo run --release
+```
+
+## 🏗️ Project Structure
+
+```
+code-forge/
+├── crates/
+│   ├── forge_main/        # CLI and main application logic
+│   ├── forge_domain/      # Core domain models and interfaces
+│   ├── forge_services/      # HTTP API and database management
+│   ├── forge_tool/        # Tool implementations
+│   └── forge_walker/      # File system operations
+```
+
+## 🛠️ Core Components
+
+- **Domain Layer** (`forge_domain`) - Core business logic and interfaces
+- **Tool Layer** (`forge_tool`) - Development tools implementation
+- **Server Layer** (`forge_services`) - API endpoints and persistence
+- **Main Application** (`forge_main`) - CLI and application coordination
+
+## 🔧 Configuration
+
+The application requires several environment variables for proper operation:
+
+```bash
+# Required environment variables
+DATABASE_URL="sqlite:path/to/database.db"
+OPENROUTER_API_KEY="your-api-key"
+```
+
+## 📚 Documentation
+
+Internal documentation:
+- [Onboarding Guide](docs/onboarding.md)
+- [Architecture Overview](docs/architecture.md)
+
+## 🔒 Proprietary Software
+
+This is proprietary software. All rights reserved.

.github/dependabot.yml

@@ -0,0 +1,26 @@
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/optimizing-pr-creation-version-updates#setting-up-a-cooldown-period-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: cargo
+    directory: /
+    schedule:
+      interval: weekly
+    labels: ["type: chore"]
+    # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference#groups--
+    groups:
+      major:
+        update-types: ["major"]
+      minor:
+        update-types: ["minor"]
+      patch:
+        update-types: ["patch"]
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    labels: ["type: chore"]
+    groups:
+      actions:
+        # Combine all images of last week
+        patterns: ["*"]

.github/labels.json

@@ -0,0 +1,349 @@
+[
+  {
+    "name": "release: skip changelog",
+    "color": "d73a4a",
+    "description": "Skip this change from release notes.",
+    "aliases": ["skip"]
+  },
+  {
+    "name": "release: breaking",
+    "color": "d73a4a",
+    "description": "Introduces a breaking change.",
+    "aliases": [
+      "breaking change",
+      "breaking-change",
+      "Type: Breaking Change",
+      "type: breaking",
+      "breaking"
+    ]
+  },
+
+  {
+    "name": "good first issue",
+    "color": "7057ff",
+    "description": "Good for newcomers.",
+    "aliases": [
+      "beginner-friendly",
+      "beginner",
+      "good for beginner",
+      "Good for beginners",
+      "good-starter-issue",
+      "starter-issue",
+      "starter",
+      "status: good starter issue"
+    ]
+  },
+  {
+    "name": "help",
+    "color": "0e8a16",
+    "aliases": [
+      "help wanted",
+      "Patch Welcome",
+      "status: help wanted",
+      "Status: PR Welcome",
+      "Status: Ready for PR"
+    ]
+  },
+  {
+    "name": "state: approved",
+    "color": "C3F7F7",
+    "description": "Approved to proceed.",
+    "aliases": ["approved"]
+  },
+  {
+    "name": "state: blocked",
+    "color": "C3F7F7",
+    "description": "Something is blocking action.",
+    "aliases": ["blocked", "Status: Blocked", "status: blocked", "blocker"]
+  },
+  {
+    "name": "state: pending",
+    "color": "C3F7F7",
+    "description": "Action is underway but awaiting other tasks or data before completion.",
+    "aliases": [
+      "in progress",
+      "on hold",
+      "pending",
+      "Status: In Progress",
+      "Status: Review Needed",
+      "watchlist"
+    ]
+  },
+  {
+    "name": "state: inactive",
+    "color": "C3F7F7",
+    "description": "No current action needed/possible; issue fixed, out of scope, or superseded.",
+    "aliases": [
+      "duplicate",
+      "invalid",
+      "stale",
+      "Status: Abandoned",
+      "status: duplicate",
+      "status: wontfix",
+      "wont-fix",
+      "wontfix"
+    ]
+  },
+  {
+    "name": "type: bug",
+    "color": "FA1DD4",
+    "description": "Something isn't working.",
+    "aliases": ["bug", "Type: Bug"]
+  },
+  {
+    "name": "type: chore",
+    "color": "FA1DD4",
+    "description": "Routine tasks like conversions, reorganization, and maintenance work.",
+    "aliases": [
+      "chore",
+      "dependabot",
+      "dependencies",
+      "dependency",
+      "greenkeeper",
+      "legal",
+      "maintenance",
+      "Type: Maintenance",
+      "type: maintenance"
+    ]
+  },
+  {
+    "name": "type: discussion",
+    "color": "FA1DD4",
+    "description": "Questions, proposals and info that requires discussion.",
+    "aliases": [
+      "discussion",
+      "feedback",
+      "help wanted",
+      "idea",
+      "Idea",
+      "Proposal",
+      "proposal",
+      "question",
+      "type: design",
+      "Type: Question",
+      "type: question"
+    ]
+  },
+  {
+    "name": "type: docs",
+    "color": "FA1DD4",
+    "description": "Related to documentation and information.",
+    "aliases": [
+      "doc",
+      "docs",
+      "document",
+      "documentation",
+      "documents",
+      "Type: Documentation",
+      "type: documentation",
+      "specification"
+    ]
+  },
+  {
+    "name": "type: feature",
+    "color": "FA1DD4",
+    "description": "Brand new functionality, features, pages, workflows, endpoints, etc.",
+    "aliases": [
+      "addition",
+      "enhancement",
+      "feature",
+      "type: enhancement",
+      "Type: Feature",
+      "performance"
+    ]
+  },
+  {
+    "name": "type: fix",
+    "color": "FA1DD4",
+    "description": "Iterations on existing features or infrastructure.",
+    "aliases": [
+      "fix",
+      "improvement",
+      "optimization",
+      "Type: Fix",
+      "bug"
+    ]
+  },
+  {
+    "name": "type: refactor",
+    "color": "FA1DD4",
+    "description": "Code refactoring and restructuring.",
+    "aliases": [
+      "refactor",
+      "Type: Refactoring",
+      "refactoring"
+    ]
+  },
+  {
+    "name": "type: security",
+    "color": "FA1DD4",
+    "description": "Something is vulnerable or not secure.",
+    "aliases": ["security"]
+  },
+  {
+    "name": "type: performance",
+    "color": "FA1DD4",
+    "description": "Improved performance."
+  },
+  {
+    "name": "type: testing",
+    "color": "FA1DD4",
+    "description": "Related to testing.",
+    "aliases": ["test", "testing"]
+  },
+  {
+    "name": "work: chaotic",
+    "color": "0288d1",
+    "description": "The situation is chaotic, novel practices used.",
+    "aliases": ["chaotic"]
+  },
+  {
+    "name": "work: complex",
+    "color": "4fc3f7",
+    "description": "The situation is complex, emergent practices used.",
+    "aliases": ["complex"]
+  },
+  {
+    "name": "work: complicated",
+    "color": "81d4fa",
+    "description": "The situation is complicated, good practices used.",
+    "aliases": ["complicated"]
+  },
+  {
+    "name": "work: obvious",
+    "color": "e1f5fe",
+    "description": "The situation is obvious, best practices used.",
+    "aliases": ["obvious"]
+  },
+  {
+    "name": "ci: lint",
+    "color": "F9D0C4",
+    "description": "Automatically fix the linters issues and make a commit",
+    "aliases": ["ci:lint"]
+  },
+  {
+    "name": "ci: benchmark",
+    "color": "F9D0C4",
+    "description": "Runs benchmarks",
+    "aliases": ["ci:benchmark"]
+  },
+  {
+    "name": "type: provider",
+    "color": "0075ca",
+    "description": "Updates provider.json configuration.",
+    "aliases": ["provider", "update: provider"]
+  },
+  {
+    "name": "ci: test-jit",
+    "color": "F9D0C4",
+    "description": "Run all integration tests for the JIT optimized engine",
+    "aliases": ["ci:test-jit"]
+  },
+  {
+    "name": "os: windows",
+    "color": "0078d4",
+    "description": "Windows-specific issue or feature.",
+    "aliases": ["windows", "win32", "platform: windows"]
+  },
+  {
+    "name": "onlydust-wave",
+    "color": "FF6D00",
+    "description": "Related to OnlyDust Wave Hackathon"
+  },
+  {
+    "name": "ci: build all targets",
+    "color": "F9D0C4",
+    "description": "Build release binaries for all target platforms in CI",
+    "aliases": ["ci:build-all-targets", "build-all-targets"]
+  },
+  {
+    "name": "bounty",
+    "color": "F9C74F",
+    "description": "This issue has a bounty attached.",
+    "aliases": ["bounty:open"]
+  },
+  {
+    "name": "bounty: $100",
+    "color": "F9C74F",
+    "description": "This issue has a $100 bounty attached.",
+    "aliases": ["bounty:100", "bounty: 💰 $100"]
+  },
+  {
+    "name": "bounty: $200",
+    "color": "F9C74F",
+    "description": "This issue has a $200 bounty attached.",
+    "aliases": ["bounty:200", "bounty: 💰 $200"]
+  },
+  {
+    "name": "bounty: $300",
+    "color": "F9C74F",
+    "description": "This issue has a $300 bounty attached.",
+    "aliases": ["bounty:300", "bounty: 💰 $300"]
+  },
+  {
+    "name": "bounty: $500",
+    "color": "F9C74F",
+    "description": "This issue has a $500 bounty attached.",
+    "aliases": ["bounty:500", "bounty: 💰 $500"]
+  },
+  {
+    "name": "bounty: $800",
+    "color": "F9C74F",
+    "description": "This issue has a $800 bounty attached.",
+    "aliases": ["bounty:800", "bounty: 💰 $800"]
+  },
+  {
+    "name": "bounty: $1300",
+    "color": "F9C74F",
+    "description": "This issue has a $1300 bounty attached.",
+    "aliases": ["bounty:1300", "bounty: 💰 $1300"]
+  },
+  {
+    "name": "bounty: $2100",
+    "color": "F9C74F",
+    "description": "This issue has a $2100 bounty attached.",
+    "aliases": ["bounty:2100", "bounty: 💰 $2100"]
+  },
+  {
+    "name": "bounty: $3400",
+    "color": "F9C74F",
+    "description": "This issue has a $3400 bounty attached.",
+    "aliases": ["bounty:3400", "bounty: 💰 $3400"]
+  },
+  {
+    "name": "bounty: $5500",
+    "color": "F9C74F",
+    "description": "This issue has a $5500 bounty attached.",
+    "aliases": ["bounty:5500", "bounty: 💰 $5500"]
+  },
+  {
+    "name": "bounty: claimed",
+    "color": "F9C74F",
+    "description": "This bounty has been claimed and is being worked on.",
+    "aliases": ["bounty:claimed"]
+  },
+  {
+    "name": "bounty: rewarded",
+    "color": "F9C74F",
+    "description": "This bounty has been completed and the reward paid out.",
+    "aliases": ["bounty:rewarded", "bounty:earned"]
+  },
+  {
+    "name": "severity: low",
+    "color": "FFEB3B",
+    "description": "Minor impact; does not significantly affect functionality.",
+    "aliases": ["severity:low"]
+  },
+  {
+    "name": "severity: high",
+    "color": "FF9800",
+    "description": "Significant impact; core functionality is impaired.",
+    "aliases": ["severity:high"]
+  },
+  {
+    "name": "severity: critical",
+    "color": "B71C1C",
+    "description": "Severe impact; system is unusable or data is at risk.",
+    "aliases": ["severity:critical", "critical"]
+  }
+]

.github/release-drafter.yml

@@ -0,0 +1,132 @@
+name-template: "v$RESOLVED_VERSION"
+tag-template: "v$RESOLVED_VERSION"
+categories:
+  - title: "🚀 Features"
+    labels:
+      - "type: feature"
+
+  - title: "📝 Documentation"
+    labels:
+      - "type: docs"
+
+  - title: "🐛 Bug Fixes"
+    labels:
+      - "type: bug"
+      - "type: fix"
+      - "type: provider"
+
+  - title: "🚀 Performance"
+    labels:
+      - "type: performance"
+
+  - title: "🧰 Maintenance"
+    labels:
+      - "type: chore"
+      - "type: testing"
+      - "type: refactor"
+      - "ratatui"
+
+  - title: "💥 Breaking Changes"
+    labels:
+      - "release: breaking"
+
+  - title: "🔒 Security"
+    labels:
+      - "type: security"
+
+change-template: "- $TITLE @$AUTHOR (#$NUMBER)"
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+
+version-resolver:
+  major:
+    labels:
+      - "release: breaking"
+  minor:
+    labels:
+      - "type: feature"
+    # Add direct commit message pattern matching for features
+    commits:
+      - "feat:"
+      - "feature:"
+  patch:
+    labels:
+      - "type: fix"
+      - "type: performance"
+      - "type: chore"
+      - "type: docs"
+      - "type: bug"
+      - "type: testing"
+      - "type: refactor"
+      - "type: provider"
+    # Add direct commit message pattern matching for patches
+    commits:
+      - "fix:"
+      - "perf:"
+      - "chore:"
+      - "docs:"
+      - "refactor:"
+  default: patch
+
+exclude-labels:
+  - "release: skip changelog"
+
+template: |
+  ## Changes
+
+  $CHANGES
+autolabeler:
+  - label: "type: chore"
+    title:
+      - /chore/i
+
+  - label: "type: refactor"
+    title:
+      - /refactor/i
+      - /refactoring/i
+
+  - label: "type: fix"
+    title:
+      - /fix/i
+      - /hotfix/i
+      - /bugfix/i
+      - /patch/i
+
+  - label: "type: feature"
+    title:
+      - /feat/i
+      - /feature/i
+      - /enhancement/i
+      - /new/i
+
+  - label: "type: docs"
+    title:
+      - /doc/i
+      - /docs/i
+
+  - label: "type: performance"
+    title:
+      - /perf/i
+
+  - label: "ci: benchmark"
+    title:
+      - /perf/i
+    files:
+      - "/benches/"
+
+  - label: "type: provider"
+    files:
+      - "provider.json"
+
+  - label: "os: windows"
+    title:
+      - /windows/i
+      - /win32/i
+      - /winapi/i
+      - /powershell/i
+      - /scoop/i
+      - /chocolatey/i
+      - /winget/i
+    body:
+      - /windows[ -](only|specific|terminal|path|installer|build)/i
+      - /multiline paste.*windows/i
+      - /windows.*multiline paste/i

.github/scripts/bounty/README.md

@@ -0,0 +1,155 @@
+# Bounty Automation
+
+Automates the full lifecycle of issue bounties — from label propagation when a PR is opened, through claiming when work begins, to rewarding when a PR is merged.
+
+## Flow
+
+```
+Issue created
+└── maintainer adds  bounty: $N  label
+        │
+        ├── parse-sync-generic-bounty.ts  →  adds generic  bounty  label
+        │
+        ▼
+Issue assigned to contributor
+└── parse-sync-claimed.ts  →  adds  bounty: claimed
+        │
+        ▼
+Contributor opens PR with "Closes #N" / "Fixes #N" / "Resolves #N"
+└── parse-propagate-label.ts  →  copies  bounty: $N  to PR
+                              →  posts comment on issue: "PR #X opened by @author"
+        │
+        ▼
+PR is merged
+└── parse-mark-rewarded.ts  →  adds  bounty: rewarded  to PR
+                            →  adds  bounty: rewarded  to linked issue(s)
+                            →  removes  bounty: claimed  from linked issue(s)
+        │
+        ▼
+Bounty lifecycle complete
+
+Removal path:
+maintainer removes last  bounty: $N  label
+└── parse-sync-generic-bounty.ts  →  removes generic  bounty  label
+```
+
+## Labels
+
+| Label                            | Applied to | Set by                                                      |
+| -------------------------------- | ---------- | ----------------------------------------------------------- |
+| `bounty: $100` … `bounty: $5500` | Issue      | Maintainer (manually)                                       |
+| `bounty`                         | Issue      | `parse-sync-generic-bounty.ts` on value label add/remove    |
+| `bounty: claimed`                | Issue      | `parse-sync-claimed.ts` on assignment                       |
+| `bounty: rewarded`               | Issue + PR | `parse-mark-rewarded.ts` on merge                           |
+
+Bounty values follow the Fibonacci sequence: **$100, $200, $300, $500, $800, $1300, $2100, $3400, $5500**.
+
+## Pipeline
+
+Each workflow job runs a single Node process (`run.ts`) that executes all three stages in memory — no temp files, no shell piping:
+
+```
+parse  →  ParsedIntent (in memory)  →  plan  →  BatchPlan (in memory)  →  execute
+```
+
+**Stage 1 — parse** (`parse-*.ts`): Reads the GitHub Actions event payload from `GITHUB_EVENT_PATH` and returns a `ParsedIntent` object. Pure — makes no API calls.
+
+**Stage 2 — plan** (`plan.ts`): Receives the `ParsedIntent`, fetches current labels only for targets not already known from the event payload, diffs desired vs actual state, and returns a `BatchPlan`. Minimises API calls by reusing label data already present in the event.
+
+**Stage 3 — execute** (`execute.ts`): Receives the `BatchPlan` and applies all mutations. Label additions per target are sent as a single batched `POST /labels` call. Each removal is a separate `DELETE` (GitHub has no bulk-remove endpoint). Comments are posted last.
+
+This design means:
+- The parse stage is trivially unit-testable (pure function, no mocks needed).
+- The plan stage is testable with a minimal mock that only needs `getLabels`.
+- The execute stage is testable with a mock that tracks calls — no HTTP.
+- API calls are minimised: additions are batched per target; label state already in the event payload is never re-fetched.
+
+## Scripts
+
+The workflow invokes a single orchestrator script (`run.ts`) per job. The parse, plan, and execute modules are called directly in the same Node process, passing objects in memory.
+
+### `run.ts`
+
+The entrypoint used by `bounty.yml`. Accepts a `--script` flag to select which parse module to run, then calls `plan` and `execute` in sequence — all in-process.
+
+```sh
+npx tsx .github/scripts/bounty/run.ts \
+  --script <parse-script> \
+  --repo <owner/repo> \
+  --token <github-token> \
+  [--pr <number> | --issue <number>]
+```
+
+### `parse-propagate-label.ts`
+
+Triggered by: `pull_request` — opened, edited, reopened.
+
+1. Parses the PR body for closing keywords (`closes`, `fixes`, `resolves`, case-insensitive).
+2. Returns a `ParsedIntent` with a `labelCopies` field — the plan stage fetches each linked issue's labels and copies any `bounty: $N` ones onto the PR.
+3. Includes a comment mutation per linked issue (the plan stage drops it if the issue has no bounty labels).
+
+### `parse-sync-claimed.ts`
+
+Triggered by: `issues` — assigned, unassigned.
+
+- **assigned**: returns `add: ["bounty: claimed"]` if the issue has a `bounty: $N` label.
+- **unassigned**: returns `remove: ["bounty: claimed"]` only when no assignees remain.
+- Issue labels are already in the event payload and supplied as `knownLabels` — no extra fetch.
+
+### `parse-sync-generic-bounty.ts`
+
+Triggered by: `issues` — labeled, unlabeled.
+
+Keeps the generic `bounty` label in sync with value labels. Inspects `event.label` (the label that just changed) and only acts when it matches `bounty: $`.
+
+- **labeled**: returns `add: ["bounty"]`.
+- **unlabeled**: returns `remove: ["bounty"]` only when no value labels remain (guards against mid-tier-swap removal when a maintainer swaps one value label for another).
+- Issue labels from the event are supplied as `knownLabels` — no extra fetch.
+
+### `parse-mark-rewarded.ts`
+
+Triggered by: `pull_request_target` — closed (merged only).
+
+1. Returns empty intent if the PR was not merged or has no `bounty: $N` label.
+2. Returns `add: ["bounty: rewarded"]` for the PR (labels known from event, no fetch).
+3. Parses the PR body for linked issues; returns `add: ["bounty: rewarded"], remove: ["bounty: claimed"]` for each (plan stage fetches their labels).
+
+Uses `pull_request_target` so the job has write access to issues and PRs from forks.
+
+### `plan.ts`
+
+Receives a `ParsedIntent` from the parse stage. Resolves `labelCopies` by fetching source issue labels. Fetches current labels for any target not already in `knownLabels`. Filters out no-op adds and removes. Returns a `BatchPlan`.
+
+### `execute.ts`
+
+Receives a `BatchPlan` from the plan stage. For each mutation: one batched `POST` for all additions, one `DELETE` per removal, one `POST` per comment.
+
+## Shared Module
+
+`github-api.ts` defines:
+- Event payload types (`PullRequestEvent`, `IssuesEvent`)
+- Pipeline types (`ParsedIntent`, `BatchPlan`, `TargetMutation`)
+- The `GitHubApi` interface (injectable for testing)
+- `GitHubRestApi` — the production implementation using `node:https`
+
+All scripts import types from `github-api.ts` and accept a `GitHubApi` instance in their `run()` / `plan()` / `execute()` function signature, making every step independently mockable.
+
+## Tests
+
+Unit tests live alongside each script (`*.test.ts`) and use Node's built-in `node:test` runner.
+
+```sh
+npm run test:bounty
+```
+
+- Parse tests: pure — no mock needed, just call `parse()` with a synthetic event.
+- Plan and execute tests: use a mock `GitHubApi` that tracks calls and returns preset label lists.
+- The CLI entrypoint in each script (yargs parsing + `GITHUB_EVENT_PATH` read) is guarded behind an `import.meta.url` check so it does not execute on import.
+
+## Workflow Source
+
+`bounty.yml` is auto-generated from Rust source in `crates/forge_ci`. Do not edit it by hand — modify `crates/forge_ci/src/workflows/bounty.rs` and regenerate with:
+
+```sh
+cargo test -p forge_ci
+```

.github/scripts/bounty/src/api.ts

@@ -0,0 +1,110 @@
+// GitHub API abstraction for v2 bounty sync.
+// Uses the native `fetch` API (available in Node 18+).
+
+import type { Issue, Label, PullRequest } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Interface — injectable for testing
+// ---------------------------------------------------------------------------
+
+/// Minimal GitHub REST API surface needed by the bounty sync commands.
+export interface GitHubApi {
+  /// Fetch a full issue by number. Throws on non-2xx.
+  getIssue(number: number): Promise<Issue>;
+  /// Fetch a full pull request by number. Throws on non-2xx.
+  getPullRequest(number: number): Promise<PullRequest>;
+  /// Fetch all open issues that have any label matching the given prefix.
+  /// Handles pagination automatically and excludes pull requests.
+  listIssuesWithLabelPrefix(prefix: string): Promise<Issue[]>;
+  /// Add one or more labels to an issue or PR. Batched into a single request.
+  addLabels(target: number, labels: string[]): Promise<void>;
+  /// Remove a single label from an issue or PR.
+  removeLabel(target: number, label: string): Promise<void>;
+  /// Post a comment on an issue or PR.
+  addComment(target: number, body: string): Promise<void>;
+}
+
+// ---------------------------------------------------------------------------
+// Production implementation
+// ---------------------------------------------------------------------------
+
+/// Calls the real GitHub REST API v3 using `fetch`.
+export class GitHubRestApi implements GitHubApi {
+  private readonly base: string;
+  private readonly headers: Record<string, string>;
+
+  constructor(
+    private readonly owner: string,
+    private readonly repo: string,
+    token: string
+  ) {
+    this.base = `https://api.github.com/repos/${owner}/${repo}`;
+    this.headers = {
+      Authorization: `Bearer ${token}`,
+      Accept: "application/vnd.github+json",
+      "X-GitHub-Api-Version": "2022-11-28",
+      "User-Agent": "bounty-bot/v2",
+      "Content-Type": "application/json",
+    };
+  }
+
+  private async request<T>(method: string, path: string, body?: unknown): Promise<T> {
+    const res = await fetch(`${this.base}${path}`, {
+      method,
+      headers: this.headers,
+      body: body !== undefined ? JSON.stringify(body) : undefined,
+    });
+
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`GitHub API ${method} ${path} → ${res.status}: ${text}`);
+    }
+
+    // 204 No Content
+    if (res.status === 204) return {} as T;
+
+    return res.json() as Promise<T>;
+  }
+
+  async getIssue(number: number): Promise<Issue> {
+    return this.request<Issue>("GET", `/issues/${number}`);
+  }
+
+  async getPullRequest(number: number): Promise<PullRequest> {
+    return this.request<PullRequest>("GET", `/pulls/${number}`);
+  }
+
+  async listIssuesWithLabelPrefix(prefix: string): Promise<Issue[]> {
+    const results: Issue[] = [];
+    let page = 1;
+    while (true) {
+      const batch = await this.request<Issue[]>(
+        "GET",
+        `/issues?state=open&per_page=100&page=${page}`
+      );
+      if (batch.length === 0) break;
+      for (const issue of batch) {
+        // Exclude pull requests (GitHub returns them in /issues)
+        if (issue.pull_request !== undefined) continue;
+        if (issue.labels.some((l) => l.name.startsWith(prefix))) {
+          results.push(issue);
+        }
+      }
+      if (batch.length < 100) break;
+      page++;
+    }
+    return results;
+  }
+
+  async addLabels(target: number, labels: string[]): Promise<void> {
+    await this.request("POST", `/issues/${target}/labels`, { labels });
+  }
+
+  async removeLabel(target: number, label: string): Promise<void> {
+    await this.request("DELETE", `/issues/${target}/labels/${encodeURIComponent(label)}`);
+  }
+
+  async addComment(target: number, body: string): Promise<void> {
+    await this.request("POST", `/issues/${target}/comments`, { body });
+  }
+}

.github/scripts/bounty/src/rules.ts

@@ -0,0 +1,187 @@
+// Rules engine for v2 bounty sync.
+//
+// Both exported functions are pure: they take current state and return the
+// minimal Patch needed to reconcile it with the desired state.
+// No I/O, no side effects — fully unit-testable without mocks.
+
+import type { IssueState, PrState, Patch, LabelOp, Issue } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Label name constants
+// ---------------------------------------------------------------------------
+
+export const BOUNTY_GENERIC = "bounty";
+export const BOUNTY_CLAIMED = "bounty: claimed";
+export const BOUNTY_REWARDED = "bounty: rewarded";
+
+/// Matches bounty value labels in both formats:
+/// - new: "bounty: 💰 $100", "bounty: 💰 $5500"
+/// - legacy: "bounty: $300", "bounty: $500"
+export const VALUE_LABEL_RE = /^bounty: (?:💰 )?\$\d/;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/// Returns true when the label name is a bounty value label.
+export function isBountyValue(label: string): boolean {
+  return VALUE_LABEL_RE.test(label);
+}
+
+/// Extract all issue numbers referenced by closing keywords in a PR body.
+/// A new RegExp is created on each call to avoid stale `lastIndex` state.
+export function linkedIssueNumbers(body: string | null): number[] {
+  const numbers: number[] = [];
+  const text = body ?? "";
+  const re = /(?:closes?|fix(?:es)?|resolves?)\s+#(\d+)/gi;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(text)) !== null) {
+    numbers.push(parseInt(m[1]!, 10));
+  }
+  return numbers;
+}
+
+/// Build a LabelOp that only includes labels that actually need to change.
+function diff(
+  target: number,
+  current: Set<string>,
+  desired: Set<string>,
+  comment?: string,
+  meta?: { title?: string; url?: string }
+): LabelOp | null {
+  const add = [...desired].filter((l) => !current.has(l));
+  const remove = [...current].filter((l) => !desired.has(l));
+  if (add.length === 0 && remove.length === 0 && !comment) return null;
+  return { target, title: meta?.title, url: meta?.url, add, remove, comment };
+}
+
+// ---------------------------------------------------------------------------
+// Issue rules
+// ---------------------------------------------------------------------------
+
+/// Compute the desired label set for an issue given its full current state.
+///
+/// Rules (all applied simultaneously, not sequentially):
+///
+/// 1. **Generic bounty** — `bounty` must be present iff any value label is present.
+/// 2. **Claimed** — `bounty: claimed` must be present iff the issue has a value
+///    label AND at least one assignee. It must be absent otherwise.
+/// 3. **Value labels and `bounty: rewarded`** — these are set by maintainers or
+///    the PR merge job and are left untouched by the issue sync.
+/// 4. All non-bounty labels are preserved as-is.
+export function computeIssuePatch({ issue, currentLabels }: IssueState): Patch {
+  const desired = new Set(currentLabels);
+
+  const hasValueLabel = [...currentLabels].some(isBountyValue);
+  const isAssigned = issue.assignees.length > 0;
+
+  // Rule 1 — generic label mirrors value label presence.
+  if (hasValueLabel) {
+    desired.add(BOUNTY_GENERIC);
+  } else {
+    desired.delete(BOUNTY_GENERIC);
+  }
+
+  // Rule 2 — claimed mirrors assignment presence (only when bounty exists).
+  if (hasValueLabel && isAssigned) {
+    desired.add(BOUNTY_CLAIMED);
+  } else {
+    desired.delete(BOUNTY_CLAIMED);
+  }
+
+  const op = diff(issue.number, currentLabels, desired, undefined, {
+    title: issue.title,
+    url: issue.html_url,
+  });
+  return { ops: op ? [op] : [] };
+}
+
+// ---------------------------------------------------------------------------
+// PR rules
+// ---------------------------------------------------------------------------
+
+/// Compute the desired state for a PR and its linked issues.
+///
+/// Rules:
+///
+/// 1. **Label propagation** — for each linked issue that has a value label,
+///    copy that value label to the PR (if the PR doesn't already have it).
+/// 2. **Rewarded on merge** — if the PR is merged and has a value label:
+///    a. Add `bounty: rewarded` to the PR.
+///    b. Add `bounty: rewarded` to each linked issue.
+///    c. Remove `bounty: claimed` from each linked issue.
+/// 3. **Comment on propagation** — when a value label is first added to the PR
+///    via rule 1, post a comment on the source issue.
+export function computePrPatch({ pr, currentLabels, linkedIssues }: PrState): Patch {
+  const ops: LabelOp[] = [];
+
+  const prDesired = new Set(currentLabels);
+
+  // Collect value labels to propagate from linked issues.
+  const labelsToPropagate: string[] = [];
+  for (const issue of linkedIssues) {
+    const issueValueLabels = issue.labels.map((l) => l.name).filter(isBountyValue);
+    for (const label of issueValueLabels) {
+      if (!prDesired.has(label)) {
+        labelsToPropagate.push(label);
+        prDesired.add(label);
+      }
+    }
+  }
+
+  const prHasValueLabel = [...prDesired].some(isBountyValue);
+
+  if (pr.merged && prHasValueLabel) {
+    // Rule 2 — rewarded lifecycle.
+    prDesired.add(BOUNTY_REWARDED);
+
+    const prOp = diff(pr.number, currentLabels, prDesired, undefined, {
+      title: pr.title,
+      url: pr.html_url,
+    });
+    if (prOp) ops.push(prOp);
+
+    // Update linked issues — only those that already have a bounty value label.
+    for (const issue of linkedIssues) {
+      const issueCurrent = new Set(issue.labels.map((l) => l.name));
+      const issueHasValue = [...issueCurrent].some(isBountyValue);
+      if (!issueHasValue) continue;
+
+      const issueDesired = new Set(issueCurrent);
+      issueDesired.add(BOUNTY_REWARDED);
+      issueDesired.delete(BOUNTY_CLAIMED);
+
+      const op = diff(issue.number, issueCurrent, issueDesired, undefined, {
+        title: issue.title,
+        url: issue.html_url,
+      });
+      if (op) ops.push(op);
+    }
+  } else {
+    // Rule 1 — label propagation (pre-merge).
+    const prOp = diff(pr.number, currentLabels, prDesired, undefined, {
+      title: pr.title,
+      url: pr.html_url,
+    });
+    if (prOp) ops.push(prOp);
+
+    // Rule 3 — comment on each issue whose value label was newly propagated.
+    if (labelsToPropagate.length > 0) {
+      for (const issue of linkedIssues) {
+        const hadValueLabel = issue.labels.map((l) => l.name).some(isBountyValue);
+        if (hadValueLabel) {
+          ops.push({
+            target: issue.number,
+            title: issue.title,
+            url: issue.html_url,
+            add: [],
+            remove: [],
+            comment: `PR [#${pr.number}](${pr.html_url}) has been opened for this bounty by @${pr.user.login}.`,
+          });
+        }
+      }
+    }
+  }
+
+  return { ops };
+}

.github/scripts/bounty/src/sync-all-issues.ts

@@ -0,0 +1,77 @@
+#!/usr/bin/env tsx
+// Syncs bounty labels across ALL open issues that carry any "bounty" label.
+//
+// Without --execute: fetches all matching issues, computes the combined patch,
+// and prints a plan showing exactly what would change. No writes are made.
+//
+// With --execute: fetches, computes, and applies the patch for every issue.
+//
+// Usage:
+//   tsx sync-all-issues.ts --repo <owner/repo> --token <token> [--execute]
+
+import * as url from "url";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { GitHubRestApi, type GitHubApi } from "./api.js";
+import { computeIssuePatch } from "./rules.js";
+import { applyPatch, printPlan, resolveToken } from "./sync-issue.js";
+import type { Patch } from "./types.js";
+
+const BOUNTY_LABEL_PREFIX = "bounty";
+
+export interface PlanAllIssuesInput {
+  api: GitHubApi;
+}
+
+/// Fetches all open issues with any bounty label and computes the combined patch.
+/// Makes no writes — safe to call at any time.
+export async function planAllIssues({ api }: PlanAllIssuesInput): Promise<Patch> {
+  const issues = await api.listIssuesWithLabelPrefix(BOUNTY_LABEL_PREFIX);
+  const ops = issues.flatMap((issue) => {
+    const currentLabels = new Set(issue.labels.map((l) => l.name));
+    return computeIssuePatch({ issue, currentLabels }).ops;
+  });
+  return { ops };
+}
+
+/// Fetches, computes, and applies the label patch for all bounty issues.
+/// Returns the patch that was applied.
+export async function syncAllIssues({ api }: PlanAllIssuesInput): Promise<Patch> {
+  const patch = await planAllIssues({ api });
+  await applyPatch(patch, api);
+  return patch;
+}
+
+// ---------------------------------------------------------------------------
+// CLI entrypoint
+// ---------------------------------------------------------------------------
+
+if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
+  const argv = await yargs(hideBin(process.argv))
+    .option("repo", { type: "string", demandOption: true, description: "owner/repo" })
+    .option("token", {
+      type: "string",
+      description: "GitHub token (falls back to GITHUB_TOKEN env var or `gh auth token`)",
+    })
+    .option("execute", {
+      type: "boolean",
+      default: false,
+      description: "Apply the patch. Without this flag only the plan is printed.",
+    })
+    .strict()
+    .parseAsync();
+
+  const [owner, repo] = argv.repo.split("/") as [string, string];
+  const token = resolveToken(argv.token);
+  const api = new GitHubRestApi(owner, repo, token);
+
+  if (argv.execute) {
+    const patch = await syncAllIssues({ api });
+    if (patch.ops.length === 0) {
+      console.log("All issues already in sync — no changes needed.");
+    }
+  } else {
+    const patch = await planAllIssues({ api });
+    printPlan(patch, "All bounty issues");
+  }
+}

.github/scripts/bounty/src/sync-issue.ts

@@ -0,0 +1,133 @@
+#!/usr/bin/env tsx
+// Syncs all bounty labels on a single issue.
+//
+// Without --execute: fetches current state, computes the patch, and prints
+// a plan showing exactly what would change. No writes are made.
+//
+// With --execute: fetches, computes, and applies the patch.
+//
+// Usage:
+//   tsx sync-issue.ts --issue <number> --repo <owner/repo> --token <token> [--execute]
+
+import * as url from "url";
+import { execSync } from "child_process";
+import chalk from "chalk";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { GitHubRestApi, type GitHubApi } from "./api.js";
+import { computeIssuePatch } from "./rules.js";
+import type { Patch } from "./types.js";
+
+/// Resolves a GitHub token from the provided value, the GITHUB_TOKEN env var,
+/// or by invoking `gh auth token` as a fallback. Exits with an error if none
+/// can be found.
+export function resolveToken(flag: string | undefined): string {
+  const token = flag || process.env["GITHUB_TOKEN"] || (() => {
+    try {
+      return execSync("gh auth token", { encoding: "utf8" }).trim();
+    } catch {
+      return "";
+    }
+  })();
+  if (!token) {
+    console.error(
+      "Error: no GitHub token found. Pass --token, set GITHUB_TOKEN, or run `gh auth login`."
+    );
+    process.exit(1);
+  }
+  return token;
+}
+
+export interface PlanIssueInput {
+  issueNumber: number;
+  api: GitHubApi;
+}
+
+/// Fetches the current issue state and computes the minimal label patch.
+/// Makes no writes — safe to call at any time.
+export async function planIssue({ issueNumber, api }: PlanIssueInput): Promise<Patch> {
+  const issue = await api.getIssue(issueNumber);
+  const currentLabels = new Set(issue.labels.map((l) => l.name));
+  return computeIssuePatch({ issue, currentLabels });
+}
+
+/// Fetches, computes, and applies the label patch for a single issue.
+/// Returns the patch that was applied (useful for tests and logging).
+export async function syncIssue({ issueNumber, api }: PlanIssueInput): Promise<Patch> {
+  const patch = await planIssue({ issueNumber, api });
+  await applyPatch(patch, api);
+  return patch;
+}
+
+/// Applies a Patch against the GitHub API.
+/// Each target gets one batched addLabels call; each removal is individual.
+export async function applyPatch(patch: Patch, api: GitHubApi): Promise<void> {
+  for (const op of patch.ops) {
+    const ref = chalk.cyan(`#${op.target}`);
+    if (op.add.length > 0) {
+      await api.addLabels(op.target, op.add);
+      console.log(`${chalk.green("✔")} ${ref}: added [${chalk.green(op.add.join(", "))}]`);
+    }
+    for (const label of op.remove) {
+      await api.removeLabel(op.target, label);
+      console.log(`${chalk.red("✖")} ${ref}: removed "${chalk.red(label)}"`);
+    }
+    if (op.comment) {
+      await api.addComment(op.target, op.comment);
+      console.log(`${chalk.yellow("✉")} ${ref}: posted comment`);
+    }
+  }
+}
+
+/// Prints a human-readable plan to stdout without making any API calls.
+export function printPlan(patch: Patch, subject: string): void {
+  if (patch.ops.length === 0) {
+    console.log(`${chalk.green("✔")} ${chalk.bold(subject)}: already in sync — no changes needed.`);
+    return;
+  }
+  console.log(`${chalk.yellow("●")} ${chalk.bold(subject)}: plan (${chalk.bold(String(patch.ops.length))} target(s) to update)\n`);
+  for (const op of patch.ops) {
+    const title = op.title ? chalk.bold(` ${op.title}`) : "";
+    const href  = op.url   ? `\n    ${chalk.dim(chalk.blue(op.url))}` : "";
+    console.log(`  ${chalk.cyan(`#${op.target}`)}${title}${href}`);
+    if (op.add.length > 0)
+      console.log(`    ${chalk.green("+")} add:     ${chalk.green(op.add.join(", "))}`);
+    if (op.remove.length > 0)
+      console.log(`    ${chalk.red("-")} remove:  ${chalk.red(op.remove.join(", "))}`);
+    if (op.comment)
+      console.log(`    ${chalk.yellow("~")} comment: ${chalk.dim(op.comment.slice(0, 80))}…`);
+  }
+  console.log(`\n${chalk.dim("Run with --execute to apply.")}`);
+}
+
+// ---------------------------------------------------------------------------
+// CLI entrypoint
+// ---------------------------------------------------------------------------
+
+if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
+  const argv = await yargs(hideBin(process.argv))
+    .option("issue", { type: "number", demandOption: true, description: "Issue number" })
+    .option("repo", { type: "string", demandOption: true, description: "owner/repo" })
+    .option("token", { type: "string", description: "GitHub token (falls back to GITHUB_TOKEN env var or `gh auth token`)" })
+    .option("execute", {
+      type: "boolean",
+      default: false,
+      description: "Apply the patch. Without this flag only the plan is printed.",
+    })
+    .strict()
+    .parseAsync();
+
+  const [owner, repo] = argv.repo.split("/") as [string, string];
+  const token = resolveToken(argv.token);
+  const api = new GitHubRestApi(owner, repo, token);
+
+  if (argv.execute) {
+    const patch = await syncIssue({ issueNumber: argv.issue, api });
+    if (patch.ops.length === 0) {
+      console.log(`Issue #${argv.issue}: already in sync, no changes needed.`);
+    }
+  } else {
+    const patch = await planIssue({ issueNumber: argv.issue, api });
+    printPlan(patch, `Issue #${argv.issue}`);
+  }
+}

.github/scripts/bounty/src/sync-pr.ts

@@ -0,0 +1,82 @@
+#!/usr/bin/env tsx
+// Syncs bounty labels on a PR (and its linked issues).
+//
+// Without --execute: fetches current state, computes the patch, and prints
+// a plan showing exactly what would change. No writes are made.
+//
+// With --execute: fetches, computes, and applies the patch.
+//
+// Usage:
+//   tsx sync-pr.ts --pr <number> --repo <owner/repo> --token <token> [--execute]
+
+import * as url from "url";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { GitHubRestApi, type GitHubApi } from "./api.js";
+import { computePrPatch, linkedIssueNumbers } from "./rules.js";
+import { applyPatch, printPlan, resolveToken } from "./sync-issue.js";
+import type { Patch } from "./types.js";
+
+export interface PlanPrInput {
+  prNumber: number;
+  api: GitHubApi;
+}
+
+/// Fetches the current PR + linked issue state and computes the minimal label patch.
+/// Makes no writes — safe to call at any time.
+export async function planPr({ prNumber, api }: PlanPrInput): Promise<Patch> {
+  const pr = await api.getPullRequest(prNumber);
+  const currentLabels = new Set(pr.labels.map((l) => l.name));
+  const issueNumbers = linkedIssueNumbers(pr.body);
+
+  const linkedIssues = await Promise.all(
+    issueNumbers.map((n) =>
+      api.getIssue(n).catch((err) => {
+        console.warn(`Could not fetch linked issue #${n}: ${String(err)}, skipping.`);
+        return null;
+      })
+    )
+  ).then((results) => results.filter((i): i is NonNullable<typeof i> => i !== null));
+
+  return computePrPatch({ pr, currentLabels, linkedIssues });
+}
+
+/// Fetches, computes, and applies the label patch for a PR and its linked issues.
+/// Returns the patch that was applied (useful for tests and logging).
+export async function syncPr({ prNumber, api }: PlanPrInput): Promise<Patch> {
+  const patch = await planPr({ prNumber, api });
+  await applyPatch(patch, api);
+  return patch;
+}
+
+// ---------------------------------------------------------------------------
+// CLI entrypoint
+// ---------------------------------------------------------------------------
+
+if (process.argv[1] === url.fileURLToPath(import.meta.url)) {
+  const argv = await yargs(hideBin(process.argv))
+    .option("pr", { type: "number", demandOption: true, description: "PR number" })
+    .option("repo", { type: "string", demandOption: true, description: "owner/repo" })
+    .option("token", { type: "string", description: "GitHub token (falls back to GITHUB_TOKEN env var or `gh auth token`)" })
+    .option("execute", {
+      type: "boolean",
+      default: false,
+      description: "Apply the patch. Without this flag only the plan is printed.",
+    })
+    .strict()
+    .parseAsync();
+
+  const [owner, repo] = argv.repo.split("/") as [string, string];
+  const token = resolveToken(argv.token);
+  const api = new GitHubRestApi(owner, repo, token);
+
+  if (argv.execute) {
+    const patch = await syncPr({ prNumber: argv.pr, api });
+    if (patch.ops.length === 0) {
+      console.log(`PR #${argv.pr}: already in sync, no changes needed.`);
+    }
+  } else {
+    const patch = await planPr({ prNumber: argv.pr, api });
+    printPlan(patch, `PR #${argv.pr}`);
+  }
+}

.github/scripts/bounty/src/types.ts

@@ -0,0 +1,82 @@
+// Core domain types for v2 bounty sync.
+//
+// Design: both sync commands work by fetching the complete current state of an
+// issue or PR from GitHub, deriving the *desired* label set from the rules, and
+// computing a minimal patch (add / remove) to reconcile the two.
+
+// ---------------------------------------------------------------------------
+// GitHub resource types (as returned by the REST API)
+// ---------------------------------------------------------------------------
+
+export interface Label {
+  name: string;
+}
+
+export interface User {
+  login: string;
+}
+
+/// Full issue as returned by GET /repos/:owner/:repo/issues/:number
+export interface Issue {
+  number: number;
+  title: string;
+  html_url: string;
+  state: "open" | "closed";
+  labels: Label[];
+  assignees: User[];
+  pull_request?: unknown; // present when the issue is actually a PR
+}
+
+/// Full pull request as returned by GET /repos/:owner/:repo/pulls/:number
+export interface PullRequest {
+  number: number;
+  title: string;
+  state: "open" | "closed";
+  merged: boolean;
+  body: string | null;
+  labels: Label[];
+  user: User;
+  html_url: string;
+}
+
+// ---------------------------------------------------------------------------
+// Derived state used by the rules engine
+// ---------------------------------------------------------------------------
+
+/// Everything the rules engine needs to know about an issue.
+export interface IssueState {
+  issue: Issue;
+  /// Current label names on the issue.
+  currentLabels: Set<string>;
+}
+
+/// Everything the rules engine needs to know about a PR and its linked issues.
+export interface PrState {
+  pr: PullRequest;
+  /// Current label names on the PR.
+  currentLabels: Set<string>;
+  /// Full state of each issue linked via "closes/fixes/resolves #N" in the PR body.
+  linkedIssues: Issue[];
+}
+
+// ---------------------------------------------------------------------------
+// Patch types — the minimal diff to apply
+// ---------------------------------------------------------------------------
+
+/// A label operation on a single target (issue or PR number).
+export interface LabelOp {
+  target: number;
+  /// Human-readable title of the issue or PR, for display in plan output.
+  title?: string;
+  /// URL of the issue or PR, for display in plan output.
+  url?: string;
+  add: string[];
+  remove: string[];
+  /// Optional comment to post on the target after label ops.
+  comment?: string;
+}
+
+/// The complete set of operations to apply.
+export interface Patch {
+  ops: LabelOp[];
+}

.github/scripts/bounty/tests/rules.test.ts

@@ -0,0 +1,369 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import {
+  computeIssuePatch,
+  computePrPatch,
+  linkedIssueNumbers,
+  BOUNTY_GENERIC,
+  BOUNTY_CLAIMED,
+  BOUNTY_REWARDED,
+} from "../src/rules.js";
+import type { Issue, PullRequest, IssueState, PrState } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+function makeIssue(overrides: Partial<Issue> & { number: number }): Issue {
+  return {
+    title: "Test issue",
+    html_url: `https://github.com/owner/repo/issues/${overrides.number}`,
+    state: "open",
+    labels: [],
+    assignees: [],
+    ...overrides,
+  };
+}
+
+function makePr(overrides: Partial<PullRequest> & { number: number }): PullRequest {
+  return {
+    title: "Test PR",
+    state: "open",
+    merged: false,
+    body: null,
+    labels: [],
+    user: { login: "contributor" },
+    html_url: `https://github.com/owner/repo/pull/${overrides.number}`,
+    ...overrides,
+  };
+}
+
+function labelNames(...names: string[]): { name: string }[] {
+  return names.map((name) => ({ name }));
+}
+
+function issueState(issue: Issue): IssueState {
+  return { issue, currentLabels: new Set(issue.labels.map((l) => l.name)) };
+}
+
+// ---------------------------------------------------------------------------
+// computeIssuePatch — issue rules
+// ---------------------------------------------------------------------------
+
+describe("computeIssuePatch", () => {
+  describe("no bounty label", () => {
+    it("produces no patch when issue has no bounty labels at all", () => {
+      const issue = makeIssue({ number: 1, labels: labelNames("type: bug") });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("removes generic bounty label when no value label present", () => {
+      const issue = makeIssue({ number: 1, labels: labelNames(BOUNTY_GENERIC) });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.equal(patch.ops.length, 1);
+      assert.deepEqual(patch.ops[0]!.remove, [BOUNTY_GENERIC]);
+      assert.deepEqual(patch.ops[0]!.add, []);
+    });
+
+    it("removes claimed label when no value label present", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames(BOUNTY_CLAIMED),
+        assignees: [{ login: "alice" }],
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.equal(patch.ops.length, 1);
+      assert.deepEqual(patch.ops[0]!.remove, [BOUNTY_CLAIMED]);
+    });
+  });
+
+  describe("has value label, no assignees", () => {
+    it("adds generic label when missing", () => {
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.equal(patch.ops.length, 1);
+      assert.deepEqual(patch.ops[0]!.add, [BOUNTY_GENERIC]);
+      assert.deepEqual(patch.ops[0]!.remove, []);
+    });
+
+    it("recognises legacy label format (no emoji) as a value label", () => {
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: $300") });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.equal(patch.ops.length, 1);
+      assert.deepEqual(patch.ops[0]!.add, [BOUNTY_GENERIC]);
+      assert.deepEqual(patch.ops[0]!.remove, []);
+    });
+
+    it("produces no patch when generic already present and no assignees", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $500", BOUNTY_GENERIC),
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("removes claimed when no assignees", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $500", BOUNTY_GENERIC, BOUNTY_CLAIMED),
+        assignees: [],
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.equal(patch.ops.length, 1);
+      assert.deepEqual(patch.ops[0]!.remove, [BOUNTY_CLAIMED]);
+      assert.deepEqual(patch.ops[0]!.add, []);
+    });
+  });
+
+  describe("has value label and assignees", () => {
+    it("adds both generic and claimed when both missing", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $300"),
+        assignees: [{ login: "alice" }],
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.equal(patch.ops.length, 1);
+      assert.deepEqual(patch.ops[0]!.add.sort(), [BOUNTY_CLAIMED, BOUNTY_GENERIC].sort());
+      assert.deepEqual(patch.ops[0]!.remove, []);
+    });
+
+    it("produces no patch when generic and claimed both already present", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $300", BOUNTY_GENERIC, BOUNTY_CLAIMED),
+        assignees: [{ login: "alice" }],
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("preserves non-bounty labels untouched", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("type: bug", "bounty: 💰 $100", BOUNTY_GENERIC, BOUNTY_CLAIMED),
+        assignees: [{ login: "alice" }],
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      assert.deepEqual(patch.ops, []);
+    });
+  });
+
+  describe("rewarded state", () => {
+    it("leaves rewarded label alone (owned by PR sync)", () => {
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $500", BOUNTY_GENERIC, BOUNTY_REWARDED),
+        assignees: [],
+      });
+      const patch = computeIssuePatch(issueState(issue));
+      // rewarded is not touched; only generic should be kept since value label present
+      assert.deepEqual(patch.ops, []);
+    });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// linkedIssueNumbers
+// ---------------------------------------------------------------------------
+
+describe("linkedIssueNumbers", () => {
+  it("returns empty array for null body", () => {
+    assert.deepEqual(linkedIssueNumbers(null), []);
+  });
+
+  it("parses closes/fixes/resolves keywords", () => {
+    const body = "Closes #10\nFixes #20\nResolves #30";
+    assert.deepEqual(linkedIssueNumbers(body), [10, 20, 30]);
+  });
+
+  it("is case-insensitive", () => {
+    const body = "CLOSES #5 fixes #6 RESOLVES #7";
+    assert.deepEqual(linkedIssueNumbers(body), [5, 6, 7]);
+  });
+
+  it("handles plural forms: close/fix/resolve", () => {
+    const body = "close #1 fix #2 resolve #3";
+    assert.deepEqual(linkedIssueNumbers(body), [1, 2, 3]);
+  });
+
+  it("returns empty when no keywords present", () => {
+    assert.deepEqual(linkedIssueNumbers("See issue #99 for context."), []);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// computePrPatch — PR rules
+// ---------------------------------------------------------------------------
+
+describe("computePrPatch", () => {
+  describe("label propagation (pre-merge)", () => {
+    it("produces no patch when PR body has no linked issues", () => {
+      const pr = makePr({ number: 10, body: "No references here." });
+      const patch = computePrPatch({ pr, currentLabels: new Set(), linkedIssues: [] });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("produces no patch when linked issues have no bounty labels", () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("type: bug") });
+      const patch = computePrPatch({ pr, currentLabels: new Set(), linkedIssues: [issue] });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("copies bounty value label from linked issue to PR", () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const patch = computePrPatch({ pr, currentLabels: new Set(), linkedIssues: [issue] });
+
+      const prOp = patch.ops.find((o) => o.target === 10);
+      assert.ok(prOp);
+      assert.deepEqual(prOp.add, ["bounty: 💰 $500"]);
+      assert.deepEqual(prOp.remove, []);
+    });
+
+    it("does not duplicate a value label already on the PR", () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $500"]),
+        linkedIssues: [issue],
+      });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("posts a comment on the linked issue when label is propagated", () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $200") });
+      const patch = computePrPatch({ pr, currentLabels: new Set(), linkedIssues: [issue] });
+
+      const issueOp = patch.ops.find((o) => o.target === 1);
+      assert.ok(issueOp);
+      assert.ok(issueOp.comment?.includes("PR"));
+      assert.ok(issueOp.comment?.includes("#10"));
+      assert.ok(issueOp.comment?.includes("contributor"));
+    });
+
+    it("does not post a comment when label was already on the PR", () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $200") });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $200"]),
+        linkedIssues: [issue],
+      });
+      const issueOp = patch.ops.find((o) => o.target === 1);
+      assert.ok(!issueOp);
+    });
+
+    it("merges value labels from multiple linked issues onto the PR", () => {
+      const pr = makePr({ number: 10, body: "Closes #1\nCloses #2" });
+      const issue1 = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $100") });
+      const issue2 = makeIssue({ number: 2, labels: labelNames("bounty: 💰 $200") });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(),
+        linkedIssues: [issue1, issue2],
+      });
+
+      const prOp = patch.ops.find((o) => o.target === 10);
+      assert.ok(prOp);
+      assert.deepEqual(prOp.add.sort(), ["bounty: 💰 $100", "bounty: 💰 $200"].sort());
+    });
+  });
+
+  describe("rewarded on merge", () => {
+    it("produces no patch when PR was closed but not merged", () => {
+      const pr = makePr({ number: 10, state: "closed", merged: false, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $500"]),
+        linkedIssues: [issue],
+      });
+      const prOp = patch.ops.find((o) => o.target === 10);
+      assert.ok(!prOp?.add.includes(BOUNTY_REWARDED));
+    });
+
+    it("adds rewarded to PR on merge", () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $500"]),
+        linkedIssues: [issue],
+      });
+
+      const prOp = patch.ops.find((o) => o.target === 10);
+      assert.ok(prOp);
+      assert.ok(prOp.add.includes(BOUNTY_REWARDED));
+    });
+
+    it("adds rewarded to linked issue on merge", () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500", BOUNTY_CLAIMED) });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $500"]),
+        linkedIssues: [issue],
+      });
+
+      const issueOp = patch.ops.find((o) => o.target === 1);
+      assert.ok(issueOp);
+      assert.ok(issueOp.add.includes(BOUNTY_REWARDED));
+      assert.ok(issueOp.remove.includes(BOUNTY_CLAIMED));
+    });
+
+    it("does not add rewarded again when already present", () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500", BOUNTY_REWARDED),
+      });
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $500", BOUNTY_REWARDED),
+      });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $500", BOUNTY_REWARDED]),
+        linkedIssues: [issue],
+      });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("skips issues with no bounty label when merged PR has a bounty", () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "Closes #1\nCloses #2",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const issue1 = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const issue2 = makeIssue({ number: 2, labels: labelNames("type: bug") });
+      const patch = computePrPatch({
+        pr,
+        currentLabels: new Set(["bounty: 💰 $500"]),
+        linkedIssues: [issue1, issue2],
+      });
+
+      // issue2 has no bounty so its desired state equals current: no op
+      const issue2Op = patch.ops.find((o) => o.target === 2);
+      assert.ok(!issue2Op);
+    });
+  });
+});

.github/scripts/bounty/tests/sync-all-issues.test.ts

@@ -0,0 +1,138 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { planAllIssues, syncAllIssues } from "../src/sync-all-issues.js";
+import type { GitHubApi } from "../src/api.js";
+import type { Issue, PullRequest } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Mock API
+// ---------------------------------------------------------------------------
+
+function makeMockApi(issues: Issue[]): GitHubApi & {
+  added: Map<number, string[][]>;
+  removed: Map<number, string[]>;
+} {
+  const added = new Map<number, string[][]>();
+  const removed = new Map<number, string[]>();
+
+  return {
+    added,
+    removed,
+    async getIssue(number) {
+      const found = issues.find((i) => i.number === number);
+      if (!found) throw new Error(`Issue #${number} not found in mock`);
+      return found;
+    },
+    async getPullRequest(): Promise<PullRequest> {
+      throw new Error("not used");
+    },
+    async listIssuesWithLabelPrefix(_prefix) {
+      // Return only real issues (not PRs — pull_request field absent)
+      return issues.filter((i) => i.pull_request === undefined);
+    },
+    async addLabels(target, labels) {
+      if (!added.has(target)) added.set(target, []);
+      added.get(target)!.push(labels);
+    },
+    async removeLabel(target, label) {
+      if (!removed.has(target)) removed.set(target, []);
+      removed.get(target)!.push(label);
+    },
+    async addComment() {},
+  };
+}
+
+function makeIssue(overrides: Partial<Issue> & { number: number }): Issue {
+  return { title: "Test issue", html_url: `https://github.com/owner/repo/issues/${overrides.number}`, state: "open", labels: [], assignees: [], ...overrides };
+}
+
+function labelNames(...names: string[]): { name: string }[] {
+  return names.map((name) => ({ name }));
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("planAllIssues", () => {
+  it("returns empty patch when no issues need changes", async () => {
+    const issues = [
+      makeIssue({ number: 1, labels: labelNames("bounty: $100", "bounty") }),
+      makeIssue({ number: 2, labels: labelNames("type: bug") }),
+    ];
+    const api = makeMockApi(issues);
+    const patch = await planAllIssues({ api });
+    assert.deepEqual(patch.ops, []);
+  });
+
+  it("plans adds for issues missing the generic bounty label", async () => {
+    const issues = [
+      makeIssue({ number: 1, labels: labelNames("bounty: $100") }),
+      makeIssue({ number: 2, labels: labelNames("bounty: $200") }),
+    ];
+    const api = makeMockApi(issues);
+    const patch = await planAllIssues({ api });
+    assert.equal(patch.ops.length, 2);
+    assert.ok(patch.ops.every((op) => op.add.includes("bounty")));
+  });
+
+  it("plans removal of stale claimed label when issue is unassigned", async () => {
+    const issues = [
+      makeIssue({
+        number: 3,
+        labels: labelNames("bounty: $500", "bounty", "bounty: claimed"),
+        assignees: [],
+      }),
+    ];
+    const api = makeMockApi(issues);
+    const patch = await planAllIssues({ api });
+    assert.equal(patch.ops.length, 1);
+    assert.deepEqual(patch.ops[0]!.remove, ["bounty: claimed"]);
+  });
+
+  it("collects ops for multiple issues with different needs", async () => {
+    const issues = [
+      makeIssue({ number: 1, labels: labelNames("bounty: $100") }), // missing generic
+      makeIssue({ number: 2, labels: labelNames("bounty: $200", "bounty") }), // already in sync
+      makeIssue({
+        number: 3,
+        labels: labelNames("bounty: $300", "bounty", "bounty: claimed"),
+        assignees: [],
+      }), // stale claimed
+    ];
+    const api = makeMockApi(issues);
+    const patch = await planAllIssues({ api });
+    assert.equal(patch.ops.length, 2);
+    const targets = patch.ops.map((op) => op.target).sort();
+    assert.deepEqual(targets, [1, 3]);
+  });
+});
+
+describe("syncAllIssues", () => {
+  it("applies adds and removals across all issues", async () => {
+    const issues = [
+      makeIssue({ number: 1, labels: labelNames("bounty: $100") }),
+      makeIssue({
+        number: 2,
+        labels: labelNames("bounty: $200", "bounty", "bounty: claimed"),
+        assignees: [],
+      }),
+    ];
+    const api = makeMockApi(issues);
+    await syncAllIssues({ api });
+
+    assert.ok(api.added.get(1)?.[0]?.includes("bounty"));
+    assert.deepEqual(api.removed.get(2), ["bounty: claimed"]);
+  });
+
+  it("returns empty patch when all issues are already in sync", async () => {
+    const issues = [
+      makeIssue({ number: 1, labels: labelNames("bounty: $100", "bounty") }),
+    ];
+    const api = makeMockApi(issues);
+    const patch = await syncAllIssues({ api });
+    assert.deepEqual(patch.ops, []);
+    assert.equal(api.added.size, 0);
+    assert.equal(api.removed.size, 0);
+  });
+});

.github/scripts/bounty/tests/sync-issue.test.ts

@@ -0,0 +1,144 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { syncIssue } from "../src/sync-issue.js";
+import type { GitHubApi } from "../src/api.js";
+import type { Issue, PullRequest } from "../src/types.js";
+
+// ---------------------------------------------------------------------------
+// Mock API
+// ---------------------------------------------------------------------------
+
+function makeMockApi(issue: Issue): GitHubApi & {
+  added: string[][];
+  removed: string[];
+  comments: string[];
+} {
+  const added: string[][] = [];
+  const removed: string[] = [];
+  const comments: string[] = [];
+
+  return {
+    added,
+    removed,
+    comments,
+    async getIssue() {
+      return issue;
+    },
+    async getPullRequest(): Promise<PullRequest> {
+      throw new Error("not used");
+    },
+    async listIssuesWithLabelPrefix(): Promise<Issue[]> {
+      throw new Error("not used");
+    },
+    async addLabels(_target, labels) {
+      added.push(labels);
+    },
+    async removeLabel(_target, label) {
+      removed.push(label);
+    },
+    async addComment(_target, body) {
+      comments.push(body);
+    },
+  };
+}
+
+function makeIssue(overrides: Partial<Issue> & { number: number }): Issue {
+  return {
+    title: "Test issue",
+    html_url: `https://github.com/owner/repo/issues/${overrides.number}`,
+    state: "open",
+    labels: [],
+    assignees: [],
+    ...overrides,
+  };
+}
+
+function labelNames(...names: string[]): { name: string }[] {
+  return names.map((name) => ({ name }));
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("syncIssue", () => {
+  it("returns empty patch and makes no API calls when issue is already in sync", async () => {
+    const issue = makeIssue({
+      number: 42,
+      labels: labelNames("bounty: 💰 $500", "bounty", "bounty: claimed"),
+      assignees: [{ login: "alice" }],
+    });
+    const api = makeMockApi(issue);
+    const patch = await syncIssue({ issueNumber: 42, api });
+
+    assert.deepEqual(patch.ops, []);
+    assert.deepEqual(api.added, []);
+    assert.deepEqual(api.removed, []);
+  });
+
+  it("adds generic bounty label when value label is present but generic is missing", async () => {
+    const issue = makeIssue({
+      number: 42,
+      labels: labelNames("bounty: 💰 $300"),
+    });
+    const api = makeMockApi(issue);
+    const patch = await syncIssue({ issueNumber: 42, api });
+
+    assert.equal(patch.ops.length, 1);
+    assert.deepEqual(api.added, [["bounty"]]);
+    assert.deepEqual(api.removed, []);
+  });
+
+  it("adds both generic and claimed when value label present and issue assigned", async () => {
+    const issue = makeIssue({
+      number: 42,
+      labels: labelNames("bounty: 💰 $800"),
+      assignees: [{ login: "bob" }],
+    });
+    const api = makeMockApi(issue);
+    await syncIssue({ issueNumber: 42, api });
+
+    assert.equal(api.added.length, 1);
+    assert.deepEqual(api.added[0]!.sort(), ["bounty", "bounty: claimed"].sort());
+  });
+
+  it("removes claimed when last assignee is removed", async () => {
+    const issue = makeIssue({
+      number: 42,
+      labels: labelNames("bounty: 💰 $500", "bounty", "bounty: claimed"),
+      assignees: [],
+    });
+    const api = makeMockApi(issue);
+    await syncIssue({ issueNumber: 42, api });
+
+    assert.deepEqual(api.removed, ["bounty: claimed"]);
+    assert.deepEqual(api.added, []);
+  });
+
+  it("removes both generic and claimed when value label is removed", async () => {
+    const issue = makeIssue({
+      number: 42,
+      labels: labelNames("bounty", "bounty: claimed"),
+      assignees: [{ login: "alice" }],
+    });
+    const api = makeMockApi(issue);
+    await syncIssue({ issueNumber: 42, api });
+
+    assert.deepEqual(api.removed.sort(), ["bounty", "bounty: claimed"].sort());
+    assert.deepEqual(api.added, []);
+  });
+
+  it("does a single batched addLabels call for multiple adds", async () => {
+    const issue = makeIssue({
+      number: 42,
+      labels: labelNames("bounty: 💰 $100"),
+      assignees: [{ login: "carol" }],
+    });
+    const api = makeMockApi(issue);
+    await syncIssue({ issueNumber: 42, api });
+
+    // One call, two labels
+    assert.equal(api.added.length, 1);
+    assert.equal(api.added[0]!.length, 2);
+  });
+});

.github/scripts/bounty/tests/sync-pr.test.ts

@@ -0,0 +1,241 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { syncPr } from "../src/sync-pr.js";
+import type { GitHubApi } from "../src/api.js";
+import type { Issue, PullRequest } from "../src/types.js";
+import { BOUNTY_CLAIMED, BOUNTY_REWARDED } from "../src/rules.js";
+
+// ---------------------------------------------------------------------------
+// Mock API
+// ---------------------------------------------------------------------------
+
+interface MockApi extends GitHubApi {
+  addedByTarget: Map<number, string[][]>;
+  removedByTarget: Map<number, string[]>;
+  commentsByTarget: Map<number, string[]>;
+}
+
+function makeMockApi(pr: PullRequest, issues: Issue[]): MockApi {
+  const addedByTarget = new Map<number, string[][]>();
+  const removedByTarget = new Map<number, string[]>();
+  const commentsByTarget = new Map<number, string[]>();
+
+  function recordAdd(target: number, labels: string[]) {
+    if (!addedByTarget.has(target)) addedByTarget.set(target, []);
+    addedByTarget.get(target)!.push(labels);
+  }
+  function recordRemove(target: number, label: string) {
+    if (!removedByTarget.has(target)) removedByTarget.set(target, []);
+    removedByTarget.get(target)!.push(label);
+  }
+  function recordComment(target: number, body: string) {
+    if (!commentsByTarget.has(target)) commentsByTarget.set(target, []);
+    commentsByTarget.get(target)!.push(body);
+  }
+
+  return {
+    addedByTarget,
+    removedByTarget,
+    commentsByTarget,
+    async getIssue(number) {
+      const found = issues.find((i) => i.number === number);
+      if (!found) throw new Error(`Issue #${number} not found in mock`);
+      return found;
+    },
+    async getPullRequest() {
+      return pr;
+    },
+    async listIssuesWithLabelPrefix(): Promise<Issue[]> {
+      throw new Error("not used");
+    },
+    async addLabels(target, labels) {
+      recordAdd(target, labels);
+    },
+    async removeLabel(target, label) {
+      recordRemove(target, label);
+    },
+    async addComment(target, body) {
+      recordComment(target, body);
+    },
+  };
+}
+
+function makeIssue(overrides: Partial<Issue> & { number: number }): Issue {
+  return { title: "issue", html_url: `https://github.com/owner/repo/issues/${overrides.number}`, state: "open", labels: [], assignees: [], ...overrides };
+}
+
+function makePr(overrides: Partial<PullRequest> & { number: number }): PullRequest {
+  return {
+    title: "Test PR",
+    state: "open",
+    merged: false,
+    body: null,
+    labels: [],
+    user: { login: "dev" },
+    html_url: `https://github.com/owner/repo/pull/${overrides.number}`,
+    ...overrides,
+  };
+}
+
+function labelNames(...names: string[]): { name: string }[] {
+  return names.map((name) => ({ name }));
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("syncPr", () => {
+  describe("label propagation", () => {
+    it("returns empty patch when PR has no linked issues", async () => {
+      const pr = makePr({ number: 10, body: "No links" });
+      const api = makeMockApi(pr, []);
+      const patch = await syncPr({ prNumber: 10, api });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("returns empty patch when linked issue has no bounty label", async () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("type: bug") });
+      const api = makeMockApi(pr, [issue]);
+      const patch = await syncPr({ prNumber: 10, api });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("copies bounty label from linked issue to PR", async () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const api = makeMockApi(pr, [issue]);
+      await syncPr({ prNumber: 10, api });
+
+      assert.ok(api.addedByTarget.get(10)?.[0]?.includes("bounty: 💰 $500"));
+    });
+
+    it("skips label already on the PR", async () => {
+      const pr = makePr({
+        number: 10,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const api = makeMockApi(pr, [issue]);
+      const patch = await syncPr({ prNumber: 10, api });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("posts comment on linked issue when label is propagated", async () => {
+      const pr = makePr({ number: 10, body: "Closes #1" });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $200") });
+      const api = makeMockApi(pr, [issue]);
+      await syncPr({ prNumber: 10, api });
+
+      const comments = api.commentsByTarget.get(1);
+      assert.ok(comments && comments.length === 1);
+      assert.ok(comments[0]!.includes("#10"));
+      assert.ok(comments[0]!.includes("dev"));
+    });
+
+    it("does not post comment when label was already on the PR", async () => {
+      const pr = makePr({
+        number: 10,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $200"),
+      });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $200") });
+      const api = makeMockApi(pr, [issue]);
+      await syncPr({ prNumber: 10, api });
+
+      assert.ok(!api.commentsByTarget.has(1));
+    });
+
+    it("fetches multiple linked issues in parallel and merges labels", async () => {
+      const pr = makePr({ number: 10, body: "Closes #1\nCloses #2" });
+      const issue1 = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $100") });
+      const issue2 = makeIssue({ number: 2, labels: labelNames("bounty: 💰 $200") });
+      const api = makeMockApi(pr, [issue1, issue2]);
+      await syncPr({ prNumber: 10, api });
+
+      const added = api.addedByTarget.get(10)?.[0] ?? [];
+      assert.ok(added.includes("bounty: 💰 $100"));
+      assert.ok(added.includes("bounty: 💰 $200"));
+    });
+
+    it("skips a linked issue that fails to fetch without aborting", async () => {
+      const pr = makePr({ number: 10, body: "Closes #1\nCloses #999" });
+      const issue1 = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $300") });
+      // issue #999 will throw (not in mock store)
+      const api = makeMockApi(pr, [issue1]);
+      const patch = await syncPr({ prNumber: 10, api });
+
+      // Issue #1 label still propagated
+      assert.ok(api.addedByTarget.get(10)?.[0]?.includes("bounty: 💰 $300"));
+    });
+  });
+
+  describe("rewarded on merge", () => {
+    it("adds rewarded to PR and linked issue on merge", async () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $500", BOUNTY_CLAIMED),
+      });
+      const api = makeMockApi(pr, [issue]);
+      await syncPr({ prNumber: 10, api });
+
+      assert.ok(api.addedByTarget.get(10)?.[0]?.includes(BOUNTY_REWARDED));
+      assert.ok(api.addedByTarget.get(1)?.[0]?.includes(BOUNTY_REWARDED));
+      assert.ok(api.removedByTarget.get(1)?.includes(BOUNTY_CLAIMED));
+    });
+
+    it("does not add rewarded again when already present on PR and issue", async () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500", BOUNTY_REWARDED),
+      });
+      const issue = makeIssue({
+        number: 1,
+        labels: labelNames("bounty: 💰 $500", BOUNTY_REWARDED),
+      });
+      const api = makeMockApi(pr, [issue]);
+      const patch = await syncPr({ prNumber: 10, api });
+      assert.deepEqual(patch.ops, []);
+    });
+
+    it("does not add rewarded when PR was closed but not merged", async () => {
+      const pr = makePr({
+        number: 10,
+        state: "closed",
+        merged: false,
+        body: "Closes #1",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const issue = makeIssue({ number: 1, labels: labelNames("bounty: 💰 $500") });
+      const api = makeMockApi(pr, [issue]);
+      await syncPr({ prNumber: 10, api });
+
+      assert.ok(!api.addedByTarget.get(10)?.[0]?.includes(BOUNTY_REWARDED));
+    });
+
+    it("handles merged PR with no linked issues gracefully", async () => {
+      const pr = makePr({
+        number: 10,
+        merged: true,
+        body: "No issue links",
+        labels: labelNames("bounty: 💰 $500"),
+      });
+      const api = makeMockApi(pr, []);
+      const patch = await syncPr({ prNumber: 10, api });
+
+      // Only the PR op
+      assert.ok(api.addedByTarget.get(10)?.[0]?.includes(BOUNTY_REWARDED));
+      assert.equal(patch.ops.length, 1);
+    });
+  });
+});

.github/workflows/autofix.yml

@@ -0,0 +1,62 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: autofix.ci
+env:
+  RUSTFLAGS: '-Dwarnings'
+'on':
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    branches:
+    - main
+  push:
+    branches:
+    - main
+jobs:
+  lint:
+    name: Lint Fix
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Install SQLite
+      run: sudo apt-get install -y libsqlite3-dev
+    - name: Setup Protobuf Compiler
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Rust Toolchain
+      uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        toolchain: nightly
+        components: clippy, rustfmt
+    - name: Cargo Fmt
+      run: cargo +nightly fmt --all
+    - name: Cargo Clippy
+      run: cargo +nightly clippy --all-features --workspace --all-targets --fix --allow-dirty -- -D warnings
+    - name: Cargo Clippy String Safety
+      run: cargo +nightly clippy --all-features --workspace -- -D clippy::string_slice -D clippy::indexing_slicing -D clippy::disallowed_methods
+    - name: Autofix
+      uses: autofix-ci/action@7a166d7532b277f34e16238930461bf77f9d7ed8
+concurrency:
+  group: autofix-${{github.ref}}
+  cancel-in-progress: false

.github/workflows/bounty.yml

@@ -0,0 +1,64 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: Bounty Management
+'on':
+  issues:
+    types:
+    - assigned
+    - unassigned
+    - labeled
+    - unlabeled
+  pull_request:
+    types:
+    - opened
+    - edited
+    - reopened
+  pull_request_target:
+    types:
+    - closed
+  schedule:
+  - cron: '0 2 * * *'
+permissions:
+  issues: write
+  pull-requests: write
+jobs:
+  sync-all-issues:
+    name: Sync all bounty issues
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+    - name: Install npm packages
+      run: npm install
+    - name: Sync all bounty labels
+      run: npx tsx .github/scripts/bounty/src/sync-all-issues.ts --repo ${{ github.repository }} --token ${{ secrets.GITHUB_TOKEN }} --execute
+  sync-pr:
+    name: Sync PR bounty labels
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+    - name: Install npm packages
+      run: npm install
+    - name: Sync bounty labels
+      run: npx tsx .github/scripts/bounty/src/sync-pr.ts --pr ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --token ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -0,0 +1,296 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: ci
+env:
+  RUSTFLAGS: '-Dwarnings'
+  OPENROUTER_API_KEY: ${{secrets.OPENROUTER_API_KEY}}
+'on':
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - labeled
+    branches:
+    - main
+  push:
+    branches:
+    - main
+    tags:
+    - v*
+jobs:
+  build:
+    name: Build and Test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Setup Protobuf Compiler
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Rust Toolchain
+      uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        toolchain: stable
+    - name: Install cargo-llvm-cov
+      run: cargo install cargo-llvm-cov
+    - name: Generate coverage
+      run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
+  zsh_rprompt_perf:
+    name: 'Performance: zsh rprompt'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Setup Protobuf Compiler
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Rust Toolchain
+      uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        toolchain: stable
+    - name: Run performance benchmark
+      run: './scripts/benchmark.sh --threshold 60 zsh rprompt'
+  draft_release:
+    needs:
+    - build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    name: Draft Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    outputs:
+      crate_release_name: ${{ steps.set_output.outputs.crate_release_name }}
+      crate_release_id: ${{ steps.set_output.outputs.crate_release_id }}
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - id: create_release
+      name: Draft Release
+      uses: release-drafter/release-drafter@v7
+      with:
+        config-name: release-drafter.yml
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - id: set_output
+      name: Export Outputs
+      run: echo "crate_release_id=${{ steps.create_release.outputs.id }}" >> $GITHUB_OUTPUT && echo "crate_release_name=${{ steps.create_release.outputs.tag_name }}" >> $GITHUB_OUTPUT
+  draft_release_pr:
+    if: 'github.event_name == ''pull_request'' && contains(github.event.pull_request.labels.*.name, ''ci: build all targets'')'
+    name: Draft Release for PR
+    runs-on: ubuntu-latest
+    outputs:
+      crate_release_name: ${{ steps.set_output.outputs.crate_release_name }}
+      crate_release_id: ${{ steps.set_output.outputs.crate_release_id }}
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - id: set_output
+      name: Set Release Version
+      run: echo "crate_release_name=pr-build-${{ github.event.number }}" >> $GITHUB_OUTPUT && echo "crate_release_id=pr-build-${{ github.event.number }}" >> $GITHUB_OUTPUT
+  build_release:
+    needs:
+    - draft_release
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    name: build-release
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: write
+      pull-requests: write
+    strategy:
+      matrix:
+        include:
+        - binary_name: forge-x86_64-unknown-linux-musl
+          binary_path: target/x86_64-unknown-linux-musl/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: x86_64-unknown-linux-musl
+        - binary_name: forge-aarch64-unknown-linux-musl
+          binary_path: target/aarch64-unknown-linux-musl/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-unknown-linux-musl
+        - binary_name: forge-x86_64-unknown-linux-gnu
+          binary_path: target/x86_64-unknown-linux-gnu/release/forge
+          cross: 'false'
+          os: ubuntu-latest
+          target: x86_64-unknown-linux-gnu
+        - binary_name: forge-aarch64-unknown-linux-gnu
+          binary_path: target/aarch64-unknown-linux-gnu/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-unknown-linux-gnu
+        - binary_name: forge-x86_64-apple-darwin
+          binary_path: target/x86_64-apple-darwin/release/forge
+          cross: 'false'
+          os: macos-latest
+          target: x86_64-apple-darwin
+        - binary_name: forge-aarch64-apple-darwin
+          binary_path: target/aarch64-apple-darwin/release/forge
+          cross: 'false'
+          os: macos-latest
+          target: aarch64-apple-darwin
+        - binary_name: forge-x86_64-pc-windows-msvc.exe
+          binary_path: target/x86_64-pc-windows-msvc/release/forge.exe
+          cross: 'false'
+          os: windows-latest
+          target: x86_64-pc-windows-msvc
+        - binary_name: forge-aarch64-pc-windows-msvc.exe
+          binary_path: target/aarch64-pc-windows-msvc/release/forge.exe
+          cross: 'false'
+          os: windows-latest
+          target: aarch64-pc-windows-msvc
+        - binary_name: forge-aarch64-linux-android
+          binary_path: target/aarch64-linux-android/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-linux-android
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Setup Protobuf Compiler
+      if: ${{ matrix.cross == 'false' }}
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Cross Toolchain
+      if: ${{ matrix.cross == 'false' }}
+      uses: taiki-e/setup-cross-toolchain-action@v1
+      with:
+        target: ${{ matrix.target }}
+    - name: Add Rust target
+      if: ${{ matrix.cross == 'false' }}
+      run: rustup target add ${{ matrix.target }}
+    - name: Set Rust Flags
+      if: '!(contains(matrix.target, ''-unknown-linux-'') || contains(matrix.target, ''-android''))'
+      run: echo "RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV
+    - name: Build Binary
+      uses: ClementTsang/cargo-action@v0.0.7
+      with:
+        command: build --release
+        args: '--target ${{ matrix.target }}'
+        use-cross: ${{ matrix.cross }}
+        cross-version: '0.2.5'
+      env:
+        RUSTFLAGS: ${{ env.RUSTFLAGS }}
+        POSTHOG_API_SECRET: ${{secrets.POSTHOG_API_SECRET}}
+        APP_VERSION: ${{ needs.draft_release.outputs.crate_release_name }}
+    - name: Copy Binary
+      run: cp ${{ matrix.binary_path }} ${{ matrix.binary_name }}
+    - name: Upload to Release
+      uses: xresloader/upload-to-github-release@v1
+      with:
+        release_id: ${{ needs.draft_release.outputs.crate_release_id }}
+        file: ${{ matrix.binary_name }}
+        overwrite: 'true'
+  build_release_pr:
+    needs:
+    - draft_release_pr
+    if: 'github.event_name == ''pull_request'' && contains(github.event.pull_request.labels.*.name, ''ci: build all targets'')'
+    name: build-release
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: write
+      pull-requests: write
+    strategy:
+      matrix:
+        include:
+        - binary_name: forge-x86_64-unknown-linux-musl
+          binary_path: target/x86_64-unknown-linux-musl/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: x86_64-unknown-linux-musl
+        - binary_name: forge-aarch64-unknown-linux-musl
+          binary_path: target/aarch64-unknown-linux-musl/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-unknown-linux-musl
+        - binary_name: forge-x86_64-unknown-linux-gnu
+          binary_path: target/x86_64-unknown-linux-gnu/release/forge
+          cross: 'false'
+          os: ubuntu-latest
+          target: x86_64-unknown-linux-gnu
+        - binary_name: forge-aarch64-unknown-linux-gnu
+          binary_path: target/aarch64-unknown-linux-gnu/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-unknown-linux-gnu
+        - binary_name: forge-x86_64-apple-darwin
+          binary_path: target/x86_64-apple-darwin/release/forge
+          cross: 'false'
+          os: macos-latest
+          target: x86_64-apple-darwin
+        - binary_name: forge-aarch64-apple-darwin
+          binary_path: target/aarch64-apple-darwin/release/forge
+          cross: 'false'
+          os: macos-latest
+          target: aarch64-apple-darwin
+        - binary_name: forge-x86_64-pc-windows-msvc.exe
+          binary_path: target/x86_64-pc-windows-msvc/release/forge.exe
+          cross: 'false'
+          os: windows-latest
+          target: x86_64-pc-windows-msvc
+        - binary_name: forge-aarch64-pc-windows-msvc.exe
+          binary_path: target/aarch64-pc-windows-msvc/release/forge.exe
+          cross: 'false'
+          os: windows-latest
+          target: aarch64-pc-windows-msvc
+        - binary_name: forge-aarch64-linux-android
+          binary_path: target/aarch64-linux-android/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-linux-android
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Setup Protobuf Compiler
+      if: ${{ matrix.cross == 'false' }}
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Cross Toolchain
+      if: ${{ matrix.cross == 'false' }}
+      uses: taiki-e/setup-cross-toolchain-action@v1
+      with:
+        target: ${{ matrix.target }}
+    - name: Add Rust target
+      if: ${{ matrix.cross == 'false' }}
+      run: rustup target add ${{ matrix.target }}
+    - name: Set Rust Flags
+      if: '!(contains(matrix.target, ''-unknown-linux-'') || contains(matrix.target, ''-android''))'
+      run: echo "RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV
+    - name: Build Binary
+      uses: ClementTsang/cargo-action@v0.0.7
+      with:
+        command: build --release
+        args: '--target ${{ matrix.target }}'
+        use-cross: ${{ matrix.cross }}
+        cross-version: '0.2.5'
+      env:
+        RUSTFLAGS: ${{ env.RUSTFLAGS }}
+        POSTHOG_API_SECRET: ${{secrets.POSTHOG_API_SECRET}}
+        APP_VERSION: ${{ needs.draft_release_pr.outputs.crate_release_name }}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/labels.yml

@@ -0,0 +1,39 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: Github Label Sync
+'on':
+  push:
+    branches:
+    - main
+permissions:
+  contents: write
+jobs:
+  label-sync:
+    name: label-sync
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+    - name: Sync labels
+      run: |-
+        npx github-label-sync \
+          --access-token ${{ secrets.GITHUB_TOKEN }} \
+          --labels ".github/labels.json" \
+          ${{ github.repository }}

.github/workflows/release-drafter.yml

@@ -0,0 +1,53 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: Release Drafter
+'on':
+  pull_request_target:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - labeled
+    - unlabeled
+    - closed
+    branches:
+    - main
+  push:
+    branches:
+    - main
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  update_release_draft:
+    name: update_release_draft
+    runs-on: ubuntu-latest
+    steps:
+    - name: Auto Labeler
+      if: github.event_name == 'pull_request_target'
+      uses: release-drafter/release-drafter/autolabeler@v7
+      with:
+        config-name: release-drafter.yml
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Release Drafter
+      uses: release-drafter/release-drafter@v7
+      with:
+        config-name: release-drafter.yml
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,155 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: Multi Channel Release
+'on':
+  release:
+    types:
+    - published
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  build_release:
+    name: build-release
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: write
+      pull-requests: write
+    strategy:
+      matrix:
+        include:
+        - binary_name: forge-x86_64-unknown-linux-musl
+          binary_path: target/x86_64-unknown-linux-musl/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: x86_64-unknown-linux-musl
+        - binary_name: forge-aarch64-unknown-linux-musl
+          binary_path: target/aarch64-unknown-linux-musl/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-unknown-linux-musl
+        - binary_name: forge-x86_64-unknown-linux-gnu
+          binary_path: target/x86_64-unknown-linux-gnu/release/forge
+          cross: 'false'
+          os: ubuntu-latest
+          target: x86_64-unknown-linux-gnu
+        - binary_name: forge-aarch64-unknown-linux-gnu
+          binary_path: target/aarch64-unknown-linux-gnu/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-unknown-linux-gnu
+        - binary_name: forge-x86_64-apple-darwin
+          binary_path: target/x86_64-apple-darwin/release/forge
+          cross: 'false'
+          os: macos-latest
+          target: x86_64-apple-darwin
+        - binary_name: forge-aarch64-apple-darwin
+          binary_path: target/aarch64-apple-darwin/release/forge
+          cross: 'false'
+          os: macos-latest
+          target: aarch64-apple-darwin
+        - binary_name: forge-x86_64-pc-windows-msvc.exe
+          binary_path: target/x86_64-pc-windows-msvc/release/forge.exe
+          cross: 'false'
+          os: windows-latest
+          target: x86_64-pc-windows-msvc
+        - binary_name: forge-aarch64-pc-windows-msvc.exe
+          binary_path: target/aarch64-pc-windows-msvc/release/forge.exe
+          cross: 'false'
+          os: windows-latest
+          target: aarch64-pc-windows-msvc
+        - binary_name: forge-aarch64-linux-android
+          binary_path: target/aarch64-linux-android/release/forge
+          cross: 'true'
+          os: ubuntu-latest
+          target: aarch64-linux-android
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+    - name: Setup Protobuf Compiler
+      if: ${{ matrix.cross == 'false' }}
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Setup Cross Toolchain
+      if: ${{ matrix.cross == 'false' }}
+      uses: taiki-e/setup-cross-toolchain-action@v1
+      with:
+        target: ${{ matrix.target }}
+    - name: Add Rust target
+      if: ${{ matrix.cross == 'false' }}
+      run: rustup target add ${{ matrix.target }}
+    - name: Set Rust Flags
+      if: '!(contains(matrix.target, ''-unknown-linux-'') || contains(matrix.target, ''-android''))'
+      run: echo "RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV
+    - name: Build Binary
+      uses: ClementTsang/cargo-action@v0.0.7
+      with:
+        command: build --release
+        args: '--target ${{ matrix.target }}'
+        use-cross: ${{ matrix.cross }}
+        cross-version: '0.2.5'
+      env:
+        RUSTFLAGS: ${{ env.RUSTFLAGS }}
+        POSTHOG_API_SECRET: ${{secrets.POSTHOG_API_SECRET}}
+        APP_VERSION: ${{ github.event.release.tag_name }}
+    - name: Copy Binary
+      run: cp ${{ matrix.binary_path }} ${{ matrix.binary_name }}
+    - name: Upload to Release
+      uses: xresloader/upload-to-github-release@v1
+      with:
+        release_id: ${{ github.event.release.id }}
+        file: ${{ matrix.binary_name }}
+        overwrite: 'true'
+  npm_release:
+    needs:
+    - build_release
+    name: npm_release
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        repository:
+        - antinomyhq/npm-code-forge
+        - antinomyhq/npm-forgecode
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+      with:
+        repository: ${{ matrix.repository }}
+        ref: main
+        token: ${{ secrets.NPM_ACCESS }}
+    - name: Update NPM Package
+      run: './update-package.sh ${{ github.event.release.tag_name }}'
+      env:
+        AUTO_PUSH: 'true'
+        CI: 'true'
+        NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+  homebrew_release:
+    needs:
+    - build_release
+    name: homebrew_release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v6
+      with:
+        repository: antinomyhq/homebrew-code-forge
+        ref: main
+        token: ${{ secrets.HOMEBREW_ACCESS }}
+    - name: Update Homebrew Formula
+      run: GITHUB_TOKEN="${{ secrets.HOMEBREW_ACCESS }}" ./update-formula.sh ${{ github.event.release.tag_name }}

.github/workflows/stale.yml

@@ -0,0 +1,51 @@
+# -------------------------------------------------------------------
+# ------------------------------- WARNING ---------------------------
+# -------------------------------------------------------------------
+#
+# This file was automatically generated by gh-workflows using the
+# gh-workflow-gen bin. You should add and commit this file to your
+# git repository. **DO NOT EDIT THIS FILE BY HAND!** Any manual changes
+# will be lost if the file is regenerated.
+#
+# To make modifications, update your `build.rs` configuration to adjust
+# the workflow description as needed, then regenerate this file to apply
+# those changes.
+#
+# -------------------------------------------------------------------
+# ----------------------------- END WARNING -------------------------
+# -------------------------------------------------------------------
+
+name: Close Stale Issues and PR
+env:
+  DAYS_BEFORE_ISSUE_STALE: '30'
+  DAYS_BEFORE_ISSUE_CLOSE: '7'
+  DAYS_BEFORE_PR_STALE: '5'
+  DAYS_BEFORE_PR_CLOSE: '10'
+'on':
+  schedule:
+  - cron: '0 * * * *'
+permissions:
+  issues: write
+  pull-requests: write
+jobs:
+  stale:
+    name: Stale Issues
+    runs-on: ubuntu-latest
+    steps:
+    - name: Mark Stale Issues
+      uses: actions/stale@v10
+      with:
+        stale-issue-label: 'state: inactive'
+        stale-pr-label: 'state: inactive'
+        stale-issue-message: |-
+          **Action required:** Issue inactive for ${{ env.DAYS_BEFORE_ISSUE_STALE }} days.
+          Status update or closure in ${{ env.DAYS_BEFORE_ISSUE_CLOSE }} days.
+        close-issue-message: Issue closed after ${{ env.DAYS_BEFORE_ISSUE_CLOSE }} days of inactivity.
+        stale-pr-message: |-
+          **Action required:** PR inactive for ${{ env.DAYS_BEFORE_PR_STALE }} days.
+          Status update or closure in ${{ env.DAYS_BEFORE_PR_CLOSE }} days.
+        close-pr-message: PR closed after ${{ env.DAYS_BEFORE_PR_CLOSE }} days of inactivity.
+        days-before-issue-stale: ${{ env.DAYS_BEFORE_ISSUE_STALE }}
+        days-before-issue-close: ${{ env.DAYS_BEFORE_ISSUE_CLOSE }}
+        days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
+        days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}

benchmarks/README.md

@@ -0,0 +1,320 @@
+# Forge Code Evaluations
+
+A flexible evaluation framework for running automated tests and benchmarks against Forge Code commands.
+
+## Quick Start
+
+### Setup
+
+Before running evaluations, create a `forgee` symlink to the debug binary:
+
+```bash
+# Create symlink in your PATH (e.g., ~/bin or /usr/local/bin)
+ln -sf /path/to/code-forge/target/debug/forge ~/forgee
+
+# Or if ~/bin is in your PATH
+ln -sf $(pwd)/target/debug/forge ~/bin/forgee
+```
+
+**Why is this needed?** Tasks execute in temporary directories, so relative paths like `../../target/debug/forge` won't work. The `forgee` symlink provides a stable absolute path that works from any directory.
+
+### Running Evaluations
+
+```bash
+# Run an evaluation
+npm run eval ./evals/create_skill/task.yml
+
+# Set custom log level
+LOG_LEVEL=debug npm run eval ./evals/create_skill/task.yml
+```
+
+## How It Works
+
+The evaluation system executes commands based on task definitions and validates their output. It supports:
+
+- **Parallel execution** with configurable concurrency
+- **Timeout handling** for long-running tasks
+- **Data-driven testing** using CSV files
+- **Output validation** using regex patterns
+- **Debug artifacts** stored with timestamps
+
+## Project Structure
+
+```
+benchmarks/
+├── cli.ts                    # Main CLI entry point
+├── command-generator.ts      # Command template rendering
+├── task-executor.ts          # Task execution with timeout support
+├── model.ts                  # TypeScript types for tasks
+├── parse.ts                  # CLI argument parsing
+└── evals/                    # Evaluation definitions
+    └── create_skill/
+        ├── task.yml          # Task definition
+        ├── create_skill_tasks.csv  # Test data
+        └── debug/            # Debug outputs (timestamped)
+```
+
+## Creating an Evaluation
+
+### 1. Create an Evaluation Directory
+
+```bash
+mkdir -p evals/my_eval
+```
+
+### 2. Create a `task.yml` File
+
+The task file defines how to run your evaluation:
+
+```yaml
+# Optional: Commands to run before evaluation starts
+before_run:
+  - cargo build
+  - npm install
+
+# Required: Command(s) to execute for each test case
+# Single command
+run: forgee -p '{{prompt}}'
+
+# Or multiple commands (executed sequentially)
+run:
+  - echo "Step 1: {{task}}"
+  - forgee -p '{{prompt}}'
+  - echo "Step 2: Complete"
+
+# Execution configuration
+parallelism: 10  # Number of tasks to run in parallel (default: 1)
+timeout: 60      # Timeout in seconds (optional)
+early_exit: true # Stop execution when validations pass (optional)
+
+# Optional: Validations to run on output
+validations:
+  - name: "Check success message"
+    type: regex
+    regex: \[[0-9:]*\] Skill create-skill
+
+# Required: Data sources for test cases
+sources:
+  - csv: my_tasks.csv
+```
+
+#### Task File Schema
+
+**`before_run`** (optional): Array of shell commands to execute before running tasks
+- Runs sequentially before the main command execution
+- Executes in a temporary directory created for the evaluation run
+- Useful for building binaries or setting up dependencies
+
+**`run`** (required): Command(s) to execute for each test case
+- Can be a single string or an array of strings
+- Commands support template placeholders (e.g., `{{variable}}`)
+- Multiple commands are executed sequentially
+- If any command fails, subsequent commands are skipped
+
+**`parallelism`** (optional): Number of tasks to run concurrently (default: 1)
+
+**`timeout`** (optional): Maximum execution time in seconds per task
+
+**`early_exit`** (optional): Stop command execution when all validations pass
+
+**`validations`** (optional): Array of validation rules
+- `name`: Human-readable description
+- `type`: Validation type. Supported values:
+  - `regex`: Match output against a regular expression pattern
+  - `shell`: Execute a shell command with output as stdin
+- For `regex` type:
+  - `regex`: Regular expression pattern to match in output
+- For `shell` type:
+  - `command`: Shell command to execute (receives task output via stdin)
+  - `exit_code`: Expected exit code (default: 0)
+
+**`sources`** (required): Array of data sources
+- Currently supports CSV files: `- csv: filename.csv`
+- Future: Command output: `- cmd: command`
+
+### 3. Create Test Data (CSV)
+
+Create a CSV file with columns matching your template variables:
+
+```csv
+prompt,expected_output
+"Create a backup script","backup.sh"
+"Generate API client","api_client.py"
+```
+
+The column names (e.g., `prompt`, `expected_output`) become template variables you can use in your command with `{{column_name}}`.
+
+### 4. Run the Evaluation
+
+```bash
+npm run eval ./evals/my_eval/task.yml
+```
+
+## Template Variables
+
+Commands support Handlebars template syntax. Variables are populated from CSV columns:
+
+```yaml
+run:
+  command: ./tool --input '{{input_file}}' --format {{format}} --verbose
+```
+
+With CSV:
+```csv
+input_file,format
+data1.txt,json
+data2.txt,yaml
+```
+
+## Output and Debugging
+
+### Debug Artifacts
+
+Each run creates a timestamped debug directory:
+
+```
+evals/my_eval/debug/2025-11-23T10-30-45-123Z/
+├── task-1.log
+├── task-2.log
+└── task-3.log
+```
+
+Each log file contains the full output (stdout + stderr) from the command execution.
+
+### Task Status
+
+Tasks can have four statuses:
+
+- **`passed`**: Task completed successfully and passed all validations
+- **`validation_failed`**: Task completed but failed one or more validations
+- **`timeout`**: Task exceeded the timeout limit
+- **`failed`**: Task execution failed (non-zero exit code)
+
+### Logging
+
+**Human-readable output (default):**
+```bash
+npm run eval ./evals/my_eval/task.yml
+```
+
+**Machine-readable JSON output:**
+```bash
+LOG_JSON=1 npm run eval ./evals/my_eval/task.yml | jq .
+```
+
+**Debug logging:**
+```bash
+LOG_LEVEL=debug npm run eval ./evals/my_eval/task.yml
+```
+
+## Examples
+
+### Example 1: Simple Sequential Execution
+
+```yaml
+run: echo "Processing {{name}}"
+sources:
+  - csv: names.csv
+```
+
+```csv
+name
+Alice
+Bob
+Charlie
+```
+
+### Example 2: Parallel Execution with Timeout
+
+```yaml
+run: ./slow_task --id {{task_id}}
+parallelism: 5
+timeout: 30
+sources:
+  - csv: tasks.csv
+```
+
+### Example 3: Multiple Commands
+
+```yaml
+run:
+  - echo "Starting task {{id}}"
+  - ./process --input {{file}}
+  - echo "Task {{id}} complete"
+parallelism: 3
+timeout: 120
+sources:
+  - csv: tasks.csv
+```
+
+### Example 4: Shell Command Validation
+
+```yaml
+run: echo "{{message}}"
+parallelism: 3
+validations:
+  # Using grep to check if output contains specific text
+  - name: "Contains 'test' word"
+    type: shell
+    command: grep -q "test"
+    exit_code: 0
+  
+  # Count words and ensure it's greater than 2
+  - name: "More than 2 words"
+    type: shell
+    command: test $(wc -w | awk '{print $1}') -gt 2
+    exit_code: 0
+  
+  # Traditional regex validation (for comparison)
+  - name: "Contains test or validation"
+    type: regex
+    regex: "test|validation"
+sources:
+  - csv: messages.csv
+```
+
+### Example 5: Regex Validation
+
+```yaml
+run: cargo test {{test_name}}
+validations:
+  - name: "All tests passed"
+    type: regex
+    regex: test result:\s+ok
+sources:
+  - csv: tests.csv
+```
+
+## Tips
+
+1. **Use quotes in commands**: When passing CSV values with spaces, wrap them in quotes:
+   ```yaml
+   command: forge -p '{{prompt}}'
+   ```
+
+2. **Build before running**: Use `before_run` to ensure binaries are up-to-date:
+   ```yaml
+   before_run:
+     - cargo build
+   ```
+
+3. **Start with low parallelism**: Test with `parallelism: 1` first, then increase:
+   ```yaml
+   parallelism: 1  # Start here
+   ```
+
+4. **Set appropriate timeouts**: Add timeouts to prevent hanging:
+   ```yaml
+   timeout: 60  # seconds
+   ```
+
+5. **Check debug logs**: When tasks fail, check the debug directory for full output:
+   ```bash
+   cat evals/my_eval/debug/*/task-1.log
+   ```
+
+## Exit Codes
+
+The CLI exits with:
+- **0**: All tasks passed
+- **1**: One or more tasks failed (excluding timeouts and validation failures)

benchmarks/cli.ts

@@ -0,0 +1,394 @@
+#!/usr/bin/env node
+
+// Handle EPIPE errors gracefully (e.g., when piping to `head` or `jq` that closes early)
+process.stdout.on("error", (error: NodeJS.ErrnoException) => {
+  if (error.code === "EPIPE") {
+    process.exit(0);
+  }
+  throw error;
+});
+
+import * as fs from "fs/promises";
+import * as path from "path";
+import { fileURLToPath } from "url";
+import { parse as parseYaml } from "yaml";
+import { exec } from "child_process";
+import { promisify } from "util";
+import pLimit from "p-limit";
+import pino from "pino";
+import { TaskStatus, type Task } from "./model.js";
+import {
+  getContextsFromSources,
+  generateCommand,
+} from "./command-generator.js";
+import { parseCliArgs } from "./parse.js";
+import { executeTask, type TaskExecutionResult } from "./task-executor.js";
+import { processValidations, type ValidationResult } from "./verification.js";
+import { createTempDir, parseCsvAsync } from "./utils.js";
+
+const execAsync = promisify(exec);
+
+export type TaskResult = {
+  index: number;
+  status: TaskStatus;
+  command: string;
+  duration: number;
+  validationResults: ValidationResult[];
+};
+
+// ESM compatibility for __dirname
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Create logger instance
+ * - Human-readable CLI output by default
+ * - Set LOG_JSON=1 for machine-readable JSON output (for piping to jq, log aggregators, etc.)
+ */
+const logger =
+  process.env.LOG_JSON === "1"
+    ? pino({
+        level: process.env.LOG_LEVEL || "info",
+        formatters: {
+          level: (label) => ({ level: label }),
+        },
+        timestamp: pino.stdTimeFunctions.isoTime,
+      })
+    : pino({
+        level: process.env.LOG_LEVEL || "info",
+        transport: {
+          target: "pino-pretty",
+          options: {
+            colorize: true,
+            translateTime: "HH:MM:ss",
+            ignore: "pid,hostname",
+            messageFormat: "{msg}",
+          },
+        },
+        formatters: {
+          level: (label) => ({ level: label }),
+        },
+        timestamp: pino.stdTimeFunctions.isoTime,
+      });
+
+async function main() {
+  // Parse command line arguments
+  let args;
+  try {
+    args = await parseCliArgs(__dirname);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Unknown error";
+    logger.error({ error: message }, "Failed to parse CLI arguments");
+    process.exit(1);
+  }
+
+  const { evalName, evalDir, taskFile } = args;
+
+  // Check if eval directory and task file exist
+  try {
+    await fs.access(evalDir);
+  } catch {
+    logger.error({ evalDir }, "Eval directory not found");
+    process.exit(1);
+  }
+
+  try {
+    await fs.access(taskFile);
+  } catch {
+    logger.error({ evalDir }, "task.yml not found");
+    process.exit(1);
+  }
+
+  // Read and parse task.yml
+  const taskContent = await fs.readFile(taskFile, "utf-8");
+  const task: Task = parseYaml(taskContent);
+
+  // Display header
+  const displayName = path.relative(__dirname, evalDir) || evalName;
+
+  // Create debug directory with timestamp
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const debugDir = path.join(evalDir, "debug", timestamp);
+  await fs.mkdir(debugDir, { recursive: true });
+
+  // Create a temp directory for setup commands
+  const setupTmpDir = await createTempDir("forge-setup-");
+
+  // Execute before_run commands
+  if (task.before_run && task.before_run.length > 0) {
+    for (const cmd of task.before_run) {
+      try {
+        logger.info(
+          { dir: setupTmpDir.name, command: cmd },
+          "Running setup command",
+        );
+        // Small delay to allow logger to flush before command output
+        await new Promise((resolve) => setTimeout(resolve, 0));
+        await execAsync(cmd, {
+          cwd: setupTmpDir.name,
+        });
+      } catch (error) {
+        logger.error({ command: cmd }, "Setup command failed");
+        process.exit(1);
+      }
+    }
+  }
+
+  // Load data from sources and create cross product
+  const sourcesData: Record<string, string>[][] = [];
+
+  for (const source of task.sources) {
+    if ("csv" in source) {
+      const csvPath = path.join(evalDir, source.csv);
+      try {
+        await fs.access(csvPath);
+      } catch {
+        logger.error({ csvPath }, "CSV file not found");
+        process.exit(1);
+      }
+
+      const csvContent = await fs.readFile(csvPath, "utf-8");
+      const csvData = await parseCsvAsync(csvContent, {
+        columns: true,
+        skip_empty_lines: true,
+      });
+      sourcesData.push(csvData);
+    } else if ("cmd" in source) {
+      logger.error("cmd source type not yet implemented");
+      process.exit(1);
+    } else if ("value" in source) {
+      sourcesData.push(source.value);
+    }
+  }
+
+  // Create cross product of all sources
+  if (sourcesData.length === 0) {
+    logger.error("No sources configured");
+    process.exit(1);
+  }
+
+  // Get contexts from sources using pure function
+  const data = getContextsFromSources(sourcesData);
+
+  const results: TaskResult[] = [];
+
+  // Get parallelism setting (default to 1 for sequential execution)
+  const parallelism = task.parallelism ?? 1;
+  const limit = pLimit(parallelism);
+
+  // Execute run command for each data row
+  // Create promises for all tasks
+  const taskPromises = data.map((row, i) => {
+    return limit(async () => {
+      // Create a unique temp directory for this task
+      const taskTmpDir = await createTempDir(`forge-task-${i + 1}-`);
+
+      // Create a 'task' subdirectory for running commands
+      const taskWorkDir = path.join(taskTmpDir.name, 'task');
+      await fs.mkdir(taskWorkDir, { recursive: true });
+
+      const logFile = path.join(taskTmpDir.name, `task.log`);
+
+      // Context for command interpolation and validations
+      const context = { ...row, dir: taskTmpDir.name };
+
+      // Support both single command and multiple commands
+      const commands = Array.isArray(task.run) ? task.run : [task.run];
+
+      // Filter out empty or non-string commands
+      const validCommands = commands.filter(cmd => typeof cmd === 'string' && cmd.trim().length > 0);
+
+      // If no valid commands, skip this task
+      if (validCommands.length === 0) {
+        logger.warn({ task_id: i + 1 }, "No valid commands found, skipping task");
+        return {
+          index: i + 1,
+          status: TaskStatus.Failed,
+          command: "No valid commands",
+          duration: 0,
+          validationResults: [],
+        };
+      }
+
+      let combinedOutput = "";
+      let totalDuration = 0;
+      let lastError: string | undefined;
+      let hasTimeout = false;
+      let hasEarlyExit = false;
+
+      // Log task launch once before executing commands
+      logger.info(
+        {
+          task_id: i + 1,
+          total_commands: validCommands.length,
+          log: logFile,
+          dir: taskTmpDir.name,
+          work_dir: taskWorkDir,
+          parameters: context,
+        },
+        "Launching task",
+      );
+
+      // Execute commands sequentially
+      for (let cmdIdx = 0; cmdIdx < validCommands.length; cmdIdx++) {
+        const commandTemplate = validCommands[cmdIdx]!; // Non-null assertion safe after filter
+
+        const command = generateCommand(commandTemplate, context);
+
+        logger.info(
+          {
+            command,
+            task_id: i + 1,
+            command_id: cmdIdx + 1,
+            total_commands: validCommands.length,
+          },
+          "Executing command",
+        );
+
+        const executionResult = await executeTask(
+          command,
+          i + 1,
+          logFile,
+          taskWorkDir, // Use taskWorkDir instead of taskTmpDir.name
+          task,
+          context,
+          cmdIdx > 0, // append if this is not the first command
+        );
+
+        totalDuration += executionResult.duration;
+
+        if (executionResult.output) {
+          combinedOutput += executionResult.output;
+        }
+
+        if (executionResult.earlyExit) {
+          hasEarlyExit = true;
+        }
+
+        // If execution failed or timed out, stop executing remaining commands
+        if (executionResult.error) {
+          lastError = executionResult.error;
+          hasTimeout = executionResult.isTimeout;
+
+          logger.warn(
+            {
+              task_id: executionResult.index,
+              command: executionResult.command,
+              command_id: cmdIdx + 1,
+              duration: executionResult.duration,
+              error: executionResult.error,
+              is_timeout: executionResult.isTimeout,
+            },
+            executionResult.isTimeout ? "Task timed out" : "Task failed",
+          );
+          break;
+        }
+      }
+
+      // If any command failed, return failure result
+      if (lastError) {
+        const { validationResults } = await processValidations(
+          combinedOutput,
+          task,
+          logger,
+          i + 1,
+          totalDuration,
+          logFile,
+          context,
+        );
+
+        return {
+          index: i + 1,
+          status: hasTimeout ? TaskStatus.Timeout : TaskStatus.Failed,
+          command: validCommands.length === 1 ? validCommands[0]! : `${validCommands.length} commands`,
+          duration: totalDuration,
+          validationResults,
+        };
+      }
+
+      // Run validations on the combined output
+      const { validationResults, status: validationStatus } =
+        await processValidations(
+          combinedOutput,
+          task,
+          logger,
+          i + 1,
+          totalDuration,
+          logFile,
+          context,
+        );
+
+      return {
+        index: i + 1,
+        status:
+          validationStatus === "passed"
+            ? TaskStatus.Passed
+            : TaskStatus.ValidationFailed,
+        command: validCommands.length === 1 ? validCommands[0]! : `${validCommands.length} commands`,
+        duration: totalDuration,
+        validationResults,
+      };
+    });
+  });
+
+  // Wait for all tasks to complete
+  const taskResults = await Promise.all(taskPromises);
+  results.push(...taskResults);
+
+  // Calculate summary statistics
+  const successCount = results.filter(
+    (r) => r.status === TaskStatus.Passed,
+  ).length;
+  const warningCount = results.filter(
+    (r) => r.status === TaskStatus.ValidationFailed,
+  ).length;
+  const timeoutCount = results.filter(
+    (r) => r.status === TaskStatus.Timeout,
+  ).length;
+  const failCount = results.filter(
+    (r) => r.status === TaskStatus.Failed,
+  ).length;
+  const totalDuration = results.reduce((sum, r) => sum + r.duration, 0);
+
+  // Calculate validation statistics
+  const totalValidations = results.reduce(
+    (sum, r) => sum + r.validationResults.length,
+    0,
+  );
+  const passedValidations = results.reduce(
+    (sum, r) => sum + r.validationResults.filter((v) => v.passed).length,
+    0,
+  );
+
+  // Print summary
+  logger.info(
+    {
+      total: results.length,
+      passed: successCount,
+      validation_failed: warningCount,
+      timeout: timeoutCount,
+      failed: failCount,
+      total_duration: totalDuration,
+      validations: {
+        total: totalValidations,
+        passed: passedValidations,
+        failed: totalValidations - passedValidations,
+      },
+      dir: setupTmpDir.name,
+    },
+    "Evaluation completed",
+  );
+
+  // Exit with error code if any task failed (excluding timeouts and validation failures)
+  if (failCount > 0) {
+    process.exit(1);
+  }
+  
+  // Exit successfully - ensures process terminates even with open handles
+  process.exit(0);
+}
+
+main().catch((error) => {
+  logger.error({ error: error.message }, "Fatal error");
+  process.exit(1);
+});

benchmarks/command-generator.ts

@@ -0,0 +1,98 @@
+import Handlebars from "handlebars";
+
+/**
+ * Loads data from CSV content string (internal helper)
+ */
+function loadCsvData(csvContent: string): Record<string, string>[] {
+  const lines = csvContent.trim().split("\n");
+  if (lines.length === 0) {
+    return [];
+  }
+
+  const firstLine = lines[0];
+  if (!firstLine) {
+    return [];
+  }
+
+  const headers = firstLine.split(",").map((h) => h.trim());
+  const data: Record<string, string>[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line || !line.trim()) continue;
+
+    const values = line.split(",").map((v) => v.trim());
+    const row: Record<string, string> = {};
+
+    for (let j = 0; j < headers.length; j++) {
+      const header = headers[j];
+      if (header) {
+        row[header] = values[j] || "";
+      }
+    }
+
+    data.push(row);
+  }
+
+  return data;
+}
+
+/**
+ * Creates a cross product of multiple data arrays (internal helper)
+ */
+function createCrossProduct(
+  sourcesData: Record<string, string>[][]
+): Record<string, string>[] {
+  if (sourcesData.length === 0) {
+    return [];
+  }
+
+  return sourcesData.reduce((acc, sourceData) => {
+    if (acc.length === 0) {
+      return sourceData;
+    }
+
+    const result: Record<string, string>[] = [];
+    for (const accItem of acc) {
+      for (const sourceItem of sourceData) {
+        result.push({ ...accItem, ...sourceItem });
+      }
+    }
+    return result;
+  }, [] as Record<string, string>[]);
+}
+
+/**
+ * Generates a command from a template and data context
+ */
+export function generateCommand(
+  commandTemplate: string,
+  context: Record<string, string>
+): string {
+  const template = Handlebars.compile(commandTemplate, {
+    strict: true,
+  });
+  return template(context);
+}
+
+/**
+ * Generates all commands from a template and cross product data (internal helper)
+ */
+function generateCommands(
+  commandTemplate: string,
+  sourcesData: Record<string, string>[][]
+): string[] {
+  const crossProduct = createCrossProduct(sourcesData);
+  return crossProduct.map((context) =>
+    generateCommand(commandTemplate, context)
+  );
+}
+
+/**
+ * Pure function to get contexts from sources data
+ */
+export function getContextsFromSources(
+  sourcesData: Record<string, string>[][]
+): Record<string, string>[] {
+  return createCrossProduct(sourcesData);
+}

benchmarks/evals/echo/task.yml

@@ -0,0 +1,26 @@
+run:
+  - git clone --depth=1 --branch main https://github.com/antinomyhq/forge
+  - ls -la
+  - echo "Forge deployment: $(readlink $(which forgee))"
+  - echo '{{message}}'
+parallelism: 5
+timeout: 10
+validations:
+  - name: "Output contains Hello"
+    type: regex
+    regex: "Hello"
+  - name: "Contains Hello using grep"
+    type: shell
+    command: 'grep -q "Hello"'
+    exit_code: 0
+  - name: "Output is not empty"
+    type: shell
+    command: 'grep -q "."'
+    exit_code: 0
+sources:
+  - value:
+      - message: "Hello, World!"
+      - message: "Hello from test message 123"
+      - message: "Hello with special chars: !@#$%"
+      - message: "Hello multi word message with spaces"
+      - message: "Hello with numbers: 42 and 3.14"

benchmarks/evals/multi_file_patch/multi_file_patch_tasks.csv

@@ -0,0 +1,5 @@
+task
+"Add a comment // TODO: Refactor this module at the top of these three files: crates/forge_domain/src/lib.rs, crates/forge_app/src/lib.rs, and crates/forge_infra/src/lib.rs"
+"Add the line edition = ""2021"" to the [package] section in these three Cargo.toml files: crates/forge_domain/Cargo.toml, crates/forge_app/Cargo.toml, and crates/forge_infra/Cargo.toml"
+"Add ## License as a new section at the end of these markdown files: README.md, AGENTS.md, and docs/tool-guidelines.md"
+"Add the import use anyhow::Result; at the top of these files (after existing imports): crates/forge_domain/src/lib.rs, crates/forge_app/src/lib.rs, and crates/forge_repo/src/lib.rs"

benchmarks/evals/multi_file_patch/task.yml

@@ -0,0 +1,21 @@
+run:
+  # Clone into `tmp/task` dir
+  - git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
+  - FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
+parallelism: 50
+timeout: 120
+early_exit: true
+validations:
+  - name: "Should use patch tool (not git apply)"
+    type: shell
+    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length > 0' {{dir}}/context.json"
+  - name: "Should NOT use shell with git apply"
+    type: shell
+    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"shell\" and (.function.arguments | contains(\"git apply\") or contains(\"git patch\")))] | length == 0' {{dir}}/context.json"
+  - name: "Should call patch tool multiple times for multi-file changes"
+    type: shell
+    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length >= 3' {{dir}}/context.json"
+sources:
+  - value:
+      - model: "openai/gpt-5.2"
+  - csv: multi_file_patch_tasks.csv

benchmarks/evals/patch_exact_match/patch_exact_match_tasks.csv

@@ -0,0 +1,7 @@
+task
+"Remove all backticks from test_files/plan.md. Make multiple separate patches to remove backticks from different sections."
+"Update test_files/plan.md to remove markdown code formatting. Apply patches to remove backticks from each task rationale."
+"Fix test_files/plan.md by converting backtick-wrapped identifiers. Use patch for each occurrence."
+"Clean up test_files/plan.md by removing backticks. Patch each task description separately."
+"In test_files/plan.md, convert backtick-enclosed terms. Use patch operations for different lines."
+"Remove markdown code formatting (backticks) from test_files/plan.md. Apply multiple patches."

benchmarks/evals/patch_exact_match/task.yml

@@ -0,0 +1,217 @@
+run:
+  - mkdir -p test_files
+  - |
+    cat > test_files/plan.md << 'EOF'
+    # Support for Remote Tools (Provider-Executed Tools)
+
+    ## Objective
+
+    Enable support for remote tools (also known as provider-executed tools) such as websearch, which are executed by the AI provider's infrastructure rather than by the client application. This allows providers like OpenAI to offer specialized capabilities like web search, code execution, and file retrieval that run on their servers and return results directly to the application.
+
+    ## Implementation Plan
+
+    - [ ] 1. Extend the ToolDefinition structure to support provider-executed tools
+      Add a new field to the `ToolDefinition` struct to distinguish between client-executed tools (current behavior) and provider-executed tools (new capability). This field will indicate that the tool should be passed directly to the provider without local execution logic.
+
+      Rationale: The current `ToolDefinition` at `crates/forge_domain/src/tools/definition/tool_definition.rs:12` only contains name, description, and input_schema. To support provider-executed tools, we need a way to mark which tools are handled by the provider infrastructure. This is similar to how Vercel AI SDK distinguishes between `function` type tools and `provider` type tools.
+
+      Affected files:
+      - `crates/forge_domain/src/tools/definition/tool_definition.rs` - Add field to ToolDefinition
+      - `crates/forge_repo/src/conversation/conversation_record.rs` - Update serialization/deserialization
+
+    - [ ] 2. Create provider-specific tool definitions for OpenAI Responses API
+      Define a set of provider-executed tools for OpenAI's Responses API, including `web_search`, `code_interpreter`, `file_search`, and other provider-native capabilities. These will be represented as `ToolDefinition` instances marked as provider-executed.
+
+      Rationale: OpenAI's Responses API provides built-in tools like `web_search` that are executed by OpenAI's infrastructure. These tools need to be defined with their specific schemas and parameters (e.g., searchContextSize, userLocation for web_search) similar to how Vercel AI SDK's `prepareResponsesTools` function handles these tools. This allows the application to offer these capabilities without implementing the execution logic.
+
+      Affected files:
+      - `crates/forge_app/src/dto/openai/provider_tools.rs` - New file for provider tool definitions
+      - `crates/forge_app/src/dto/openai/mod.rs` - Export new module
+
+    - [ ] 3. Implement tool conversion logic for provider-executed tools
+      Create a conversion function that transforms provider-executed `ToolDefinition` instances into the provider-specific format required by the API. For OpenAI, this means converting provider tools to the appropriate structure (e.g., `{type: "web_search", ...}`) instead of the standard function tool format.
+
+      Rationale: The current `From<ToolDefinition> for Tool` implementation at `crates/forge_app/src/dto/openai/request.rs:297` converts all tools to the standard function format. Provider-executed tools need special handling to be converted to their provider-specific formats. For OpenAI, this means converting provider tools to the appropriate structure with a type field indicating the specific tool type instead of using the standard function tool format. This mirrors Vercel AI SDK's `prepareResponsesTools` approach where provider tools are handled differently from function tools.
+
+      Affected files:
+      - `crates/forge_app/src/dto/openai/request.rs` - Modify Tool conversion logic
+      - `crates/forge_app/src/dto/openai/provider_tools.rs` - Implement conversion functions
+
+    - [ ] 4. Extend the Context to Tool request conversion
+      Update the `From<Context> for Request` implementation at `crates/forge_app/src/dto/openai/request.rs:323` to handle provider-executed tools. This involves filtering and converting provider tools separately from standard function tools when building the request.
+
+      Rationale: The current implementation treats all tools uniformly and converts them using the same logic. Provider-executed tools need to be identified and converted using the new provider-specific conversion logic before being included in the API request. This ensures that tools like `web_search` are sent in the correct format to OpenAI.
+
+      Affected files:
+      - `crates/forge_app/src/dto/openai/request.rs` - Update Context to Request conversion
+
+    - [ ] 5. Update the tool registry to handle provider-executed tool results
+      Modify the `ToolRegistry` at `crates/forge_app/src/tool_registry.rs:21` to recognize that provider-executed tools do not require local execution. When a provider tool is invoked, the tool registry should pass the result directly from the provider response without attempting local execution.
+
+      Rationale: Currently, the tool registry attempts to execute all tools locally through `ToolExecutor`, `AgentExecutor`, or `McpExecutor`. Provider-executed tools are different - the provider executes them and returns the result. The tool registry needs to distinguish between these cases and handle provider tool results appropriately.
+
+      Affected files:
+      - `crates/forge_app/src/tool_registry.rs` - Add logic to skip execution for provider tools
+      - `crates/forge_app/src/tool_registry.rs` - Handle provider tool results from responses
+
+    - [ ] 6. Add provider tool configuration to agent definitions
+      Extend the agent definition to include optional provider tool configurations. This allows agents to be configured with specific provider-executed tools (e.g., `web_search` enabled with specific parameters) while maintaining compatibility with existing agent configurations.
+
+      Rationale: Different agents may require different provider tools. For example, a research agent might need `web_search` enabled, while a coding agent might need `code_interpreter`. Adding this configuration to agent definitions provides fine-grained control over which provider tools are available to each agent.
+
+      Affected files:
+      - `crates/forge_domain/src/agent_definition.rs` - Add provider tools configuration
+      - `crates/forge_app/src/dto/tools_overview.rs` - Include provider tools in overview
+
+    - [ ] 7. Create provider tool factory and registration
+      Implement a factory pattern for registering provider tools with the system. This factory will create provider tool definitions for different providers (OpenAI, Anthropic, etc.) and make them available to agents that request them.
+
+      Rationale: Different providers support different provider-executed tools. A factory pattern allows the system to create the appropriate tool definitions based on the provider being used. This is extensible - new providers can be added by implementing their tool factory without modifying core logic.
+
+      Affected files:
+      - `crates/forge_app/src/provider_tools/factory.rs` - New file for tool factory
+      - `crates/forge_app/src/provider_tools/mod.rs` - New module
+      - `crates/forge_app/src/dto/tools_overview.rs` - Integrate with factory
+
+    - [ ] 8. Update the response parsing to handle provider tool results
+      Modify the response parsing logic in `crates/forge_app/src/dto/openai/response.rs` to correctly parse and handle tool results from provider-executed tools. This includes recognizing when a tool result comes from a provider tool and ensuring it's properly formatted.
+
+      Rationale: Provider tool results may have different structures than standard tool results. The response parser needs to handle these differences and convert them to the internal `ToolResult` format consistently. This ensures that tool results are processed correctly regardless of whether they came from a client-executed or provider-executed tool.
+
+      Affected files:
+      - `crates/forge_app/src/dto/openai/response.rs` - Update tool result parsing
+      - `crates/forge_app/src/dto/openai/response.rs` - Handle provider-specific result formats
+
+    - [ ] 9. Add provider tool support to the tool overview API
+      Update the `ToolsOverview` structure and related API endpoints to include provider-executed tools alongside system tools, agent tools, and MCP tools. This allows clients to discover which provider tools are available.
+
+      Rationale: The current `ToolsOverview` at `crates/forge_app/src/dto/tools_overview.rs:9` includes system tools, agents, and MCP tools. Provider tools should be included in this overview so clients can see all available tools, including those executed by providers. This maintains consistency with the existing tool discovery mechanism.
+
+      Affected files:
+      - `crates/forge_app/src/dto/tools_overview.rs` - Add provider tools field
+      - `crates/forge_app/src/tool_registry.rs` - Update tools_overview method
+
+    - [ ] 10. Implement Anthropic provider tools support
+      Create provider tool definitions for Anthropic's provider-executed tools (e.g., `computer_20241022`, `textEditor_20241022`, `bash_20241022`, `webSearch_20250305`, `codeExecution_20250522`). This demonstrates the extensibility of the provider tool system to multiple providers.
+
+      Rationale: Anthropic offers provider-executed tools similar to OpenAI but with different names and capabilities. Implementing support for Anthropic provider tools validates that the architecture is provider-agnostic and extensible. It follows the same pattern established for OpenAI tools.
+
+      Affected files:
+      - `crates/forge_app/src/dto/anthropic/provider_tools.rs` - New file for Anthropic provider tools
+      - `crates/forge_app/src/dto/anthropic/request.rs` - Update conversion logic
+      - `crates/forge_app/src/provider_tools/factory.rs` - Add Anthropic tool factory
+
+    - [ ] 11. Add tests for provider tool conversion and execution
+      Create comprehensive tests covering provider tool definition, conversion to provider formats, integration with Context to Request conversion, and handling of provider tool results. Tests should verify that provider tools are correctly formatted and that client-executed tools remain unaffected.
+
+      Rationale: Provider tools introduce new code paths and conversion logic that need thorough testing. Tests should ensure that provider tools are correctly converted to their provider-specific formats, that they don't interfere with standard tool execution, and that the system handles mixed scenarios (both provider and client tools) correctly.
+
+      Affected files:
+      - `crates/forge_app/src/dto/openai/request.rs` - Add conversion tests
+      - `crates/forge_app/src/dto/anthropic/request.rs` - Add conversion tests
+      - `crates/forge_app/src/tool_registry.rs` - Add execution handling tests
+
+    - [ ] 12. Update documentation and examples
+      Document the provider tool system, including how to define provider tools, how they are converted to provider formats, and how agents can be configured to use them. Include examples of using `web_search` and other provider tools.
+
+      Rationale: Provider tools are a new concept in the system. Documentation is essential for developers to understand when and how to use provider-executed tools versus client-executed tools. Examples should demonstrate practical usage patterns and configuration options.
+
+      Affected files:
+      - `docs/provider-tools.md` - New documentation file
+      - `examples/` - Add example configurations
+
+    ## Verification Criteria
+
+    - [ ] Provider tool definitions can be created and marked as provider-executed
+    - [ ] Provider tools are correctly converted to OpenAI-specific format (for example, using the type field to specify web search instead of function format)
+    - [ ] Provider tools are correctly converted to Anthropic-specific format
+    - [ ] Context to Request conversion includes provider tools in the correct format
+    - [ ] Tool registry recognizes provider tools and skips local execution
+    - [ ] Provider tool results from API responses are correctly parsed and handled
+    - [ ] Agent definitions can be configured with provider tools
+    - [ ] Tool overview API includes provider tools alongside other tool types
+    - [ ] All existing tests pass without modification
+    - [ ] New tests cover provider tool conversion, execution, and result handling
+    - [ ] Documentation describes the provider tool system with examples
+
+    ## Potential Risks and Mitigations
+
+    1. **Breaking changes to existing tool definitions**
+       Risk: Adding a new field to `ToolDefinition` could break existing serialization/deserialization or require migrations.
+
+       Mitigation: Make the new field optional with a default value of `false` (client-executed). This ensures backward compatibility with existing tool definitions. Run all existing tests to verify no regressions.
+
+    2. **Provider tool format incompatibility**
+       Risk: Different providers may have incompatible tool formats, making it difficult to create a unified abstraction.
+
+       Mitigation: Use provider-specific conversion modules (e.g., `openai_provider_tools`, `anthropic_provider_tools`) that handle format conversion independently. The common `ToolDefinition` serves as an abstraction, but each provider has its own conversion logic.
+
+    3. **Tool result handling complexity**
+       Risk: Provider tool results may have different structures that are difficult to normalize into a common `ToolResult` format.
+
+       Mitigation: Create provider-specific result parsers that normalize results to the internal format. Handle unknown or unexpected result formats gracefully with appropriate error messages.
+
+    4. **Execution model confusion**
+       Risk: Developers may be confused about which tools are executed locally versus by the provider, leading to unexpected behavior.
+
+       Mitigation: Clearly document the distinction in the `ToolDefinition` structure and provide tool overview information that indicates execution location (client vs. provider). Use descriptive naming conventions.
+
+    5. **Testing complexity**
+       Risk: Testing provider tools requires mocking provider responses, which can be complex and brittle.
+
+       Mitigation: Create integration tests that use real provider APIs for critical paths, supplemented by unit tests with mocked responses. Use snapshot testing to verify correct request formatting.
+
+    ## Alternative Approaches
+
+    1. **Separate tool type enum instead of boolean flag**
+       Instead of adding a boolean field to `ToolDefinition`, create an enum `ToolExecutionType` with variants `Client` and `Provider`. This is more explicit and extensible, allowing future execution types (e.g., `Hybrid`, `Delegated`).
+
+       Trade-offs: More explicit and type-safe, but adds complexity. The boolean flag is simpler for the current use case and can be extended to an enum later if needed.
+
+    2. **Provider-specific tool registries**
+       Create separate tool registries for each provider (OpenAI, Anthropic, etc.) rather than a unified registry. Each provider would manage its own tools, including provider-executed tools.
+
+       Trade-offs: Cleaner separation of concerns and easier provider-specific logic, but increases code duplication and makes it harder to share tools across providers.
+
+    3. **Dynamic tool discovery from provider APIs**
+       Instead of hard-coding provider tool definitions, query provider APIs to discover available tools and their schemas dynamically. This would automatically support new provider tools as they are added.
+
+       Trade-offs: More flexible and future-proof, but introduces runtime dependencies on provider APIs and may be slower. Requires robust error handling for API failures.
+
+    4. **Tool execution delegation pattern**
+       Implement a delegation pattern where tools specify an executor (client, provider, MCP, etc.) through a trait. The tool registry would delegate execution based on the executor specified by the tool.
+
+       Trade-offs: More flexible and extensible architecture, but adds abstraction layers and complexity. The current approach of checking tool type is simpler for the immediate use case.
+    EOF
+  - |  
+      FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
+parallelism: 2
+timeout: 180
+early_exit: true
+validations:
+  - name: "Should use patch tool"
+    type: shell
+    command: "jq -e '[.messages[]?.tool_calls[]? | select(.function.name == \"patch\")] | length > 0' {{dir}}/context.json"
+    
+  - name: "Should NOT have patch failures due to missing operation field"
+    type: shell
+    command: "! jq -e '[.messages[]? | select(.role == \"tool\" and .name == \"patch\" and (.content | contains(\"missing field\") and contains(\"operation\")))] | length > 0' {{dir}}/context.json"
+    
+  - name: "Should NOT have patch failures due to text mismatch"
+    type: shell
+    command: "! jq -e '[.messages[]? | select(.role == \"tool\" and .name == \"patch\" and (.content | contains(\"Could not find match for search text\")))] | length > 0' {{dir}}/context.json"
+    
+  - name: "Patch search text should match file content exactly"
+    type: shell
+    command: |
+      # Extract all patch tool calls and verify search text exists in the file
+      # This is a heuristic check - we look for patch calls that should have succeeded
+      jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "patch")] | length > 0' {{dir}}/context.json
+
+sources:
+  - value:
+      - model: "anthropic/claude-sonnet-4.5"
+      - model: "z-ai/glm-4.6:exacto"
+      - model: "minimax/minimax-m2.1"
+
+  - csv: patch_exact_match_tasks.csv

benchmarks/evals/redundant_cd_with_cwd/task.yml

@@ -0,0 +1,45 @@
+run:
+  - git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
+  - mkdir ../test-dir && cd ../test-dir
+  - FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
+parallelism: 5
+timeout: 120
+early_exit: false  # We want to see all cases, not just the first success
+
+validations:
+  # Check if any shell tool call contains "cd" command when cwd is also provided
+  - name: "No redundant cd with cwd"
+    type: shell
+    command: |
+      jq -e '
+        [.messages[]?.tool_calls[]? | 
+         select(.function.name == "shell") | 
+         .function.arguments | fromjson | 
+         select((.command | contains("cd "))) |
+         {command: .command, cwd: .cwd}
+        ] | length == 0
+      ' {{dir}}/context.json
+
+  # Detect specific pattern: "cd <path> && <command>" with cwd set
+  - name: "No cd && pattern with cwd"
+    type: shell
+    command: |
+      jq -e '
+        [.messages[]?.tool_calls[]? | 
+         select(.function.name == "shell") | 
+         .function.arguments | fromjson | 
+         select((.command | test("cd\\s+[^;]+\\s*&&"))) |
+         {command: .command, cwd: .cwd, issue: "redundant_cd_with_cwd"}
+        ] | length == 0
+      ' {{dir}}/context.json
+
+sources:
+  - value:
+      - model: "anthropic/claude-sonnet-4.5"
+      - model: "z-ai/glm-4.6:exacto"
+  - value:
+      - task: "clean ~/tailcall"
+      - task: "format code in ~/task/crates/forge_domain"
+      - task: "install dependencies in ~/task/benchmarks"
+      - task: "cd to ~/task/crates and run cargo test"
+      - task: "go to ~/task/docs and list files"

benchmarks/evals/search_over_find/task.yml

@@ -0,0 +1,58 @@
+run:
+  - git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
+  - FORGE_DEBUG_REQUESTS='{{dir}}/context.json' forgee --provider open_router --model {{model}} -p '{{task}}'
+parallelism: 10
+timeout: 180
+early_exit: true
+
+validations:
+  # Primary: Model should NOT use shell find command for searching code
+  - name: "No shell find command usage"
+    type: shell
+    command: |
+      jq -e '
+        [.messages[]?.tool_calls[]? | 
+         select(.function.name == "shell") | 
+         .function.arguments | fromjson | 
+         select(.command | test("\\bfind\\s+"))
+        ] | length == 0
+      ' {{dir}}/context.json
+
+  # Model SHOULD use builtin search tools instead
+  - name: "Uses builtin search tools"
+    type: shell
+    command: |
+      jq -e '
+        [.messages[]?.tool_calls[]? | 
+         select(.function.name == "search" or 
+                .function.name == "fs_search" or 
+                .function.name == "sem_search")
+        ] | length > 0
+      ' {{dir}}/context.json
+
+  # Detect specific anti-pattern: find with grep pipeline
+  - name: "No find + grep anti-pattern"
+    type: shell
+    command: |
+      jq -e '
+        [.messages[]?.tool_calls[]? | 
+         select(.function.name == "shell") | 
+         .function.arguments | fromjson | 
+         select(.command | test("find.*\\|.*grep"))
+        ] | length == 0
+      ' {{dir}}/context.json
+
+sources:
+  - value:
+      - model: "anthropic/claude-sonnet-4.5"
+      - model: "z-ai/glm-4.6:exacto"
+  - value:
+      # File discovery tasks - should use fs_search, NOT find command
+      - task: "I need to merge service and app crates. Tell me what files exist in the crates/ directory."
+      - task: "Find all Rust source files in the crates/service directory."
+      - task: "List all Rust files in crates/app to understand the structure."
+      - task: "Show me all .rs files in the project so I can plan refactoring."
+      
+      # Structure understanding - should use fs_search or sem_search
+      - task: "Find all source files under crates/service and crates/app for analysis."
+      - task: "List all Rust source files in the workspace for documentation purposes."

benchmarks/evals/sem_search/README.md

@@ -0,0 +1,33 @@
+# Semantic Search Tool Selection Evaluation
+
+This evaluation validates that the Forge agent correctly identifies and uses the `sem_search` (Codebase Search) tool when presented with conceptual, functionality-based code location queries.
+
+## What We're Testing
+
+The agent's ability to recognize when a user query requires semantic understanding rather than exact pattern matching. Specifically, we test whether the agent:
+
+1. **Correctly invokes `sem_search`** for conceptual queries about code functionality
+2. **Understands the distinction** between semantic search (conceptual) vs regex search (exact patterns)
+3. **Recognizes appropriate use cases** such as:
+   - Finding code by its purpose/behavior (e.g., "retry logic with exponential backoff")
+   - Locating implementation patterns (e.g., "authentication token validation")
+   - Discovering architectural components (e.g., "message transformation between AI providers")
+   - Identifying system behaviors (e.g., "rate limiting for API requests")
+
+## Test Scenarios
+
+The evaluation uses real-world queries that describe:
+- **Message transformation** between different AI provider formats
+- **Retry mechanisms** with exponential backoff patterns  
+- **Authentication flows** including token validation and refresh
+- **Error handling** for network failures
+- **Caching strategies** for API responses
+- **Streaming response** handling from LLM APIs
+- **File validation** including upload and size checks
+- **Rate limiting** implementations for API requests
+- **Tool registration** and availability management
+- **Context management** including conversation history truncation
+
+## Expected Behavior
+
+For all test queries, the agent should invoke the `sem_search` tool (displayed as "Codebase Search" in logs) rather than falling back to regex-based `search` tool, demonstrating understanding that these are conceptual queries requiring semantic code comprehension.

benchmarks/evals/sem_search/task.yml

@@ -0,0 +1,138 @@
+run:
+  # Clone into `tmp/task` dir
+  - git clone --depth=1 --branch main https://github.com/tailcallhq/forgecode .
+  - forge workspace init --yes
+  - forge workspace sync
+  - FORGE_DEBUG_REQUESTS='{{dir}}/context.jsonl' FORGE_SESSION__PROVIDER_ID=open_router FORGE_SESSION__MODEL_ID={{model}} forge -p '{{task}}'
+parallelism: 8
+timeout: 60
+early_exit: true
+validations:
+  - name: "Uses codebase search tool"
+    type: shell
+    command: grep -q "Codebase Search" '{{dir}}/task.log'
+  - name: "Does not call task tool before sem_search"
+    type: shell
+    command: "python3 -c \"\nimport sys\nlog = open('{{dir}}/task.log').read().splitlines()\ntask_line = next((i for i, l in enumerate(log) if '[Agent]' in l), None)\nsearch_line = next((i for i, l in enumerate(log) if 'Codebase Search' in l), None)\nif task_line is None: sys.exit(0)\nif search_line is None: sys.exit(1)\nsys.exit(0 if search_line < task_line else 1)\n\""
+sources:
+  - value:
+      # - model: "x-ai/grok-code-fast-1"
+      - model: "anthropic/claude-sonnet-4.6"
+  - value:
+      # # Location questions - "where is"
+      # - task: "Where is the code that transforms messages between different AI provider formats?"
+      # - task: "Where is the error handling code for network failures?"
+      # - task: "Where is file upload validation and size checking implemented?"
+      # - task: "Where are environment variables parsed and configuration loaded?"
+      # - task: "Where is the logic for selecting which AI model to use for a task?"
+
+      # # Discovery questions - "find"
+      # - task: "Find the code that caches API responses to reduce costs"
+      # - task: "Find where tool definitions are registered and made available to the AI"
+      # - task: "Find the code that manages API keys, tokens, and credentials securely"
+      # - task: "Find where Git operations like commit, diff, and status are implemented"
+
+      # # Understanding questions - "how does"
+      # - task: "How does the system handle differences between OpenAI and Anthropic APIs?"
+      # - task: "How does the tool execution pipeline process and validate tool calls?"
+      # - task: "How are streaming responses from LLMs processed?"
+
+      # # Need-based questions - "I need to"
+      # - task: "I need to find retry logic with exponential backoff for API calls"
+      # - task: "I need to understand how streaming responses from LLM APIs work"
+      # - task: "I need to see how conversation history is truncated when context is too large"
+      # - task: "I need to find examples of rate limiting implementations"
+
+      # # Show me questions
+      # - task: "Show me where authentication tokens are validated and refreshed"
+      # - task: "Show me the code that handles rate limiting for API requests"
+      # - task: "Show me the code that compacts conversation context when tokens exceed limits"
+      # - task: "Show me how the system handles request timeouts and cancellation"
+      # - task: "Show me the permission checking and security policy enforcement code"
+
+      # # Strategy/pattern questions
+      # - task: "I want to understand the caching strategy for semantic search embeddings"
+      # - task: "What's the strategy for managing concurrent API requests?"
+      # - task: "What's the approach for handling API errors and retries?"
+
+      # # Troubleshooting questions
+      # - task: "What could cause API requests to fail?"
+      # - task: "How are timeout errors handled in the system?"
+
+      # # Architecture questions
+      # - task: "How do the different AI provider adapters integrate with the main system?"
+      # - task: "What's the flow of a user request from input to response?"
+      # - task: "How are tools registered and executed?"
+
+      # # Capability questions
+      # - task: "How does the system track token usage?"
+      # - task: "What mechanism prevents infinite loops in tool execution?"
+
+      # Human review tasks - integrated from human-review-sem-search-tasks.json
+      - task: "Now that we support piping, how can I use an external editor to type a large command and pipe it to the forge cli?"
+      - task: "Is there a type-safe version of commander that can make sure the command is parsed into a type-safe object"
+      - task: "Search for ZAI-related model fetching code in the codebase"
+      - task: "Look for API endpoints and authentication patterns used for ZAI"
+      - task: "Starting a new session always loads the older node version"
+      - task: "Examine any existing curl examples or HTTP client implementations"
+      - task: "Before  sorting the tools we will dedupe all the tools"
+      - task: "Identify the current embedding implementation and how it interfaces with OpenRouter"
+      - task: "Is there a place in the code base where we are subscribing to events?"
+      - task: "There is a bug, where when I paste text the syntax highlighting is lost for a moment until some key is pressed again."
+      - task: "Search for any existing validation patterns in the forge_domain crate, particularly looking for validation methods, validate functions, or error types used for domain validation"
+      - task: "Show me the temperature type"
+      - task: "Search for cancellation mechanisms including cancellation tokens, abort signals, task cancellation, and how the system handles user interrupts or cancellation of in-flight requests across all crates"
+      - task: "In the workspace info, we should return the creation time of the workspace too."
+      - task: "token cost is being sent but we are not able to show it on the ui. Our cost conversation in some place is getting missed."
+      - task: "Look at workflow-related files to understand how model should be obtained from API"
+      - task: "Lets use the newtype crate for all the type in domain that are newtypes so that we don't need to implement all this boilerplate"
+      - task: "Look for any existing patterns in the codebase for ensuring migrations run exactly once across all tests, such as global mutexes or initialization guards"
+      - task: "The clear conversation doesn't happen on CMD+K."
+      - task: "Use plimit and prompt.all to run it concurrently"
+      - task: "Like infra I need another crate called - test_infra that looks similar and has a ForgeInfra struct that has test implementations of stuff."
+      - task: "Infra query nodes should take Query<Embedding> instead of each param as a separate arg."
+      - task: "Add health check endpoints for monitoring"
+      - task: "Examine conversation ID generation and management in domain layer"      
+      - task: "Lets recursively derive Dummy from fake, on Node and its children."
+      - task: "Add a new infra component called Reranker. It takes a list of nodes and reranks them."
+      - task: "Update the implementations of Cache based on the new trait implementation."
+      - task: "In skill lets add description also"
+      - task: "Investigate signal handling and graceful shutdown mechanisms in /Volumes/Bran/code-forge-workspace/code-forge, particularly looking for SIGINT, SIGTERM handling and how they affect ongoing operations"
+      - task: "Lets add a new range.rs under arbitrary/ and implement Arbitrary for all the std Range types eg: Range, RangeFrom etc."
+      - task: "Make `-v` optional. It should not be required."
+      - task: "in ProviderRegistry I need a few more methods - get_active_provider, set_active_provider and get_all_providers"
+      - task: "Lets use derive_setters on Config. Drop the handwritten setters"
+      - task: "Lets drop path and env from the `info` command. Its a bit too noisy."
+      - task: "The Secret type should print the last 4 chars of the secret in debug."
+      - task: "What's the purpose of ModelId?"
+      - task: "Where is the Agent struct defined?"
+      - task: "**Questions**\n\n1. Switching agent happens via CLI `forge config --set-agent muse`  or `forge config -i` opens up an interactive configuration manager (RATATUI)\n2. How do I know the active agent and model? `forge config --get-agent` show in via a ZSH theme\n3. Additional CLI commands to be added\n    1.  `forge conversation --set-active <cid>`\n    2. `forge info` Show current info including agent, model etc."
+      - task: "Find all places where event names are constructed or used, particularly in relation to agent IDs. Look for patterns like 'forge', 'muse', 'sage' in event names and how they map to agents."
+      - task: "Change the structure to be a simple - Vec of Vec of Option of String."
+      - task: "Where are the fixtures located?"
+      - task: "Move McpToolCache to forge_app dto"
+      - task: "based on your understanding of the project, where are the tables created in supbase?"
+      - task: "Examine how the truncation functions are called and where the results are used. Identify the call sites and understand the data flow from truncation to final output."
+      - task: "Like ShowCommands, I need ShowModels and ShowConversations"                              
+      - task: "Lets implement a function on SummaryMessage that can merge consecutive calls together. If the nth can be merged with nth-1, then we should remove the nth-1 and insert the new value."
+      - task: "Lets have upload take a glob pattern for upload"      
+      - task: "Check this for security issues: eval(user_input)"
+      - task: "Update the vector dimension to be 4096"
+      - task: "Fix this - 'docker run tailcallhq/forgecode'"      
+      - task: "Document the adapter registration and discovery mechanism"
+      - task: "There are multiple places where we throw - \"Conversation not found\" how do u suggest we consolidate and reuse."      
+      - task: "Is there a crate for humanizing date time format that we are using?"            
+      - task: "Lets move the domain compaction transformers into app/compact/transformers. Make sure you move them in bulk."
+      - task: "Identify the main entry point (main.rs or equivalent) and understand how the application initializes and starts the server."
+      - task: "Lets use chrono-humanize instead of humantime in forge_main everywhere."      
+      - task: "Look at all the docs in .rs files and remove the args and return info from the docs."
+      - task: "Investigate why each approach (1-4) has failed - what are the technical limitations we're hitting with ZLE widgets, terminal control sequences, and shell execution contexts"
+      - task: "as per rust best practices, what would you recommend should we call arbit? and how should the file be structured. Show me the best examples - purest form"            
+      - task: "Explain the flow of authentication in the project. I need to understand it deeply, going from high level to micro details."
+      - task: "Research xml.rs in forge_domain and create a plan to add helper functions for common XML operations"
+      - task: "Add JWT-based authentication system"
+      - task: "How does FinishReason work?"
+      - task: "Check what changes were made to api_key.rs conversions and identify why the key field might be getting the wrong value"
+      - task: "Trace the flow from skill listing to system prompt generation to identify where the integration is missing or broken"
+      - task: "What could cause memory leaks in long-running sessions?"      
+      - task: "I am not able to make send of these name - DefinitionBatchGenerator DefinitionBatch. I need something much better,"                  

benchmarks/evals/semantic_search_quality/README.md

@@ -0,0 +1,197 @@
+# Semantic Search Quality Evaluation
+
+This evaluation measures the quality of semantic search queries generated by Forge using an LLM-as-judge approach.
+
+## Quick Start
+
+### Test Your Queries Manually
+
+Before running the full eval, test your queries to get immediate feedback:
+
+```bash
+# Interactive mode
+npx tsx benchmarks/evals/semantic_search_quality/test_queries.ts
+
+# Non-interactive mode
+npx tsx benchmarks/evals/semantic_search_quality/test_queries.ts \
+  "retry mechanism with exponential backoff" \
+  "I need to find the implementation code for retry logic"
+```
+
+The query tester provides:
+- Detailed scores for embedding and reranking queries
+- Specific strengths and weaknesses
+- Actionable suggestions for improvement
+- Differentiation analysis
+
+### Run the Full Evaluation
+
+```bash
+# Run the entire semantic search quality eval
+npm run eval semantic_search_quality
+
+# Run with specific parallelism
+npm run eval semantic_search_quality -- --parallelism 10
+
+# Run with specific model
+npm run eval semantic_search_quality -- --model "anthropic/claude-sonnet-4.5"
+```
+
+## Purpose
+
+The semantic search feature uses two types of queries:
+1. **Embedding Query (`query`)**: Converted to vector embeddings for semantic similarity search
+2. **Reranking Query (`use_case`)**: Used by the reranker to filter results based on intent
+
+This eval verifies that:
+- Queries are well-formed and contain sufficient context
+- The two queries are different and serve distinct purposes
+- Results match the user's intent (implementation vs docs vs tests, etc.)
+- Correct file types are returned based on intent
+- Unwanted file types are avoided
+
+## How It Works
+
+### 1. Test Execution
+The eval runs various semantic search tasks covering different intents:
+- **Implementation**: Find actual code, not documentation
+- **Flow Understanding**: Understand how systems work
+- **Architecture**: Understand structural design
+- **Tests**: Find test examples and fixtures
+- **Documentation**: Find docs and guides
+- **Debugging**: Investigate issues
+- **Modification**: Find code to modify
+- **Configuration**: Find config files and schemas
+
+### 2. LLM Judge Evaluation
+After each task, an LLM judge (Gemini 3 Pro) evaluates:
+
+#### Query Quality (40 points)
+- **Embedding Query (15pts)**: Domain terms, technical context, behavior description
+- **Reranking Query (15pts)**: Intent clarity, context about WHY, specificity
+- **Query Differentiation (10pts)**: Queries serve different purposes
+
+#### Result Relevance (60 points)
+- **Intent Matching (25pts)**: Results match the stated intent
+- **File Type Accuracy (20pts)**: Correct file types returned
+- **Avoidance Compliance (15pts)**: Unwanted file types avoided
+
+#### Pass Criteria
+- Total score >= 70/100
+- No critical issues (e.g., identical queries, severe intent mismatch)
+
+### 3. Structured Output
+The judge uses structured output (Zod schema) to provide:
+- Detailed scores for each dimension
+- Specific feedback on what's good/bad
+- List of critical issues
+- Overall pass/fail determination
+
+## Environment Variables
+
+Required:
+- `GOOGLE_APPLICATION_CREDENTIALS`: Path to Google Cloud service account JSON credentials file
+
+Or authenticate using `gcloud auth application-default login`.
+
+The LLM judge uses Vertex AI to run Gemini 3 Pro. It supports Google's Application Default Credentials through the `google-auth-library`. The most common authentication method is to set the path to a JSON credentials file in the `GOOGLE_APPLICATION_CREDENTIALS` environment variable.
+
+## Test Cases
+
+The eval includes diverse test cases:
+
+### Implementation Queries
+```yaml
+- task: "Find the retry mechanism with exponential backoff implementation"
+  intent: "implementation"
+  expected_file_types: "rust,typescript,javascript"
+  should_avoid: "markdown,txt,documentation"
+```
+
+### Flow Understanding
+```yaml
+- task: "How does the authentication flow work from request to response?"
+  intent: "flow_understanding"
+  expected_file_types: "rust,typescript"
+  should_avoid: "test,markdown"
+```
+
+### Documentation
+```yaml
+- task: "Find documentation on how to configure semantic search"
+  intent: "documentation"
+  expected_file_types: "markdown,txt"
+  should_avoid: "rust,typescript,javascript"
+```
+
+## Interpreting Results
+
+### High Scores (>80)
+Queries are excellent - domain-specific, well-differentiated, and would return highly relevant results.
+
+### Medium Scores (60-80)
+Queries are functional but have room for improvement. Check feedback for specific suggestions.
+
+### Low Scores (<60)
+Queries have significant issues:
+- Too generic or vague
+- Embedding and reranking queries too similar
+- Intent mismatch
+- Would return wrong file types
+
+## Common Issues
+
+1. **Identical Queries**: Embedding and reranking queries are the same or too similar
+   - **Fix**: Make reranking query focus on intent, not just rephrasing
+
+2. **Generic Queries**: Lack domain-specific context
+   - **Fix**: Add technical terms, describe behavior, include framework names
+
+3. **Intent Mismatch**: Queries would return docs when implementation is needed
+   - **Fix**: Be explicit in reranking query about what type of code is needed
+
+4. **Poor Differentiation**: Both queries try to do the same thing
+   - **Fix**: Embedding = WHAT code does, Reranking = WHY you need it
+
+## Architecture
+
+```
+task.yml
+├── Runs Forge with semantic search task
+├── Captures debug output (context.json)
+└── Validates with llm_judge.ts
+    ├── Extracts sem_search calls
+    ├── Calls Gemini 3 Pro with structured output
+    └── Returns evaluation with scores
+```
+
+## Extending the Eval
+
+To add new test cases:
+
+1. Add to `task.yml` under `sources`:
+```yaml
+- task: "Your task description"
+  intent: "implementation|flow_understanding|tests|documentation|debugging|modification|architecture|configuration"
+  expected_file_types: "comma,separated,extensions"
+  should_avoid: "comma,separated,extensions"
+```
+
+2. Choose appropriate intent:
+   - `implementation`: Actual code implementation
+   - `flow_understanding`: Understanding how things work
+   - `architecture`: Structural/design understanding
+   - `tests`: Test files and fixtures
+   - `documentation`: Docs, README, guides
+   - `debugging`: Finding bugs/issues
+   - `modification`: Code to modify/extend
+   - `configuration`: Config files, schemas
+
+## Future Improvements
+
+- [ ] Add actual result verification (not just query quality)
+- [ ] Measure embedding quality directly
+- [ ] Test reranking effectiveness with real results
+- [ ] Add precision/recall metrics
+- [ ] Compare against human-labeled ground truth
+- [ ] Test edge cases (very short queries, multi-lingual code, etc.)

benchmarks/evals/semantic_search_quality/llm_judge.ts

@@ -0,0 +1,400 @@
+#!/usr/bin/env tsx
+
+/**
+ * LLM-as-Judge for Semantic Search Quality Evaluation
+ *
+ * This script evaluates the quality of semantic search queries and results
+ * using Google's Gemini 3 Pro via Vertex AI as a judge. It verifies:
+ * 1. Query quality - Are the embedding and reranking queries well-formed?
+ * 2. Result relevance - Do the results match the stated intent?
+ * 3. File type accuracy - Are the correct file types returned?
+ * 4. Avoidance compliance - Are unwanted file types properly avoided?
+ */
+
+import { readFileSync } from "fs";
+import { vertex } from "@ai-sdk/google-vertex";
+import { generateObject } from "ai";
+import { z } from "zod";
+
+// Evaluation result schema
+const EvaluationSchema = z.object({
+  query_quality: z.object({
+    embedding_query_score: z
+      .number()
+      .min(0)
+      .max(15)
+      .describe("Score for the embedding query quality (0-15)"),
+    embedding_query_feedback: z
+      .string()
+      .describe("Specific feedback on the embedding query"),
+    reranking_query_score: z
+      .number()
+      .min(0)
+      .max(15)
+      .describe("Score for the reranking query quality (0-15)"),
+    reranking_query_feedback: z
+      .string()
+      .describe("Specific feedback on the reranking query"),
+    construct_keywords_score: z
+      .number()
+      .min(0)
+      .max(10)
+      .describe(
+        "Score for codebase construct keywords in reranking query (0-10)",
+      ),
+    construct_keywords_feedback: z
+      .string()
+      .describe(
+        "Feedback on use of codebase constructs (struct, trait, impl, interface, class, function, definition, implementation, declaration)",
+      ),
+    queries_differentiated: z
+      .boolean()
+      .describe(
+        "Whether the embedding and reranking queries are sufficiently different",
+      ),
+  }),
+  result_relevance: z.object({
+    intent_match_score: z
+      .number()
+      .min(0)
+      .max(25)
+      .describe("How well results match the stated intent (0-25)"),
+    intent_match_feedback: z
+      .string()
+      .describe("Explanation of intent matching"),
+    file_type_accuracy_score: z
+      .number()
+      .min(0)
+      .max(20)
+      .describe("Correctness of file types in results (0-20)"),
+    file_type_feedback: z.string().describe("Feedback on file type selection"),
+    avoidance_compliance_score: z
+      .number()
+      .min(0)
+      .max(15)
+      .describe("How well unwanted file types were avoided (0-15)"),
+    avoidance_feedback: z.string().describe("Feedback on file type avoidance"),
+  }),
+  overall: z.object({
+    passed: z
+      .boolean()
+      .describe("Whether the search quality passes the evaluation"),
+    total_score: z
+      .number()
+      .min(0)
+      .max(100)
+      .describe("Total score (0-100)"),
+    summary: z
+      .string()
+      .describe("Brief summary of the evaluation (2-3 sentences)"),
+    critical_issues: z
+      .array(z.string())
+      .describe("List of critical issues that caused failure"),
+  }),
+});
+
+type Evaluation = z.infer<typeof EvaluationSchema>;
+
+interface ToolCall {
+  id: string;
+  type: string;
+  function: {
+    name: string;
+    arguments: string;
+  };
+}
+
+interface Message {
+  tool_calls?: ToolCall[];
+}
+
+interface ContextData {
+  messages: Message[];
+}
+
+interface Args {
+  context: string;
+  intent: string;
+  expectedFileTypes: string;
+  shouldAvoid: string;
+}
+
+function parseArgs(): Args {
+  const args = process.argv.slice(2);
+  const parsed: Record<string, string> = {};
+
+  for (let i = 0; i < args.length; i += 2) {
+    const key = args[i]?.replace(/^--/, "").replace(/-/g, "_");
+    const value = args[i + 1];
+    if (key && value) {
+      parsed[key] = value;
+    }
+  }
+
+  if (
+    !parsed.context ||
+    !parsed.intent ||
+    !parsed.expected_file_types ||
+    !parsed.should_avoid
+  ) {
+    console.error("Missing required arguments");
+    console.error(
+      "Usage: llm_judge.ts --context <file> --intent <intent> --expected-file-types <types> --should-avoid <types>",
+    );
+    process.exit(1);
+  }
+
+  return {
+    context: parsed.context,
+    intent: parsed.intent,
+    expectedFileTypes: parsed.expected_file_types,
+    shouldAvoid: parsed.should_avoid,
+  };
+}
+
+function extractSemanticSearchCalls(contextData: ContextData): Array<{
+  query: string;
+  use_case: string;
+}> {
+  const searches: Array<{ query: string; use_case: string }> = [];
+
+  for (const message of contextData.messages) {
+    if (message.tool_calls) {
+      for (const toolCall of message.tool_calls) {
+        if (toolCall.function.name === "sem_search") {
+          try {
+            const args = JSON.parse(toolCall.function.arguments);
+            if (args.queries && Array.isArray(args.queries)) {
+              for (const q of args.queries) {
+                if (q.query && q.use_case) {
+                  searches.push({
+                    query: q.query,
+                    use_case: q.use_case,
+                  });
+                }
+              }
+            }
+          } catch (e) {
+            // Skip invalid JSON
+          }
+        }
+      }
+    }
+  }
+
+  return searches;
+}
+
+async function evaluateWithLLM(
+  searches: Array<{ query: string; use_case: string }>,
+  intent: string,
+  expectedFileTypes: string,
+  shouldAvoid: string,
+): Promise<Evaluation> {
+  const prompt = `You are an expert evaluator of semantic code search quality. Your task is to evaluate the quality of semantic search queries and assess whether they would return relevant results based on the user's intent.
+
+## User's Intent
+The user wants to: ${intent}
+
+## Expected File Types
+The search should primarily return files of these types: ${expectedFileTypes}
+
+## File Types to Avoid
+The search should avoid returning files of these types: ${shouldAvoid}
+
+## Semantic Search Queries Generated
+${searches.map((s, i) => `### Query ${i + 1}
+**Embedding Query (for vector search):** ${s.query}
+**Reranking Query (for result filtering):** ${s.use_case}
+`).join("\n")}
+
+## Evaluation Criteria
+
+### Query Quality (50 points)
+1. **Embedding Query (15 points):**
+   - Contains domain-specific terms and technical context
+   - Describes behavior and functionality, not meta-descriptions
+   - Is complete and descriptive, not overly generic
+   - Would generate embeddings that match similar code
+
+2. **Reranking Query (15 points):**
+   - Expresses the specific intent and use case
+   - Provides context about WHY the code is needed
+   - Specifies whether implementation/docs/tests are needed
+   - Is different from the embedding query (not verbatim)
+
+3. **Codebase Construct Keywords (10 points):**
+   - **CRITICAL**: When looking for code (not documentation), the reranking query MUST include specific construct keywords
+   - These keywords get HIGH WEIGHTAGE in reranking: struct, trait, impl, interface, class, function, definition, implementation, declaration, type
+   - Examples of GOOD use_case queries:
+     * "I need the struct definition for User authentication"
+     * "Show me the trait implementation for DatabaseConnection"
+     * "Find the function implementation that handles file patching"
+     * "I need the type declarations and interface definitions for the tool registry"
+   - Examples of BAD use_case queries (missing construct keywords):
+     * "I need code that handles authentication" (missing: struct/trait/impl/function)
+     * "Show me the database logic" (missing: trait/impl/function)
+     * "Find file patching code" (missing: function/impl/struct)
+   - **Scoring**:
+     * 10 points: Multiple specific construct keywords (struct, trait, impl, function, etc.)
+     * 7-9 points: At least one specific construct keyword
+     * 4-6 points: Generic keywords (code, implementation) without specifics
+     * 0-3 points: No construct keywords at all
+   - **Exception**: Documentation/config intents don't need construct keywords
+
+4. **Query Differentiation (10 points):**
+   - The two queries serve different purposes
+   - Embedding query focuses on WHAT (semantic similarity)
+   - Reranking query focuses on INTENT + CONSTRUCTS (contextual relevance + code structure)
+
+### Result Relevance (50 points)
+Based on the queries, evaluate whether they would return appropriate results:
+
+1. **Intent Matching (25 points):**
+   - Would the queries find code matching the user's intent?
+   - For "implementation" intent: would it find actual code, not docs?
+   - For "documentation" intent: would it find docs, not code?
+   - For "tests" intent: would it find test files?
+   - For "flow_understanding": would it find the relevant code flow?
+   - For "debugging": would it find the problematic areas?
+   - For "modification": would it find the right code to modify?
+
+2. **File Type Accuracy (20 points):**
+   - Would the queries naturally return the expected file types?
+   - Are the queries specific enough to filter for the right extensions?
+
+3. **Avoidance Compliance (15 points):**
+   - Would the queries avoid returning the unwanted file types?
+   - Is the intent clear enough to prevent irrelevant results?
+
+## Scoring Guidelines
+- **9-10:** Excellent - Professional quality, no significant issues
+- **7-8:** Good - Minor improvements possible but functional
+- **5-6:** Adequate - Some issues but would work in most cases
+- **3-4:** Poor - Significant issues that impact effectiveness
+- **0-2:** Failing - Critical problems, would not work as intended
+
+## Pass/Fail Criteria
+- **PASS:** Total score >= 70/100 AND no critical issues
+- **FAIL:** Total score < 70/100 OR has critical issues
+
+Critical issues include:
+- Embedding and reranking queries are identical or nearly identical
+- Query is too generic to return relevant results
+- Intent mismatch is severe (e.g., asking for implementation but would return docs)
+- Would definitely return wrong file types for the intent
+
+Please evaluate these queries and provide detailed, constructive feedback.`;
+
+  try {
+    // @ts-ignore - Type instantiation depth issue with complex Zod schemas
+    const result = await generateObject({
+      model: vertex("gemini-3-pro-preview"),
+      schema: EvaluationSchema,
+      prompt,
+      temperature: 0.3, // Lower temperature for more consistent evaluations
+    });
+
+    return result.object as Evaluation;
+  } catch (error) {
+    console.error("Error calling Gemini API:", error);
+    throw error;
+  }
+}
+
+function formatEvaluation(evaluation: Evaluation): string {
+  const lines: string[] = [];
+
+  lines.push("=== SEMANTIC SEARCH QUALITY EVALUATION ===\n");
+
+  // Query Quality
+  lines.push("## Query Quality");
+  lines.push(
+    `Embedding Query: ${evaluation.query_quality.embedding_query_score}/15`,
+  );
+  lines.push(`  ${evaluation.query_quality.embedding_query_feedback}`);
+  lines.push(
+    `Reranking Query: ${evaluation.query_quality.reranking_query_score}/15`,
+  );
+  lines.push(`  ${evaluation.query_quality.reranking_query_feedback}`);
+  lines.push(
+    `Construct Keywords: ${evaluation.query_quality.construct_keywords_score}/10`,
+  );
+  lines.push(`  ${evaluation.query_quality.construct_keywords_feedback}`);
+  lines.push(
+    `Queries Differentiated: ${evaluation.query_quality.queries_differentiated ? "✓ Yes" : "✗ No"}`,
+  );
+  lines.push("");
+
+  // Result Relevance
+  lines.push("## Result Relevance");
+  lines.push(
+    `Intent Match: ${evaluation.result_relevance.intent_match_score}/25`,
+  );
+  lines.push(`  ${evaluation.result_relevance.intent_match_feedback}`);
+  lines.push(
+    `File Type Accuracy: ${evaluation.result_relevance.file_type_accuracy_score}/20`,
+  );
+  lines.push(`  ${evaluation.result_relevance.file_type_feedback}`);
+  lines.push(
+    `Avoidance Compliance: ${evaluation.result_relevance.avoidance_compliance_score}/15`,
+  );
+  lines.push(`  ${evaluation.result_relevance.avoidance_feedback}`);
+  lines.push("");
+
+  // Overall
+  lines.push("## Overall Assessment");
+  lines.push(`Total Score: ${evaluation.overall.total_score}/100`);
+  lines.push(`Status: ${evaluation.overall.passed ? "✓ PASS" : "✗ FAIL"}`);
+  lines.push(`Summary: ${evaluation.overall.summary}`);
+
+  if (evaluation.overall.critical_issues.length > 0) {
+    lines.push("\n## Critical Issues:");
+    for (const issue of evaluation.overall.critical_issues) {
+      lines.push(`  - ${issue}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+async function main() {
+  const args = parseArgs();
+
+  // Read context file
+  let contextData: ContextData;
+  try {
+    const contextContent = readFileSync(args.context, "utf-8");
+    contextData = JSON.parse(contextContent);
+  } catch (error) {
+    console.error(`Error reading context file: ${error}`);
+    process.exit(1);
+  }
+
+  // Extract semantic search calls
+  const searches = extractSemanticSearchCalls(contextData);
+
+  if (searches.length === 0) {
+    console.error("No semantic search calls found in context");
+    process.exit(1);
+  }
+
+  // Evaluate with LLM
+  const evaluation = await evaluateWithLLM(
+    searches,
+    args.intent,
+    args.expectedFileTypes,
+    args.shouldAvoid,
+  );
+
+  // Output results
+  console.log(formatEvaluation(evaluation));
+
+  // Exit with appropriate code
+  process.exit(evaluation.overall.passed ? 0 : 1);
+}
+
+main().catch((error) => {
+  console.error("Unexpected error:", error);
+  process.exit(1);
+});

benchmarks/evals/semantic_search_quality/run_eval.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+set -e
+
+CONTEXT_FILE="$1"
+if [ -z "$CONTEXT_FILE" ]; then
+  echo "Usage: $0 <context_file>"
+  exit 1
+fi
+
+echo "=========================================="
+echo "Running Semantic Search Quality Eval"
+echo "=========================================="
+echo ""
+
+# Step 1: Check if sem_search was used
+echo "Step 1: Checking for sem_search tool usage..."
+if cat "$CONTEXT_FILE" | jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search")] | any' > /dev/null 2>&1; then
+  echo "✓ sem_search tool found in context"
+else
+  echo "✗ FAILED: sem_search tool not used"
+  exit 1
+fi
+
+# Step 2: Extract intent and queries
+echo ""
+echo "Step 2: Extracting task intent and queries..."
+INTENT=$(cat "$CONTEXT_FILE" | jq -r '.task.intent // "implementation"')
+echo "  Intent: $INTENT"
+
+# Extract expected file types and avoid list from task
+EXPECTED_TYPES=$(cat "$CONTEXT_FILE" | jq -r '.task.expected_file_types // [] | join(",")')
+SHOULD_AVOID=$(cat "$CONTEXT_FILE" | jq -r '.task.should_avoid // [] | join(",")')
+
+# Extract queries from tool calls
+QUERIES=$(cat "$CONTEXT_FILE" | jq -r '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search") | .function.arguments] | .[0]' 2>/dev/null || echo '{}')
+echo "  Queries extracted: $(echo "$QUERIES" | jq -r '.queries | length // 0') query pairs"
+
+# Step 3: Check authentication
+echo ""
+echo "Step 3: Checking Google Cloud authentication..."
+if gcloud auth application-default print-access-token > /dev/null 2>&1; then
+  echo "✓ Google Cloud authentication available"
+  RUN_LLM_JUDGE=true
+elif [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ] && [ -f "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
+  echo "✓ Service account credentials found"
+  RUN_LLM_JUDGE=true
+else
+  echo "⚠ Google Cloud authentication not configured"
+  echo "  Skipping LLM judge evaluation"
+  echo "  To enable: run 'gcloud auth application-default login'"
+  RUN_LLM_JUDGE=false
+fi
+
+# Step 4: Run LLM judge (conditional)
+echo ""
+if [ "$RUN_LLM_JUDGE" = true ]; then
+  echo "Step 4: Running LLM judge evaluation..."
+  
+  # Build command with parameters (use empty strings if not set)
+  if [ -z "$EXPECTED_TYPES" ]; then
+    EXPECTED_TYPES=""
+  fi
+  
+  if [ -z "$SHOULD_AVOID" ]; then
+    SHOULD_AVOID=""
+  fi
+  
+  CMD="npx tsx llm_judge.ts --context \"$CONTEXT_FILE\" --intent \"$INTENT\" --expected-file-types \"$EXPECTED_TYPES\" --should-avoid \"$SHOULD_AVOID\""
+  
+  eval $CMD
+  
+  if [ $? -eq 0 ]; then
+    echo ""
+    echo "=========================================="
+    echo "✓ EVALUATION PASSED"
+    echo "=========================================="
+    exit 0
+  else
+    echo ""
+    echo "=========================================="
+    echo "✗ EVALUATION FAILED"
+    echo "=========================================="
+    exit 1
+  fi
+else
+  echo "Step 4: Skipped (authentication required)"
+  echo ""
+  echo "=========================================="
+  echo "✓ VALIDATION PASSED (LLM judge skipped)"
+  echo "=========================================="
+  echo ""
+  echo "Note: Full evaluation requires Google Cloud auth"
+  exit 0
+fi

benchmarks/evals/semantic_search_quality/test_context.json

@@ -0,0 +1,22 @@
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": "Find the retry mechanism with exponential backoff implementation"
+    },
+    {
+      "role": "assistant",
+      "content": "I'll search for the retry mechanism implementation.",
+      "tool_calls": [
+        {
+          "id": "call_1",
+          "type": "function",
+          "function": {
+            "name": "sem_search",
+            "arguments": "{\"queries\":[{\"query\":\"retry mechanism with exponential backoff for API calls handling network failures\",\"use_case\":\"I need to find the implementation code for retry logic with backoff, not documentation or test files\"}]}"
+          }
+        }
+      ]
+    }
+  ]
+}

benchmarks/evals/semantic_search_quality/test_queries.ts

@@ -0,0 +1,232 @@
+#!/usr/bin/env tsx
+
+/**
+ * Manual Query Tester for Semantic Search
+ * 
+ * This script helps developers test and improve their semantic search queries
+ * before committing them to the codebase using Gemini 3 Pro via Vertex AI.
+ */
+
+import { vertex } from "@ai-sdk/google-vertex";
+import { generateObject } from "ai";
+import { z } from "zod";
+import * as readline from "readline";
+
+const QueryEvaluationSchema = z.object({
+  embedding_query: z.object({
+    score: z.number().min(0).max(10),
+    strengths: z.array(z.string()),
+    weaknesses: z.array(z.string()),
+    suggestions: z.array(z.string()),
+  }),
+  reranking_query: z.object({
+    score: z.number().min(0).max(10),
+    strengths: z.array(z.string()),
+    weaknesses: z.array(z.string()),
+    suggestions: z.array(z.string()),
+  }),
+  differentiation: z.object({
+    are_different: z.boolean(),
+    explanation: z.string(),
+    suggestion: z.string().optional(),
+  }),
+  overall_recommendation: z.string(),
+});
+
+async function evaluateQueries(
+  embeddingQuery: string,
+  rerankingQuery: string,
+): Promise<z.infer<typeof QueryEvaluationSchema>> {
+  const prompt = `You are an expert in semantic code search. Evaluate these two queries:
+
+**Embedding Query (for vector search):**
+${embeddingQuery}
+
+**Reranking Query (for result filtering):**
+${rerankingQuery}
+
+Provide detailed, constructive feedback on:
+1. The quality of each query
+2. Whether they are sufficiently different
+3. How to improve them
+
+Scoring (0-10):
+- 9-10: Excellent
+- 7-8: Good
+- 5-6: Adequate
+- 3-4: Poor
+- 0-2: Failing
+
+Focus on practical, actionable suggestions.`;
+
+  // @ts-ignore - Type instantiation depth issue with complex Zod schemas
+  const result = await generateObject({
+    model: vertex("gemini-3-pro-preview"),
+    schema: QueryEvaluationSchema,
+    prompt,
+    temperature: 0.3,
+  });
+  return result.object as z.infer<typeof QueryEvaluationSchema>;
+}
+
+function formatEvaluation(evaluation: z.infer<typeof QueryEvaluationSchema>): string {
+  const lines: string[] = [];
+
+  lines.push("\n╔═══════════════════════════════════════════════════════════╗");
+  lines.push("║          SEMANTIC SEARCH QUERY EVALUATION                 ║");
+  lines.push("╚═══════════════════════════════════════════════════════════╝\n");
+
+  // Embedding Query
+  lines.push("📊 EMBEDDING QUERY ANALYSIS");
+  lines.push(`   Score: ${evaluation.embedding_query.score}/10 ${getScoreEmoji(evaluation.embedding_query.score)}\n`);
+  
+  if (evaluation.embedding_query.strengths.length > 0) {
+    lines.push("   ✅ Strengths:");
+    for (const strength of evaluation.embedding_query.strengths) {
+      lines.push(`      • ${strength}`);
+    }
+    lines.push("");
+  }
+
+  if (evaluation.embedding_query.weaknesses.length > 0) {
+    lines.push("   ⚠️  Weaknesses:");
+    for (const weakness of evaluation.embedding_query.weaknesses) {
+      lines.push(`      • ${weakness}`);
+    }
+    lines.push("");
+  }
+
+  if (evaluation.embedding_query.suggestions.length > 0) {
+    lines.push("   💡 Suggestions:");
+    for (const suggestion of evaluation.embedding_query.suggestions) {
+      lines.push(`      • ${suggestion}`);
+    }
+    lines.push("");
+  }
+
+  // Reranking Query
+  lines.push("📊 RERANKING QUERY ANALYSIS");
+  lines.push(`   Score: ${evaluation.reranking_query.score}/10 ${getScoreEmoji(evaluation.reranking_query.score)}\n`);
+
+  if (evaluation.reranking_query.strengths.length > 0) {
+    lines.push("   ✅ Strengths:");
+    for (const strength of evaluation.reranking_query.strengths) {
+      lines.push(`      • ${strength}`);
+    }
+    lines.push("");
+  }
+
+  if (evaluation.reranking_query.weaknesses.length > 0) {
+    lines.push("   ⚠️  Weaknesses:");
+    for (const weakness of evaluation.reranking_query.weaknesses) {
+      lines.push(`      • ${weakness}`);
+    }
+    lines.push("");
+  }
+
+  if (evaluation.reranking_query.suggestions.length > 0) {
+    lines.push("   💡 Suggestions:");
+    for (const suggestion of evaluation.reranking_query.suggestions) {
+      lines.push(`      • ${suggestion}`);
+    }
+    lines.push("");
+  }
+
+  // Differentiation
+  lines.push("🔄 QUERY DIFFERENTIATION");
+  lines.push(`   ${evaluation.differentiation.are_different ? "✅" : "❌"} Queries are ${evaluation.differentiation.are_different ? "sufficiently different" : "too similar"}`);
+  lines.push(`   ${evaluation.differentiation.explanation}\n`);
+  
+  if (evaluation.differentiation.suggestion) {
+    lines.push(`   💡 ${evaluation.differentiation.suggestion}\n`);
+  }
+
+  // Overall
+  lines.push("🎯 OVERALL RECOMMENDATION");
+  lines.push(`   ${evaluation.overall_recommendation}\n`);
+
+  return lines.join("\n");
+}
+
+function getScoreEmoji(score: number): string {
+  if (score >= 9) return "🌟";
+  if (score >= 7) return "👍";
+  if (score >= 5) return "👌";
+  if (score >= 3) return "⚠️";
+  return "❌";
+}
+
+async function promptUser(question: string): Promise<string> {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  return new Promise((resolve) => {
+    rl.question(question, (answer) => {
+      rl.close();
+      resolve(answer.trim());
+    });
+  });
+}
+
+async function main() {
+  console.log("\n╔═══════════════════════════════════════════════════════════╗");
+  console.log("║     SEMANTIC SEARCH QUERY TESTER                          ║");
+  console.log("║     Test and improve your queries with AI feedback        ║");
+  console.log("╚═══════════════════════════════════════════════════════════╝\n");
+
+  // Get queries from user
+  const embeddingQuery = await promptUser(
+    "Enter your embedding query (for vector search):\n> ",
+  );
+  
+  if (!embeddingQuery) {
+    console.error("Embedding query is required");
+    process.exit(1);
+  }
+
+  console.log("");
+  const rerankingQuery = await promptUser(
+    "Enter your reranking query (for result filtering):\n> ",
+  );
+
+  if (!rerankingQuery) {
+    console.error("Reranking query is required");
+    process.exit(1);
+  }
+
+  console.log("\n🔄 Evaluating your queries...\n");
+
+  try {
+    const evaluation = await evaluateQueries(embeddingQuery, rerankingQuery);
+    console.log(formatEvaluation(evaluation));
+  } catch (error) {
+    console.error("Error evaluating queries:", error);
+    process.exit(1);
+  }
+}
+
+// Check if running in interactive mode or with args
+const args = process.argv.slice(2);
+if (args.length >= 2) {
+  // Non-interactive mode
+  const embeddingQuery = args[0];
+  const rerankingQuery = args[1];
+  
+  console.log("\n🔄 Evaluating your queries...\n");
+  evaluateQueries(embeddingQuery!, rerankingQuery!)
+    .then((evaluation) => {
+      console.log(formatEvaluation(evaluation));
+    })
+    .catch((error) => {
+      console.error("Error evaluating queries:", error);
+      process.exit(1);
+    });
+} else {
+  // Interactive mode
+  main().catch((error) => {
+    console.error("Unexpected error:", error);
+    process.exit(1);
+  });
+}

benchmarks/evals/suggest/task.yml

@@ -0,0 +1,47 @@
+run:
+  - mkdir -p test_dir
+  - cd test_dir && touch file1.txt file2.py README.md
+  - cd test_dir && forgee --provider {{provider}} --model {{model}} suggest "{{prompt}}"
+
+parallelism: 2
+timeout: 60
+early_exit: true
+
+validations:
+  - name: "Should generate output (non-empty)"
+    type: shell
+    command: "test $(wc -c | awk '{print $1}') -gt 0"
+    
+  - name: "Should NOT contain markdown code blocks"
+    type: shell
+    command: "! grep -qE '^```'"
+    
+  - name: "Should NOT contain JSON formatting tags"
+    type: shell
+    command: "! grep -qE '\\{\\s*\"command\"\\s*:'"
+
+sources:
+  - value:
+      - provider: "anthropic"
+        model: "claude-sonnet-4-5-20250929"
+      - provider: "open_router"
+        model: "z-ai/glm-4.7"
+      - provider: "open_router"
+        model: "minimax/minimax-m2.1"
+
+  - value:
+      - prompt: "list all files"
+      - prompt: "find all python files"
+      - prompt: "show disk usage"
+      - prompt: "git status"
+      - prompt: "count lines in all files"
+      - prompt: "search for TODO in code"
+      - prompt: "check running processes"
+      - prompt: "compress this directory"
+      - prompt: "show last 20 commits"
+      - prompt: "find files larger than 1MB"
+      - prompt: "kill process on port 3000"
+      - prompt: "show system memory usage"
+      - prompt: "download file from url"
+      - prompt: "create git branch"
+      - prompt: "watch log file changes"

benchmarks/model.ts

@@ -0,0 +1,38 @@
+export type Task = {
+  before_run: Array<string>;
+  run: string | Array<string>;
+  parallelism?: number;
+  timeout?: number;
+  early_exit?: boolean;
+  validations?: Array<Validation>;
+  sources: Array<Source>;
+};
+
+export type Validation = 
+  | {
+      name: string;
+      type: "regex";
+      regex: string;
+    }
+  | {
+      name: string;
+      type: "shell";
+      command: string;
+      exit_code?: number;
+    }
+  | {
+      name: string;
+      type: "llm";
+      evaluator: string;
+      criteria: Record<string, any>;
+    };
+
+export type Source = { csv: string } | { cmd: string } | { value: Record<string, string>[] };
+
+
+export enum TaskStatus {
+  Passed = "passed",
+  ValidationFailed = "validation_failed",
+  Timeout = "timeout",
+  Failed = "failed",
+}

benchmarks/parse.ts

@@ -0,0 +1,52 @@
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import path from "path";
+
+export type CliArgs = {
+  evalName: string;
+  evalDir: string;
+  taskFile: string;
+};
+
+/**
+ * Parses command line arguments and resolves paths
+ */
+export async function parseCliArgs(dirname: string): Promise<CliArgs> {
+  const argv = await yargs(hideBin(process.argv))
+    .usage("Usage: $0 <eval-name> [options]")
+    .command("$0 <eval-name>", "Run an evaluation")
+    .positional("eval-name", {
+      describe: "Name of the evaluation to run",
+      type: "string",
+    })
+    .help()
+    .alias("h", "help")
+    .parseAsync();
+
+  const evalName = argv["eval-name"];
+
+  // Ensure evalName is provided
+  if (!evalName) {
+    throw new Error("eval-name is required");
+  }
+
+  // Support both directory path and direct task.yml path
+  let evalDir: string;
+  let taskFile: string;
+
+  if (evalName.endsWith("task.yml") || evalName.endsWith(".yml") || evalName.endsWith(".yaml")) {
+    // Direct path to task file
+    taskFile = path.isAbsolute(evalName) ? evalName : path.join(dirname, evalName);
+    evalDir = path.dirname(taskFile);
+  } else {
+    // Directory path (original behavior)
+    evalDir = path.join(dirname, evalName);
+    taskFile = path.join(evalDir, "task.yml");
+  }
+
+  return {
+    evalName,
+    evalDir,
+    taskFile,
+  };
+}

benchmarks/task-executor.ts

@@ -0,0 +1,184 @@
+import * as fs from "fs";
+import * as path from "path";
+import { spawn } from "child_process";
+import stripAnsi from "strip-ansi";
+import type { Validation, Task } from "./model.js";
+import { runValidations, allValidationsPassed } from "./verification.js";
+import { formatTimestamp } from "./utils.js";
+
+export type TaskExecutionResult = {
+  index: number;
+  command: string;
+  duration: number;
+  output?: string;
+  error?: string;
+  isTimeout: boolean;
+  earlyExit?: boolean;
+};
+
+/**
+ * Executes a single task command and returns the result
+ */
+export async function executeTask(
+  command: string,
+  index: number,
+  logFile: string,
+  cwd: string,
+  task: Task,
+  context?: Record<string, string>,
+  append: boolean = false,
+): Promise<TaskExecutionResult> {
+  const startTime = Date.now();
+
+  // Create log stream for this task (append if this is not the first command)
+  const logStream = fs.createWriteStream(logFile, {
+    flags: append ? "a" : "w",
+  });
+
+  // Write command at the top of the log file
+  logStream.write(`Command: ${command}\n`);
+  logStream.write(`Started: ${formatTimestamp(new Date())}\n`);
+  logStream.write(`${"=".repeat(80)}\n\n`);
+
+  try {
+    // Track timeout state outside the promise
+    let timedOut = false;
+    let exitedEarly = false;
+
+    // Execute command and stream output to log file
+    const output = await new Promise<string>((resolve, reject) => {
+      const child = spawn(command, {
+        shell: true,
+        cwd: cwd,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+
+      let stdout = "";
+      let stderr = "";
+      let timeoutId: NodeJS.Timeout | null = null;
+
+      // Helper function to check validations after each write
+      const checkValidations = async () => {
+        if (exitedEarly || timedOut) return;
+
+        if (
+          task.early_exit &&
+          task.validations &&
+          task.validations.length > 0
+        ) {
+          const currentOutput = stdout + stderr;
+          if (currentOutput) {
+            const results = await runValidations(
+              currentOutput,
+              task.validations,
+              context,
+            );
+            if (allValidationsPassed(results)) {
+              exitedEarly = true;
+              if (timeoutId) clearTimeout(timeoutId);
+              if (logStream.writable) {
+                logStream.write(
+                  `\n${"=".repeat(80)}\nEarly exit: All validations passed\nKilling process...\n`,
+                  () => {
+                    logStream.end();
+                  },
+                );
+              }
+              child.kill("SIGTERM");
+              resolve(currentOutput);
+            }
+          }
+        }
+      };
+
+      // Set up timeout if configured
+      if (task.timeout) {
+        timeoutId = setTimeout(() => {
+          timedOut = true;
+          if (logStream.writable) {
+            logStream.write(`\n${"=".repeat(80)}\n`);
+            logStream.write(`Timeout: ${task.timeout}s exceeded\n`);
+            logStream.write(`Killing process...\n`);
+            logStream.end();
+          }
+          child.kill("SIGKILL");
+          // Resolve with captured output so far
+          resolve(stdout + stderr);
+        }, task.timeout * 1000);
+      }
+
+      // Stream stdout to both log file and capture for validation
+      child.stdout?.on("data", (data) => {
+        const text = data.toString();
+        stdout += text;
+        if (logStream.writable) {
+          logStream.write(stripAnsi(text));
+        }
+        checkValidations();
+      });
+
+      // Stream stderr to both log file and capture for validation
+      child.stderr?.on("data", (data) => {
+        const text = data.toString();
+        stderr += text;
+        if (logStream.writable) {
+          logStream.write(stripAnsi(text));
+        }
+        checkValidations();
+      });
+
+      child.on("close", (code) => {
+        if (timeoutId) clearTimeout(timeoutId);
+
+        // Don't log or resolve if already timed out or exited early
+        if (timedOut || exitedEarly) return;
+
+        logStream.write(`\n${"=".repeat(80)}\n`);
+        logStream.write(`Finished: ${formatTimestamp(new Date())}\n`);
+        logStream.write(`Exit Code: ${code}\n`);
+        logStream.end();
+
+        if (code === 0) {
+          resolve(stdout + stderr);
+        } else {
+          reject(new Error(`Command failed with exit code ${code}`));
+        }
+      });
+
+      child.on("error", (err) => {
+        if (timeoutId) clearTimeout(timeoutId);
+
+        // Don't log if already timed out or exited early
+        if (timedOut || exitedEarly) return;
+
+        logStream.write(`\nError: ${err.message}\n`);
+        logStream.end();
+        reject(err);
+      });
+    });
+
+    const duration = Date.now() - startTime;
+
+    return {
+      index,
+      command,
+      duration,
+      output,
+      isTimeout: timedOut,
+      earlyExit: exitedEarly,
+    };
+  } catch (error) {
+    const duration = Date.now() - startTime;
+    const errorMessage =
+      error instanceof Error ? error.message : "Command failed";
+
+    return {
+      index,
+      command,
+      duration,
+      error: errorMessage,
+      isTimeout: false,
+      earlyExit: false,
+    };
+  }
+}

benchmarks/tsconfig.json

@@ -0,0 +1,47 @@
+{
+  // Visit https://aka.ms/tsconfig to read more about this file
+  "compilerOptions": {
+    // File Layout
+    "rootDir": ".",
+    "outDir": "dist",
+
+    // Environment Settings
+    // See also https://aka.ms/tsconfig/module
+    "module": "nodenext",
+    "target": "esnext",
+    "types": [],
+    // For nodejs:
+    // "lib": ["esnext"],
+    // "types": ["node"],
+    // and npm install -D @types/node
+
+    // Other Outputs
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+
+    // Stricter Typechecking Options
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+
+    // Style Options
+    // "noImplicitReturns": true,
+    // "noImplicitOverride": true,
+    // "noUnusedLocals": true,
+    // "noUnusedParameters": true,
+    // "noFallthroughCasesInSwitch": true,
+    // "noPropertyAccessFromIndexSignature": true,
+
+    // Recommended Options
+    "strict": true,
+    "jsx": "react-jsx",
+    "verbatimModuleSyntax": true,
+    "isolatedModules": true,
+    "noUncheckedSideEffectImports": true,
+    "moduleDetection": "force",
+    "skipLibCheck": true,
+
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+  }
+}