# GitHub Actions CI Pipeline # Runs tests and evaluation on pull requests name: CI Pipeline on: push: branches: [main, develop] pull_request: branches: [main, develop] env: PYTHON_VERSION: "3.13" NODE_VERSION: "20" jobs: # ============== Lint & Type Check ============== lint: name: Lint & Type Check runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip pip install "ruff>=0.16,<0.17" mypy pip install -r requirements.txt - name: Run Ruff linter run: ruff check src/ --output-format=github - name: Run type check run: mypy src/ --ignore-missing-imports --no-error-summary continue-on-error: true # ============== Unit Tests ============== test: name: Unit Tests runs-on: ubuntu-latest needs: lint steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install pytest pytest-cov pytest-asyncio - name: Run tests run: | pytest tests/ -v --cov=src --cov-report=xml --cov-report=term-missing env: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - name: Upload coverage uses: codecov/codecov-action@v4 with: files: ./coverage.xml fail_ci_if_error: false # ============== RAG Evaluation ============== evaluate: name: RAG Evaluation runs-on: ubuntu-latest needs: test if: github.event_name == 'pull_request' steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - name: Install Promptfoo run: npm install -g promptfoo # Informational quality gate: it depends on an external API key and on # non-deterministic LLM grading, so it must not block the PR. It is # skipped entirely when GOOGLE_API_KEY is not configured. - name: Run Promptfoo evaluation id: eval continue-on-error: true env: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} run: | if [ -z "$GOOGLE_API_KEY" ]; then echo "::warning::GOOGLE_API_KEY secret not set - skipping RAG evaluation." exit 0 fi mkdir -p eval/results cd eval npx promptfoo eval -c promptfooconfig.yaml -o results/ci-eval.json --no-cache - name: Upload evaluation results if: hashFiles('eval/results/ci-eval.json') != '' uses: actions/upload-artifact@v4 with: name: eval-results path: eval/results/ - name: Comment PR with results if: github.event_name == 'pull_request' && hashFiles('eval/results/ci-eval.json') != '' continue-on-error: true uses: actions/github-script@v7 with: script: | const fs = require('fs'); const raw = JSON.parse(fs.readFileSync('eval/results/ci-eval.json')); // promptfoo nests results as { results: { results: [...] } } in newer // versions and { results: [...] } in older ones. const cases = raw.results?.results ?? raw.results ?? []; const passed = cases.filter(r => r.success).length; const total = cases.length; const passRate = total > 0 ? ((passed / total) * 100).toFixed(1) : 0; const body = `## 🧪 RAG Evaluation Results | Metric | Value | |--------|-------| | Tests Passed | ${passed}/${total} | | Pass Rate | ${passRate}% | [View full results](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) `; github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: body }); # ============== Build Check ============== build: name: Build Check runs-on: ubuntu-latest needs: test steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - name: Verify imports run: | python -c "from src.rag import RAGPipeline; print('RAG Pipeline OK')" python -c "from src.api.main import app; print('FastAPI OK')" python -c "from src.llmops import LangfuseTracer; print('LLMOps OK')"