lumen-rag / .github /workflows /eval-report.yml
WickTech's picture
Deploy from push.sh
f8a5189 verified
Raw History Blame Contribute Delete
3.39 kB
name: Eval report
# Measures the retrieval eval set (data/eval_v2) on the pull-request head and on
# its base commit, posts the metric diff as a sticky PR comment, and fails if a
# tracked metric drops by more than 0.01. Retrieval is deterministic offline
# (hashing embedder, no reranker), so a drop is a real change, not noise.
#
# Uses the plain `pull_request` trigger on purpose: code from the PR runs here,
# so it must never get secrets or a write token to the repository contents.
on:
pull_request:
paths:
- "lumen_rag/**"
- "scripts/eval_report.py"
- "data/eval_v2/**"
- "pyproject.toml"
- ".github/workflows/eval-report.yml"
permissions:
contents: read
pull-requests: write
jobs:
eval-report:
name: Retrieval eval report
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
cache: pip
- run: pip install -e ".[dev]"
- name: Measure head
run: python scripts/eval_report.py run --root . --out "$RUNNER_TEMP/head.json"
- name: Measure base
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
# A base that cannot be measured (older than the v2 set) is not an error:
# the report then shows head numbers only and nothing blocks the merge.
git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA"
python scripts/eval_report.py run --root "$RUNNER_TEMP/base" --out "$RUNNER_TEMP/base.json" \
|| echo "::warning::could not measure the base commit; reporting head only"
- name: Compare
id: compare
continue-on-error: true
run: |
python scripts/eval_report.py compare \
--base "$RUNNER_TEMP/base.json" --head "$RUNNER_TEMP/head.json" \
--max-drop 0.01 --out "$RUNNER_TEMP/report.md"
- name: Job summary
if: always()
run: |
if [ -f "$RUNNER_TEMP/report.md" ]; then cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY"; fi
- name: Post or update PR comment
if: always()
# Fork PRs get a read-only token, so commenting can fail; the job summary above still has the report.
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
test -f "$RUNNER_TEMP/report.md"
existing=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" --paginate \
--jq '[.[] | select(.body | contains("<!-- lumen-eval-report -->")) | .id] | first // empty')
if [ -n "$existing" ]; then
gh api --method PATCH "repos/$REPO/issues/comments/$existing" -F body=@"$RUNNER_TEMP/report.md" > /dev/null
else
gh api --method POST "repos/$REPO/issues/$PR_NUMBER/comments" -F body=@"$RUNNER_TEMP/report.md" > /dev/null
fi
- name: Fail on regression
if: steps.compare.outcome == 'failure'
run: |
echo "A tracked retrieval metric dropped by more than 0.01; see the report above."
exit 1