Add CI retrieval evaluation gate and lock dependencies

.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: Tests and Retrieval Eval
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-and-evaluate:
+    name: Pytest and Recall Gate
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    env:
+      TOKENIZERS_PARALLELISM: "false"
+      PYTHONUNBUFFERED: "1"
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: requirements-lock.txt
+
+      - name: Install locked dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-lock.txt
+
+      - name: Run unit tests
+        run: |
+          pytest tests/ -v
+
+      - name: Build a fresh vector index
+        run: |
+          rm -rf data/chroma
+          python src/index.py
+
+      - name: Run retrieval evaluation gate
+        run: |
+          python src/eval.py \
+            --mode dense \
+            --k 5 \
+            --min-recall 0.90

app.py

@@ -94,7 +94,7 @@ with st.sidebar:
     c4.metric("refusal", "7/7")
     st.divider()
     st.markdown("[💻 Source & write-up]"
-                "(https://github.com/Pace200413/codebase-rag)")
+                "(https://github.com/islam-mamedov/codebase-rag)")
     if st.button("🗑 Clear conversation", use_container_width=True):
         st.session_state.messages = []
         st.rerun()

requirements-lock.txt

@@ -0,0 +1,124 @@
+GitPython==3.1.50
+Jinja2==3.1.6
+MarkupSafe==3.0.3
+PyGithub==2.9.1
+PyJWT==2.13.0
+PyNaCl==1.6.2
+PyPika==0.51.1
+PyYAML==6.0.3
+Pygments==2.20.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.1
+aiosignal==1.4.0
+altair==6.2.2
+annotated-doc==0.0.4
+annotated-types==0.7.0
+anyio==4.14.1
+attrs==26.1.0
+bcrypt==5.0.0
+blinker==1.9.0
+build==1.5.1
+cachetools==7.1.4
+certifi==2026.6.17
+cffi==2.1.0
+charset-normalizer==3.4.9
+chromadb==1.5.9
+click==8.4.2
+cryptography==49.0.0
+distro==1.9.0
+durationpy==0.10
+filelock==3.29.7
+flatbuffers==25.12.19
+frozenlist==1.8.0
+fsspec==2026.6.0
+gitdb==4.0.12
+googleapis-common-protos==1.75.0
+groq==1.5.0
+grpcio==1.82.1
+h11==0.16.0
+hf-xet==1.5.1
+httpcore==1.0.9
+httptools==0.8.0
+httpx==0.28.1
+huggingface_hub==1.23.0
+idna==3.18
+importlib_resources==7.1.0
+iniconfig==2.3.0
+itsdangerous==2.2.0
+joblib==1.5.3
+jsonschema-specifications==2025.9.1
+jsonschema==4.26.0
+kubernetes==36.0.2
+markdown-it-py==4.2.0
+mdurl==0.1.2
+mmh3==5.2.1
+mpmath==1.3.0
+multidict==6.7.1
+narwhals==2.23.0
+networkx==3.6.1
+numpy==2.4.6
+oauthlib==3.3.1
+onnxruntime==1.27.0
+opentelemetry-api==1.43.0
+opentelemetry-exporter-otlp-proto-common==1.43.0
+opentelemetry-exporter-otlp-proto-grpc==1.43.0
+opentelemetry-proto==1.43.0
+opentelemetry-sdk==1.43.0
+opentelemetry-semantic-conventions==0.64b0
+orjson==3.11.9
+overrides==7.7.0
+packaging==26.2
+pandas==3.0.3
+pillow==12.3.0
+pluggy==1.6.0
+propcache==0.5.2
+protobuf==7.35.1
+pyarrow==25.0.0
+pybase64==1.4.3
+pycparser==3.0
+pydantic-settings==2.14.2
+pydantic==2.13.4
+pydantic_core==2.46.4
+pydeck==0.9.3
+pyproject_hooks==1.2.0
+pytest==9.1.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.2
+python-multipart==0.0.32
+rank-bm25==0.2.2
+referencing==0.37.0
+regex==2026.7.10
+requests-oauthlib==2.0.0
+requests==2.34.2
+rich==15.0.0
+rpds-py==2026.6.3
+safetensors==0.8.0
+scikit-learn==1.9.0
+scipy==1.17.1
+sentence-transformers==5.6.0
+shellingham==1.5.4
+six==1.17.0
+smmap==5.0.3
+sniffio==1.3.1
+starlette==1.3.1
+streamlit==1.59.1
+sympy==1.14.0
+tenacity==9.1.4
+threadpoolctl==3.6.0
+tokenizers==0.22.2
+toml==0.10.2
+torch==2.13.0
+tqdm==4.68.4
+transformers==5.13.0
+tree-sitter-python==0.25.0
+tree-sitter==0.26.0
+typer==0.26.8
+typing-inspection==0.4.2
+typing_extensions==4.16.0
+urllib3==2.7.0
+uvicorn==0.51.0
+uvloop==0.22.1
+watchfiles==1.2.0
+websocket-client==1.9.0
+websockets==16.1
+yarl==1.24.2

requirements.txt

@@ -3,3 +3,8 @@ chromadb
 sentence-transformers
 groq
 rank-bm25
+python-dotenv
+PyGithub
+tree-sitter
+tree-sitter-python
+pytest

src/eval.py

@@ -78,6 +78,12 @@ def main() -> None:
     parser.add_argument("--answers", action="store_true",
                         help="also generate answers and run the LLM judge")
     parser.add_argument("--k", type=int, default=K)
+    parser.add_argument(
+        "--min-recall",
+        type=float,
+        default=None,
+        help="Exit with code 1 when recall@k is below this threshold.",
+    )
     args = parser.parse_args()
 
     items = [json.loads(line)
@@ -102,14 +108,35 @@ def main() -> None:
         if not rank:
             print(f"  [miss] {item['question']}")
 
-    print(f"\n=== Retrieval (mode={args.mode}, k={args.k}) ===")
-    print(f"recall@{args.k}: {sum(recalls)/len(recalls):.2f}  "
-          f"({int(sum(recalls))}/{len(recalls)})")
-    print(f"MRR:       {sum(mrrs)/len(mrrs):.2f}")
-
-    if not args.answers:
-        print("\n(retrieval-only run; add --answers for generation metrics)")
-        return
+        if not recalls:
+            raise RuntimeError("The evaluation set contains no answerable questions.")
+
+        recall_at_k = sum(recalls) / len(recalls)
+        mrr = sum(mrrs) / len(mrrs)
+
+        print(f"\n=== Retrieval (mode={args.mode}, k={args.k}) ===")
+        print(
+            f"recall@{args.k}: {recall_at_k:.2f}  "
+            f"({int(sum(recalls))}/{len(recalls)})"
+        )
+        print(f"MRR:       {mrr:.2f}")
+
+        if args.min_recall is not None:
+            if recall_at_k < args.min_recall:
+                print(
+                    f"\n[gate] FAIL: recall@{args.k}={recall_at_k:.4f} "
+                    f"is below the required {args.min_recall:.4f}"
+                )
+                raise SystemExit(1)
+
+            print(
+                f"\n[gate] PASS: recall@{args.k}={recall_at_k:.4f} "
+                f"meets the required {args.min_recall:.4f}"
+            )
+
+        if not args.answers:
+            print("\n(retrieval-only run; add --answers for generation metrics)")
+            return
 
     # -------- generation + judge metrics --------
     import os