change to grok

.env.example

@@ -1 +1 @@
-GEMINI_API_KEY=your_key_here
+GROQ_API_KEY=your_key_here

app_gradio.py

@@ -98,7 +98,6 @@ def answer_question(question: str, history: list[dict], expand_query: bool):
         index=index,
         k=5,
         expand_query=expand_query,
-        generator=generator,
     )
     results = search_resp.chunks
 

evaluation/judge.py

@@ -1,18 +1,18 @@
-"""LLM-as-judge scoring via Gemini: faithfulness and answer relevance."""
+"""LLM-as-judge scoring via Groq: faithfulness and answer relevance."""
 
 import logging
 import os
 import re
 import time
 
-import google.generativeai as genai
+from groq import Groq
 from dotenv import load_dotenv
 
 load_dotenv()
 logger = logging.getLogger(__name__)
 
-_JUDGE_MODEL = "gemini-flash-latest"
-_ENV_KEY = "GEMINI_API_KEY"
+_JUDGE_MODEL = "llama-3.3-70b-versatile"
+_ENV_KEY = "GROQ_API_KEY"
 
 _FAITHFULNESS_PROMPT = """\
 You are a grader assessing whether a RAG system's answer is supported by the context.
@@ -56,31 +56,32 @@ The answer addresses the main question but omits one important detail.\
 
 
 class LLMJudge:
-    """Thin Gemini wrapper for judge scoring with built-in rate limiting."""
+    """Thin Groq wrapper for judge scoring with built-in rate limiting."""
 
     def __init__(self, rate_limit_delay: float = 1.0) -> None:
         api_key = os.getenv(_ENV_KEY)
         if not api_key:
             raise EnvironmentError(
-                f"Gemini API key not found. Set {_ENV_KEY} or add it to .env."
+                f"Groq API key not found. Set {_ENV_KEY} or add it to .env."
             )
-        genai.configure(api_key=api_key)
-        self._model = genai.GenerativeModel(
-            _JUDGE_MODEL,
-            generation_config=genai.GenerationConfig(temperature=0.0, max_output_tokens=128),
-        )
+        self._client = Groq(api_key=api_key)
         self._delay = rate_limit_delay
         self._last_call_at: float = 0.0
 
     def score(self, prompt: str) -> tuple[float, str]:
-        """Call Gemini and return (score 0–1, explanation). Returns (0.0, error) on failure."""
+        """Call Groq and return (score 0–1, explanation). Returns (0.0, error) on failure."""
         wait = self._delay - (time.monotonic() - self._last_call_at)
         if wait > 0:
             time.sleep(wait)
         try:
-            response = self._model.generate_content(prompt)
+            response = self._client.chat.completions.create(
+                model=_JUDGE_MODEL,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0.0,
+                max_tokens=128,
+            )
             self._last_call_at = time.monotonic()
-            return _parse_response(response.text)
+            return _parse_response(response.choices[0].message.content)
         except Exception as exc:
             self._last_call_at = time.monotonic()
             logger.warning("Judge call failed: %s", exc)

generation/generator.py

@@ -3,7 +3,7 @@ import logging
 from dataclasses import dataclass, field
 from typing import Iterator
 
-import google.generativeai as genai
+from groq import Groq
 from dotenv import load_dotenv
 
 from retrieval.index import SearchResult
@@ -12,8 +12,8 @@ load_dotenv()
 
 logger = logging.getLogger(__name__)
 
-_MODEL_NAME = "gemini-flash-latest"
-_ENV_KEY = "GEMINI_API_KEY"
+_MODEL_NAME = "llama-3.3-70b-versatile"
+_ENV_KEY = "GROQ_API_KEY"
 
 _SYSTEM_PROMPT = """\
 You are a precise document Q&A assistant. Follow these rules without exception:
@@ -45,11 +45,11 @@ class Answer:
 
 
 class Generator:
-    """Wraps Gemini 1.5 Flash for grounded, citation-aware answer generation.
+    """Wraps Groq (llama-3.3-70b-versatile) for grounded, citation-aware answer generation.
 
-    The API key is read from the GEMINI_API_KEY environment variable (or a
+    The API key is read from the GROQ_API_KEY environment variable (or a
     .env file in the working directory).  A low temperature keeps answers
-    factual; max_output_tokens caps runaway responses.
+    factual; max_tokens caps runaway responses.
     """
 
     def __init__(
@@ -61,22 +61,14 @@ class Generator:
         api_key = os.getenv(_ENV_KEY)
         if not api_key:
             raise EnvironmentError(
-                f"Gemini API key not found. Set the {_ENV_KEY} environment variable "
+                f"Groq API key not found. Set the {_ENV_KEY} environment variable "
                 "or add it to a .env file."
             )
 
         self._low_threshold = low_confidence_threshold
         self._medium_threshold = medium_confidence_threshold
-
-        genai.configure(api_key=api_key)
-        self._model = genai.GenerativeModel(
-            model_name,
-            system_instruction=_SYSTEM_PROMPT,
-            generation_config=genai.GenerationConfig(
-                temperature=0.1,
-                max_output_tokens=2048,
-            ),
-        )
+        self._model_name = model_name
+        self._client = Groq(api_key=api_key)
         logger.info("Generator initialised with model: %s", model_name)
 
     def _confidence_level(self, max_score: float) -> str:
@@ -92,7 +84,7 @@ class Generator:
         results: list[SearchResult],
         max_score: float = 1.0,
     ) -> Answer:
-        """Build a grounded prompt from retrieved chunks and call Gemini.
+        """Build a grounded prompt from retrieved chunks and call Groq.
 
         Args:
             question:  The user's question string.
@@ -100,11 +92,11 @@ class Generator:
                        Each result's metadata must contain 'text', 'source',
                        and 'page_num' keys (set by IngestionPipeline).
             max_score: Highest retrieval score from the search step, used to
-                       gate the Gemini call and set confidence metadata.
+                       gate the API call and set confidence metadata.
 
         Returns:
             Answer with generated text, source metadata, and confidence fields.
-            When max_score is below the low threshold, Gemini is not called.
+            When max_score is below the low threshold, the LLM is not called.
         """
         if not results:
             return Answer(
@@ -118,7 +110,7 @@ class Generator:
         level = self._confidence_level(max_score)
 
         if level == "low":
-            logger.debug("Skipping Gemini call: max_score=%.4f below low threshold %.2f", max_score, self._low_threshold)
+            logger.debug("Skipping LLM call: max_score=%.4f below low threshold %.2f", max_score, self._low_threshold)
             return Answer(
                 question=question,
                 answer=_LOW_CONFIDENCE_MESSAGE,
@@ -134,9 +126,17 @@ class Generator:
             "ANSWER:"
         )
 
-        logger.debug("Sending prompt to Gemini (%d chars)", len(prompt))
-        response = self._model.generate_content(prompt)
-        answer_text = response.text.strip()
+        logger.debug("Sending prompt to Groq (%d chars)", len(prompt))
+        response = self._client.chat.completions.create(
+            model=self._model_name,
+            messages=[
+                {"role": "system", "content": _SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+            temperature=0.1,
+            max_tokens=2048,
+        )
+        answer_text = response.choices[0].message.content.strip()
         if level == "medium":
             answer_text += _MEDIUM_CONFIDENCE_NOTE
 
@@ -154,19 +154,19 @@ class Generator:
         results: list[SearchResult],
         max_score: float = 1.0,
     ) -> Iterator[str]:
-        """Stream the answer token-by-token using Gemini's streaming API.
+        """Stream the answer token-by-token using Groq's streaming API.
 
         Args:
             question:  The user's question string.
             results:   Ranked SearchResult list (same as generate_answer).
-            max_score: Highest retrieval score; gates the Gemini call and
+            max_score: Highest retrieval score; gates the API call and
                        appends a warning note for medium-confidence answers.
 
         Yields:
-            Text chunks as they arrive from Gemini.
+            Text chunks as they arrive from Groq.
 
         Raises:
-            Exception: Re-raises any Gemini API error so the caller can handle it.
+            Exception: Re-raises any Groq API error so the caller can handle it.
         """
         if not results:
             yield "No relevant context was retrieved to answer this question."
@@ -175,7 +175,7 @@ class Generator:
         level = self._confidence_level(max_score)
 
         if level == "low":
-            logger.debug("Skipping Gemini call: max_score=%.4f below low threshold %.2f", max_score, self._low_threshold)
+            logger.debug("Skipping LLM call: max_score=%.4f below low threshold %.2f", max_score, self._low_threshold)
             yield _LOW_CONFIDENCE_MESSAGE
             return
 
@@ -186,11 +186,20 @@ class Generator:
             "ANSWER:"
         )
 
-        logger.debug("Sending streaming prompt to Gemini (%d chars)", len(prompt))
-        response = self._model.generate_content(prompt, stream=True)
-        for chunk in response:
-            if chunk.text:
-                yield chunk.text
+        logger.debug("Sending streaming prompt to Groq (%d chars)", len(prompt))
+        stream = self._client.chat.completions.create(
+            model=self._model_name,
+            messages=[
+                {"role": "system", "content": _SYSTEM_PROMPT},
+                {"role": "user", "content": prompt},
+            ],
+            temperature=0.1,
+            max_tokens=2048,
+            stream=True,
+        )
+        for chunk in stream:
+            if chunk.choices[0].delta.content:
+                yield chunk.choices[0].delta.content
 
         if level == "medium":
             yield _MEDIUM_CONFIDENCE_NOTE

requirements.txt

@@ -16,8 +16,8 @@ sentence-transformers==3.3.1
 # Vector index
 faiss-cpu==1.9.0
 
-# LLM (Gemini)
-google-generativeai==0.8.3
+# LLM (Groq)
+groq
 
 # Utilities
 numpy==1.26.4

retrieval/searcher.py

@@ -1,21 +1,17 @@
 from __future__ import annotations
 
 import logging
+import os
 import re
-from typing import TYPE_CHECKING
-
-import google.generativeai as genai
-
 from dataclasses import dataclass
 
+from groq import Groq
+
 from ingestion.embedder import Embedder
 from retrieval.index import VectorIndex, SearchResult
 from retrieval.bm25_index import BM25Index
 from retrieval.reranker import Reranker
 
-if TYPE_CHECKING:
-    from generation.generator import Generator
-
 logger = logging.getLogger(__name__)
 
 
@@ -27,6 +23,8 @@ class SearchResponse:
 
 _reranker = Reranker()
 
+_GROQ_EXPANSION_MODEL = "llama-3.3-70b-versatile"
+
 _EXPANSION_PROMPT = (
     "Generate 3 alternative phrasings of this question that might match "
     "different parts of a document. Return only the rephrased questions, "
@@ -45,7 +43,6 @@ def search(
     bm25_index: BM25Index | None = None,
     alpha: float = 0.7,
     expand_query: bool = False,
-    generator: Generator | None = None,
     use_reranking: bool = True,
     expansion_threshold: float = 0.45,
 ) -> SearchResponse:
@@ -101,29 +98,26 @@ def search(
     results = _single_search(query, embedder, index, bm25_index, candidates, alpha)
 
     if expand_query:
-        if generator is None:
-            logger.warning("expand_query=True but no generator provided — skipping expansion.")
+        top_score = results[0].score if results else 0.0
+        if top_score >= expansion_threshold:
+            logger.debug(
+                "Expansion skipped: top_score=%.4f >= threshold=%.4f",
+                top_score, expansion_threshold,
+            )
         else:
-            top_score = results[0].score if results else 0.0
-            if top_score >= expansion_threshold:
-                logger.debug(
-                    "Expansion skipped: top_score=%.4f >= threshold=%.4f",
-                    top_score, expansion_threshold,
-                )
-            else:
-                logger.debug(
-                    "Expansion triggered: top_score=%.4f < threshold=%.4f",
-                    top_score, expansion_threshold,
-                )
-                expansions = _expand_query(query, generator)
-                if expansions:
-                    logger.debug("Query expanded to %d variants: %s", len(expansions) + 1, [query] + expansions)
-                    expanded_results: list[list[SearchResult]] = [results] + [
-                        _single_search(q, embedder, index, bm25_index, candidates, alpha)
-                        for q in expansions
-                    ]
-                    results = _merge_by_best_score(expanded_results, candidates)
-                    expansion_used = True
+            logger.debug(
+                "Expansion triggered: top_score=%.4f < threshold=%.4f",
+                top_score, expansion_threshold,
+            )
+            expansions = _expand_query(query)
+            if expansions:
+                logger.debug("Query expanded to %d variants: %s", len(expansions) + 1, [query] + expansions)
+                expanded_results: list[list[SearchResult]] = [results] + [
+                    _single_search(q, embedder, index, bm25_index, candidates, alpha)
+                    for q in expansions
+                ]
+                results = _merge_by_best_score(expanded_results, candidates)
+                expansion_used = True
 
     # Capture cosine similarity score before reranker overwrites scores with
     # cross-encoder values (range -10..+10), which would break confidence gating.
@@ -141,22 +135,26 @@ def search(
 # Query expansion
 # ---------------------------------------------------------------------------
 
-def _expand_query(query: str, generator: Generator) -> list[str]:
-    """Ask Gemini for 3 alternative phrasings of the query.
+def _expand_query(query: str) -> list[str]:
+    """Ask Groq for 3 alternative phrasings of the query.
 
-    Uses a fresh GenerativeModel without the Q&A system instruction so the
-    model isn't constrained to answer from context.  Falls back to an empty
-    list on any API error so the caller always gets at least the original query.
+    Falls back to an empty list on any API error so the caller always gets
+    at least the original query.
     """
+    api_key = os.getenv("GROQ_API_KEY")
+    if not api_key:
+        logger.warning("GROQ_API_KEY not set — skipping query expansion.")
+        return []
     try:
-        # Reuse the same underlying model name but without system_instruction.
-        expansion_model = genai.GenerativeModel(
-            generator._model.model_name,
-            generation_config=genai.GenerationConfig(temperature=0.7, max_output_tokens=256),
-        )
+        client = Groq(api_key=api_key)
         prompt = _EXPANSION_PROMPT.format(question=query)
-        response = expansion_model.generate_content(prompt)
-        return _parse_expansion_lines(response.text)
+        response = client.chat.completions.create(
+            model=_GROQ_EXPANSION_MODEL,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.7,
+            max_tokens=256,
+        )
+        return _parse_expansion_lines(response.choices[0].message.content)
     except Exception as exc:
         logger.warning("Query expansion failed (%s) — using original query only.", exc)
         return []