Islam Mamedov commited on
Commit
80a78e0
·
1 Parent(s): 546986c

Day 7: refusal-hardened prompt (7/7), strict citations; faithful 0.89, correct 0.91

Browse files
Files changed (5) hide show
  1. data/eval_cache.json +0 -0
  2. data/rewrite_cache.json +44 -0
  3. src/ask.py +13 -4
  4. src/eval.py +2 -1
  5. src/retrieval.py +81 -27
data/eval_cache.json CHANGED
The diff for this file is too large to render. See raw diff
 
data/rewrite_cache.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "How do I set a custom status code for an endpoint?": "How do I set a custom status code for an endpoint? `FastAPI custom status code endpoint response HTTPStatus`",
3
+ "How do I receive an uploaded file in an endpoint?": "How do I receive an uploaded file in an endpoint? `FileResponse + UploadFile + endpoint FastAPI`",
4
+ "How do I run a task in the background after returning a response?": "How do I run a task in the background after returning a response? `background task FastAPI run after response` \n\nor \n\n`background tasks fastapi response` \n\nor \n\n`asyncio.run_in_executor fastapi` \n\nor \n\n`\"BackgroundTasks\" \"add_task\" fastapi` \n\nor \n\n`fastapi.background`",
5
+ "How do I enable CORS so my frontend can call the API?": "How do I enable CORS so my frontend can call the API? `cors fastapi enable + CORSMiddleware`",
6
+ "How do I declare optional query parameters with default values?": "How do I declare optional query parameters with default values? `optional query parameters default value FastAPI Path Parameters`",
7
+ "How do I declare a path parameter with a type like int?": "How do I declare a path parameter with a type like int? `path parameter type int FastAPI PathParam`",
8
+ "How do I receive a JSON request body using a Pydantic model?": "How do I receive a JSON request body using a Pydantic model? `pydantic model json request body FastAPI` \n\nor \n\n`FastAPI request body JSON Pydantic BaseModel` \n\nor \n\n`from fastapi import FastAPI, Request; from pydantic import BaseModel;`",
9
+ "How do I receive form data instead of JSON?": "How do I receive form data instead of JSON? `receive form data FastAPI` \n\nor \n\n`FastAPI request form data instead of JSON`\n\nor \n\n`python FastAPI form data Body` \n\nor \n\n`FastAPI FileResponse form data` \n\nor \n\n`\"Form data\" in: FastAPI Request`",
10
+ "How do I return a custom error response with a specific status code and message?": "How do I return a custom error response with a specific status code and message? `HTTPException` `Response` `status_code` `FastAPI` `error handling` `exceptions`",
11
+ "How do I use response_model to control which fields are returned?": "How do I use response_model to control which fields are returned? `response_model` `FastAPI` `pydantic` `response_model_exclude` `response_model_include` `main.py`",
12
+ "How do I serve static files like images or CSS?": "How do I serve static files like images or CSS? `serving static files FastAPI StaticFiles`",
13
+ "How do I write tests for my FastAPI endpoints?": "How do I write tests for my FastAPI endpoints? `test FastAPI endpoints pytest testclient`",
14
+ "How do I use WebSockets in FastAPI?": "How do I use WebSockets in FastAPI? `FastAPI WebSockets WebSocket`",
15
+ "How do I set a cookie in a response?": "How do I set a cookie in a response? `set cookie response FastAPI Response Cookies`",
16
+ "How do I add custom headers to a response?": "How do I add custom headers to a response? `add custom headers response FastAPI ResponseHeader ResponseObject`",
17
+ "How do I implement JWT token authentication?": "How do I implement JWT token authentication? `fastapi jwt authentication token`",
18
+ "How does FastAPI's dependency injection system work?": "How does FastAPI's dependency injection system work? `FastAPI dependency injection` OR `FastAPI Depends` OR `fastapi.diplomatic` OR `fastapi.dependencies` OR `APIRouter dependencies`",
19
+ "What does jsonable_encoder do and when do I need it?": "What does jsonable_encoder do and when do I need it? `jsonable_encoder` `FastAPI` `response` `encoder` `json` `pydantic`",
20
+ "What is the difference between async def and def endpoints?": "What is the difference between async def and def endpoints? `async def vs def endpoints FastAPI`",
21
+ "How can a dependency run cleanup code after the response is sent?": "How can a dependency run cleanup code after the response is sent? `FastAPI dependency cleanup after response sent` \n`yield response` \n`try-finally` \n`Request response cycle` \n`fastapi/dependencies.py` \n`APIRouter` \n`use_route_handlers` \n`BackgroundTasks`",
22
+ "How can I customize or extend the generated OpenAPI schema?": "How can I customize or extend the generated OpenAPI schema? `openapi schema customization extend FastAPI get_openapi`",
23
+ "What error does FastAPI return when request validation fails, and how do I customize it?": "What error does FastAPI return when request validation fails, and how do I customize it? `request validation error FastAPI RequestValidationError custom error handling`",
24
+ "How does the OAuth2 password flow work in FastAPI?": "How does the OAuth2 password flow work in FastAPI? `OAuth2 password flow FastAPI` \n\nalternatively: \n`FastAPI OAuth2PasswordBearer` \nor \n`FastAPI security OAuth2 password` \nor \n`\"oauth2 password\" site:fastapi.tiangolo.com`",
25
+ "How do I run code on application startup and shutdown?": "How do I run code on application startup and shutdown? `FastAPI startup shutdown code execution events`",
26
+ "Can I mount another application under a path prefix?": "Can I mount another application under a path prefix? \"FastAPI mount application under path prefix\" OR \"FastAPI subapplication\" OR \"Mount subapp\" OR \"FastAPI include_router prefix\" OR \"FastAPI APP mount\"",
27
+ "How does FastAPI run regular def endpoints without blocking the event loop?": "How does FastAPI run regular def endpoints without blocking the event loop? `run_in_threadpool def endpoint async event loop fastapi`",
28
+ "How do I manage configuration and settings with environment variables?": "How do I manage configuration and settings with environment variables? `FastAPI env variables settings config`",
29
+ "Where is the APIRouter class defined?": "Where is the APIRouter class defined? `APIRouter class definition FastAPI source code`",
30
+ "Where is the OAuth2PasswordBearer class implemented?": "Where is the OAuth2PasswordBearer class implemented? `OAuth2PasswordBearer class FastAPI security`",
31
+ "Where is the jsonable_encoder function implemented?": "Where is the jsonable_encoder function implemented? `jsonable_encoder function FastAPI source code`",
32
+ "Where is the BackgroundTasks class defined?": "Where is the BackgroundTasks class defined? `BackgroundTasks class definition FastAPI` \n\nor \n\n`BackgroundTasks FastAPI python source code` \n\nor \n\n`BackgroundTasks definition in fastapi/background_tasks.py`",
33
+ "Where is FastAPI's HTTPException defined?": "Where is FastAPI's HTTPException defined? `FastAPI HTTPException definition`",
34
+ "Where is the main FastAPI application class defined?": "Where is the main FastAPI application class defined? `FastAPI main application class definition site:github.com/tiangolo/fastapi` \n\nor \n\n`FastAPI app class definition main.py` \n\nor \n\n`FastAPI __init__.py FastAPI class`",
35
+ "Where are the Query, Path, and Body parameter functions defined?": "Where are the Query, Path, and Body parameter functions defined? `python fastapi Path Body parameter functions definition`",
36
+ "Where is the TestClient that FastAPI provides for testing?": "Where is the TestClient that FastAPI provides for testing? `TestClient FastAPI testing`",
37
+ "How do I connect FastAPI to MongoDB?": "How do I connect FastAPI to MongoDB? `FastAPI + MongoDB integration` \n`python motor async` \n`FastAPI MongoDB connection` \n`pydantic MongoDB` \n`main.py uvicorn mongo` \n`FastAPI Depends MongoClient` \n`path operation MongoDB` \n`python fastapi[all] motor`",
38
+ "Does FastAPI have built-in rate limiting?": "Does FastAPI have built-in rate limiting? `FastAPI rate limiting` `Limit` `api` `dependencies` `fastapi/security`",
39
+ "How do I integrate Celery task queues with FastAPI?": "How do I integrate Celery task queues with FastAPI? `Celery task queue integration FastAPI` \n\n Alternately: \n `asyncio Celery BackgroundTasks FastAPI` \n or \n `Celery broker FastAPI uvicorn` \n or \n `tasks queue async FastAPI` \n or \n `Celery FastAPI main.py`",
40
+ "How do I use FastAPI with the Django ORM?": "How do I use FastAPI with the Django ORM? `FastAPI + Django ORM integration` \n`fastapi` `django` `orm` \n`main.py` `app` `models` \n`sqlalchemy` `database`",
41
+ "Does FastAPI include a built-in admin dashboard?": "Does FastAPI include a built-in admin dashboard? `FastAPI admin dashboard`",
42
+ "How do I schedule recurring cron jobs inside FastAPI?": "How do I schedule recurring cron jobs inside FastAPI? `fastapi cron job scheduling recurring apirouter BackgroundTasks`",
43
+ "Does FastAPI have built-in database migration support?": "Does FastAPI have built-in database migration support? `FastAPI + database migration + SQLAlchemy`"
44
+ }
src/ask.py CHANGED
@@ -19,15 +19,20 @@ import sys
19
 
20
  from retrieval import retrieve as retrieve_chunks
21
 
 
22
  LLM_MODEL = os.environ.get("GROQ_MODEL", "openai/gpt-oss-120b")
23
  TOP_K = 5
24
 
25
  SYSTEM_PROMPT = """\
26
  You are a precise assistant answering questions about the FastAPI codebase.
27
  Answer ONLY from the numbered context chunks provided. Rules:
28
- - Cite chunks inline like [1] or [2][3] after each claim they support.
29
- - If the context does not contain the answer, say exactly:
30
- "I couldn't find this in the indexed codebase." Do not guess.
 
 
 
 
31
  - Prefer short code examples when the context contains them.
32
  - Be concise."""
33
 
@@ -69,7 +74,11 @@ def main() -> None:
69
  args = parser.parse_args()
70
 
71
  hits = retrieve_chunks(args.question, k=args.k, mode=args.mode)
72
-
 
 
 
 
73
  if args.show_chunks:
74
  for i, h in enumerate(hits, 1):
75
  print(f"\n=== [{i}] {h['meta']['path']} :: "
 
19
 
20
  from retrieval import retrieve as retrieve_chunks
21
 
22
+ REFUSAL_TEXT = "I couldn't find this in the indexed codebase."
23
  LLM_MODEL = os.environ.get("GROQ_MODEL", "openai/gpt-oss-120b")
24
  TOP_K = 5
25
 
26
  SYSTEM_PROMPT = """\
27
  You are a precise assistant answering questions about the FastAPI codebase.
28
  Answer ONLY from the numbered context chunks provided. Rules:
29
+ - Cite chunks inline using square brackets like [1] or [2][3]. Never use
30
+ any other citation format.
31
+ - The context chunks are search results: they may be loosely RELATED to
32
+ the question without actually answering it. If they do not contain the
33
+ answer, reply with exactly:
34
+ "I couldn't find this in the indexed codebase."
35
+ Do not answer from general knowledge. Do not guess.
36
  - Prefer short code examples when the context contains them.
37
  - Be concise."""
38
 
 
74
  args = parser.parse_args()
75
 
76
  hits = retrieve_chunks(args.question, k=args.k, mode=args.mode)
77
+ top_score = hits[0].get("score")
78
+ if top_score is not None and top_score < SCORE_THRESHOLD:
79
+ print(f"\n{REFUSAL_TEXT}")
80
+ return
81
+
82
  if args.show_chunks:
83
  for i, h in enumerate(hits, 1):
84
  print(f"\n=== [{i}] {h['meta']['path']} :: "
src/eval.py CHANGED
@@ -71,7 +71,8 @@ def is_gold_hit(hit: dict, gold: list[str]) -> bool:
71
  def main() -> None:
72
  parser = argparse.ArgumentParser()
73
  parser.add_argument("--mode", default="dense",
74
- choices=["dense", "hybrid", "hybrid_rerank", "dense_rerank"])
 
75
  parser.add_argument("--answers", action="store_true",
76
  help="also generate answers and run the LLM judge")
77
  parser.add_argument("--k", type=int, default=K)
 
71
  def main() -> None:
72
  parser = argparse.ArgumentParser()
73
  parser.add_argument("--mode", default="dense",
74
+ choices=["dense", "dense_rw", "dense_rerank",
75
+ "hybrid", "hybrid_rerank"])
76
  parser.add_argument("--answers", action="store_true",
77
  help="also generate answers and run the LLM judge")
78
  parser.add_argument("--k", type=int, default=K)
src/retrieval.py CHANGED
@@ -1,21 +1,22 @@
1
- """Retrieval backends: dense, hybrid (dense + BM25), and hybrid + reranker.
2
 
3
  Modes:
4
  dense - embedding similarity only (the v0 baseline)
5
- hybrid - dense + BM25 keyword search, fused with Reciprocal Rank
6
- Fusion (RRF)
7
- hybrid_rerank - hybrid to get ~20 candidates, then a cross-encoder
8
- re-scores each (question, chunk) pair and keeps the best
9
-
10
- Why BM25 helps this corpus: code questions contain exact identifiers
11
- ("APIRouter", "jsonable_encoder"). Embeddings blur those into meaning;
12
- BM25 matches them literally. The two are complementary, and RRF merges
13
- their rankings without needing to calibrate scores against each other.
14
-
15
- Why the reranker helps: the embedding compares question and chunk as two
16
- separate vectors. A cross-encoder reads them TOGETHER, token by token, so
17
- it is much better at judging true relevance - but too slow to run on all
18
- 1300 chunks, which is why it only re-scores the top candidates.
 
19
  """
20
 
21
  import json
@@ -30,8 +31,19 @@ DATA_DIR = Path("data")
30
  EMBED_MODEL = "BAAI/bge-small-en-v1.5"
31
  RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
32
  QUERY_PREFIX = "Represent this sentence for searching relevant passages: "
33
- CANDIDATES = 20 # how many candidates hybrid gathers before final cut
34
  RRF_K = 60 # standard RRF constant
 
 
 
 
 
 
 
 
 
 
 
35
 
36
  # Lazy singletons so models/indexes load once per process, not per query
37
  _embedder = None
@@ -63,14 +75,48 @@ def _load() -> None:
63
  def _get_reranker() -> CrossEncoder:
64
  global _reranker
65
  if _reranker is None:
66
- _reranker = CrossEncoder(RERANK_MODEL) # first run downloads ~1GB
67
  return _reranker
68
 
69
 
70
- def _dense_ids(question: str, n: int) -> list[str]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  emb = _embedder.encode(QUERY_PREFIX + question, normalize_embeddings=True)
72
  res = _collection.query(query_embeddings=[emb.tolist()], n_results=n)
73
- return res["ids"][0]
 
 
 
 
74
 
75
 
76
  def _bm25_ids(question: str, n: int) -> list[str]:
@@ -88,6 +134,13 @@ def _rrf_fuse(rankings: list[list[str]], weights: tuple) -> list[str]:
88
  return sorted(scores, key=scores.get, reverse=True)
89
 
90
 
 
 
 
 
 
 
 
91
  def _to_hit(cid: str) -> dict:
92
  c = _chunk_by_id[cid]
93
  return {
@@ -101,22 +154,23 @@ def _to_hit(cid: str) -> dict:
101
  },
102
  }
103
 
104
- def _rerank(question: str, candidates: list[str]) -> list[str]:
105
- pairs = [(question, _chunk_by_id[cid]["text"][:2000])
106
- for cid in candidates]
107
- scores = _get_reranker().predict(pairs)
108
- return [cid for _, cid in sorted(zip(scores, candidates), reverse=True)]
109
 
110
  def retrieve(question: str, k: int = 5, mode: str = "dense") -> list[dict]:
111
  _load()
112
  if mode == "dense":
113
- ids = _dense_ids(question, k)
 
 
 
 
 
 
114
  elif mode == "dense_rerank":
115
  ids = _rerank(question, _dense_ids(question, CANDIDATES))[:k]
116
  elif mode in ("hybrid", "hybrid_rerank"):
117
  fused = _rrf_fuse([_dense_ids(question, CANDIDATES),
118
- _bm25_ids(question, CANDIDATES)],
119
- weights=(2.0, 1.0))
120
  candidates = fused[:CANDIDATES]
121
  if mode == "hybrid_rerank":
122
  candidates = _rerank(question, candidates)
 
1
+ """Retrieval backends: dense, dense + query rewriting, hybrid, and reranked.
2
 
3
  Modes:
4
  dense - embedding similarity only (the v0 baseline)
5
+ dense_rw - LLM expands the question with likely code identifiers
6
+ and file names BEFORE embedding it (query rewriting)
7
+ dense_rerank - dense candidates re-scored by a cross-encoder
8
+ hybrid - dense + BM25 keyword search, fused with weighted RRF
9
+ hybrid_rerank - hybrid candidates re-scored by a cross-encoder
10
+
11
+ Ablation findings on this corpus (42-question eval set):
12
+ - dense won over hybrid: BM25 pulled in issue chunks that share common
13
+ query words, displacing correct chunks from the top-5.
14
+ - Two reranker generations (bge-reranker-base and v2-m3) both LOWERED
15
+ MRR: cross-encoders demote raw code chunks for "where is X defined"
16
+ questions because code *is* the answer without discussing it.
17
+ - dense_rw targets the remaining misses: questions whose words are all
18
+ too common to retrieve well ("Query, Path, and Body parameter
19
+ functions") get expanded with identifiers an LLM can guess.
20
  """
21
 
22
  import json
 
31
  EMBED_MODEL = "BAAI/bge-small-en-v1.5"
32
  RERANK_MODEL = "BAAI/bge-reranker-v2-m3"
33
  QUERY_PREFIX = "Represent this sentence for searching relevant passages: "
34
+ CANDIDATES = 20 # how many candidates hybrid/rerank gather before final cut
35
  RRF_K = 60 # standard RRF constant
36
+ RRF_WEIGHTS = (2.0, 1.0) # dense vs BM25 in the fused ranking
37
+
38
+ # --- query rewriting ---
39
+ REWRITE_CACHE = DATA_DIR / "rewrite_cache.json"
40
+ REWRITE_PROMPT = """\
41
+ You improve search queries for a search engine over the FastAPI repository
42
+ (source code, docs, GitHub issues). Rewrite the question as a short search
43
+ query, adding likely Python identifiers, class/function names, and file
44
+ names from FastAPI. Output ONLY the query, no explanation.
45
+
46
+ Question: {question}"""
47
 
48
  # Lazy singletons so models/indexes load once per process, not per query
49
  _embedder = None
 
75
  def _get_reranker() -> CrossEncoder:
76
  global _reranker
77
  if _reranker is None:
78
+ _reranker = CrossEncoder(RERANK_MODEL) # first run downloads ~2.3GB
79
  return _reranker
80
 
81
 
82
+ def _rewrite(question: str) -> str:
83
+ """Expand the question with likely identifiers via one cached LLM call.
84
+
85
+ Falls back to the original question on any failure (no key, rate
86
+ limit, network) so retrieval never breaks because of the rewriter.
87
+ """
88
+ import os
89
+ cache = (json.loads(REWRITE_CACHE.read_text())
90
+ if REWRITE_CACHE.exists() else {})
91
+ if question in cache:
92
+ return cache[question]
93
+ try:
94
+ from groq import Groq
95
+ client = Groq(api_key=os.environ["GROQ_API_KEY"])
96
+ model = os.environ.get("GROQ_MODEL", "openai/gpt-oss-120b")
97
+ out = client.chat.completions.create(
98
+ model=model,
99
+ messages=[{"role": "user",
100
+ "content": REWRITE_PROMPT.format(question=question)}],
101
+ temperature=0.0).choices[0].message.content.strip()
102
+ expanded = f"{question} {out}"
103
+ except Exception as e:
104
+ print(f" [rewrite failed ({e.__class__.__name__}); using original]")
105
+ return question
106
+ cache[question] = expanded
107
+ REWRITE_CACHE.write_text(json.dumps(cache, ensure_ascii=False, indent=2))
108
+ return expanded
109
+
110
+
111
+ def _dense_ids(question: str, n: int,
112
+ with_scores: bool = False):
113
  emb = _embedder.encode(QUERY_PREFIX + question, normalize_embeddings=True)
114
  res = _collection.query(query_embeddings=[emb.tolist()], n_results=n)
115
+ ids = res["ids"][0]
116
+ if with_scores:
117
+ # cosine distance -> similarity (1.0 = identical, 0.0 = unrelated)
118
+ return ids, [1.0 - d for d in res["distances"][0]]
119
+ return ids
120
 
121
 
122
  def _bm25_ids(question: str, n: int) -> list[str]:
 
134
  return sorted(scores, key=scores.get, reverse=True)
135
 
136
 
137
+ def _rerank(question: str, candidates: list[str]) -> list[str]:
138
+ pairs = [(question, _chunk_by_id[cid]["text"][:2000])
139
+ for cid in candidates]
140
+ scores = _get_reranker().predict(pairs)
141
+ return [cid for _, cid in sorted(zip(scores, candidates), reverse=True)]
142
+
143
+
144
  def _to_hit(cid: str) -> dict:
145
  c = _chunk_by_id[cid]
146
  return {
 
154
  },
155
  }
156
 
 
 
 
 
 
157
 
158
  def retrieve(question: str, k: int = 5, mode: str = "dense") -> list[dict]:
159
  _load()
160
  if mode == "dense":
161
+ ids, scores = _dense_ids(question, k, with_scores=True)
162
+ hits = [_to_hit(cid) for cid in ids]
163
+ for h, s in zip(hits, scores):
164
+ h["score"] = s
165
+ return hits
166
+ elif mode == "dense_rw":
167
+ ids = _dense_ids(_rewrite(question), k)
168
  elif mode == "dense_rerank":
169
  ids = _rerank(question, _dense_ids(question, CANDIDATES))[:k]
170
  elif mode in ("hybrid", "hybrid_rerank"):
171
  fused = _rrf_fuse([_dense_ids(question, CANDIDATES),
172
+ _bm25_ids(question, CANDIDATES)],
173
+ weights=RRF_WEIGHTS)
174
  candidates = fused[:CANDIDATES]
175
  if mode == "hybrid_rerank":
176
  candidates = _rerank(question, candidates)