Spaces:
Running on Zero
Running on Zero
File size: 9,899 Bytes
a964978 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 | import json
from pathlib import Path
from typing import List
from fastapi.testclient import TestClient
from app.main import app
from app.pipeline.answer import (
ABSENT_REFUSAL_TEXT,
FRIENDLY_REFUSAL_TEXT,
post_pass_citations,
)
from app.pipeline.chunker import Chunk
from app.pipeline.twohop import TwoHopChunk
def parse_sse_events(text: str) -> List[dict]:
"""Parse raw SSE text into a list of event objects."""
events = []
current_event = "message"
for line in text.splitlines():
line = line.strip()
if not line:
continue
if line.startswith("event:"):
current_event = line[len("event:"):].strip()
elif line.startswith("data:"):
data_str = line[len("data:"):].strip()
try:
data_json = json.loads(data_str)
except json.JSONDecodeError:
data_json = data_str
events.append({"event": current_event, "data": data_json})
return events
def test_post_pass_citations_strips_fake_markers():
"""Verify citation post-pass cleans hallucinated markers and extracts valid ones."""
src1 = TwoHopChunk(
chunk=Chunk(
id="c1",
doc_id="d1",
content_hash="h1",
text="Notice period is 60 calendar days.",
section_path=["3. Separation"],
token_count=10,
page=3,
),
is_two_hop=False,
)
src2 = TwoHopChunk(
chunk=Chunk(
id="c2",
doc_id="d1",
content_hash="h2",
text="Accrued leave is encashed at base salary daily rate.",
section_path=["5. Statutory Rules"],
token_count=12,
page=4,
),
is_two_hop=True,
added_reason="Defined in rule:63.7.4",
)
sources = [src1, src2]
# Raw answer has valid [S1], [S2] and fake hallucinated [S99], [S5]
raw_answer = "Resignation notice is 60 days [S1]. Encashed leave is capped [S2]. Outside claim [S99] [S5]."
cleaned, citations = post_pass_citations(raw_answer, sources)
# Fake markers must be stripped
assert "[S99]" not in cleaned
assert "[S5]" not in cleaned
assert "[S1]" in cleaned
assert "[S2]" in cleaned
# Valid citations extracted
assert len(citations) == 2
assert citations[0].label == "S1"
assert citations[0].chunk_id == "c1"
assert not citations[0].is_two_hop
assert citations[1].label == "S2"
assert citations[1].chunk_id == "c2"
assert citations[1].is_two_hop
assert citations[1].added_reason == "Defined in rule:63.7.4"
def test_post_pass_refusal_returns_zero_citations():
"""Verify that absent or friendly refusal responses yield zero citations."""
sources = [
TwoHopChunk(
chunk=Chunk(id="c1", doc_id="d1", content_hash="h1", text="Text", section_path=[], token_count=5),
is_two_hop=False,
)
]
cleaned1, citations1 = post_pass_citations(ABSENT_REFUSAL_TEXT, sources)
assert cleaned1 == ABSENT_REFUSAL_TEXT
assert len(citations1) == 0
cleaned2, citations2 = post_pass_citations(FRIENDLY_REFUSAL_TEXT, sources)
assert cleaned2 == FRIENDLY_REFUSAL_TEXT
assert len(citations2) == 0
def test_sse_endpoint_three_live_gate_checks():
"""
Done when gate: 3 live checks in browser/curl:
1. Normal cited answer (answers with [S#] and citations payload).
2. Off-topic request returns the friendly refusal card text (0 LLM calls).
3. Absent-question answers 'Not covered in this document.'
"""
client = TestClient(app)
# Ingest fixture
fixture_path = Path(__file__).resolve().parent.parent.parent / "eval" / "fixtures" / "handbook.pdf"
assert fixture_path.exists(), f"Fixture missing at {fixture_path}"
with open(fixture_path, "rb") as f:
ingest_resp = client.post("/v1/ingest", files={"file": ("handbook.pdf", f, "application/pdf")})
assert ingest_resp.status_code == 200, f"Ingest failed: {ingest_resp.text}"
session_id = ingest_resp.json()["session_id"]
# -------------------------------------------------------------
# LIVE CHECK 1: Normal cited answer
# -------------------------------------------------------------
resp1 = client.post(
"/v1/query",
json={"session_id": session_id, "query": "What is the notice period for voluntary resignation?"},
)
assert resp1.status_code == 200
assert "text/event-stream" in resp1.headers["content-type"]
events1 = parse_sse_events(resp1.text)
event_types1 = [e["event"] for e in events1]
# Verify event protocol sequence
assert "route" in event_types1
assert "cache" in event_types1
assert "rewrite" in event_types1
assert "retrieve" in event_types1
assert "rerank" in event_types1
assert "expand" in event_types1
assert "token" in event_types1
assert "citations" in event_types1
assert "done" in event_types1
done_event1 = next(e["data"] for e in events1 if e["event"] == "done")
assert not done_event1["refused"]
assert "60" in done_event1["answer"]
assert "[S" in done_event1["answer"]
citations_event1 = next(e["data"] for e in events1 if e["event"] == "citations")
assert len(citations_event1["citations"]) >= 1
assert citations_event1["citations"][0]["label"] == "S1"
# -------------------------------------------------------------
# LIVE CHECK 2: Off-topic request returns the refusal card text
# -------------------------------------------------------------
resp2 = client.post(
"/v1/query",
json={"session_id": session_id, "query": "Write me a Python script to scrape a website"},
)
assert resp2.status_code == 200
events2 = parse_sse_events(resp2.text)
done_event2 = next(e["data"] for e in events2 if e["event"] == "done")
assert done_event2["refused"]
assert done_event2["reason"] == "low_relevance"
assert done_event2["answer"] == FRIENDLY_REFUSAL_TEXT
# -------------------------------------------------------------
# LIVE CHECK 3: Absent question answers 'Not covered in this document.'
# -------------------------------------------------------------
resp3 = client.post(
"/v1/query",
json={"session_id": session_id, "query": "What is the corporate pet policy for bringing dogs to the office?"},
)
assert resp3.status_code == 200
events3 = parse_sse_events(resp3.text)
done_event3 = next(e["data"] for e in events3 if e["event"] == "done")
assert done_event3["refused"]
assert done_event3["answer"] in (ABSENT_REFUSAL_TEXT, FRIENDLY_REFUSAL_TEXT)
citations_event3 = next(e["data"] for e in events3 if e["event"] == "citations")
assert len(citations_event3["citations"]) == 0
def test_byok_switches_provider_midsession_without_leaks(monkeypatch):
"""
Gate check for STEP 24:
BYOK key switches provider mid-session; no key ever appears in any network log or response.
"""
client = TestClient(app)
fixture_path = Path(__file__).resolve().parent.parent.parent / "eval" / "fixtures" / "handbook.pdf"
with open(fixture_path, "rb") as f:
ingest_resp = client.post("/v1/ingest", files={"file": ("handbook.pdf", f, "application/pdf")})
assert ingest_resp.status_code == 200
session_id = ingest_resp.json()["session_id"]
captured_providers = []
from app.llm.client import BrainClient
async def mock_stream_generate(self, messages, provider=None, temperature=0.1, max_tokens=700):
if provider:
captured_providers.append(provider)
tokens = ["Notice", " period", " is", " 60", " days", " [S1]."]
for t in tokens:
yield t
monkeypatch.setattr(BrainClient, "stream_generate", mock_stream_generate)
# 1. Query with stub provider
resp1 = client.post(
"/v1/query",
json={"session_id": session_id, "query": "What is the notice period for voluntary resignation?"},
headers={"X-Provider": "stub"},
)
assert resp1.status_code == 200
events1 = parse_sse_events(resp1.text)
done1 = next(e["data"] for e in events1 if e["event"] == "done")
assert done1["model"] == "stub-llm"
assert captured_providers[-1].provider_type == "stub"
# 2. Mid-session switch to BYOK provider via headers
secret_key = "sk-browser-user-secret-abc123xyz"
resp2 = client.post(
"/v1/query",
json={"session_id": session_id, "query": "What is the notice period for voluntary resignation?"},
headers={
"X-BYOK-Key": secret_key,
"X-BYOK-Model": "custom-gpt-4o-byok",
"X-BYOK-Base-URL": "https://api.openai.com/v1",
},
)
assert resp2.status_code == 200
events2 = parse_sse_events(resp2.text)
done2 = next(e["data"] for e in events2 if e["event"] == "done")
assert done2["model"] == "custom-gpt-4o-byok"
assert captured_providers[-1].provider_type == "byok"
assert captured_providers[-1].api_key == secret_key
assert captured_providers[-1].is_byok is True
# CRITICAL: Verify secret key is never echoed or present in response body
assert secret_key not in resp2.text
# 3. Mid-session switch to local Ollama provider via headers
resp3 = client.post(
"/v1/query",
json={"session_id": session_id, "query": "What is the notice period for voluntary resignation?"},
headers={
"X-Provider": "ollama",
"X-BYOK-Model": "llama3.2",
},
)
assert resp3.status_code == 200
events3 = parse_sse_events(resp3.text)
done3 = next(e["data"] for e in events3 if e["event"] == "done")
assert done3["model"] == "llama3.2"
assert captured_providers[-1].provider_type == "ollama"
assert captured_providers[-1].model == "llama3.2"
|