allcited-api / tests /test_feedback.py
allcited-deploy
deploy 2026-10-25-1939
a964978
Raw History Blame Contribute Delete
3.94 kB
import json
import subprocess
import sys
from pathlib import Path
from fastapi.testclient import TestClient
from app.feedback import get_feedback_store
from app.main import app
from app.trace import TraceRecord, get_trace_manager
def test_submit_positive_and_negative_feedback(tmp_path: Path):
"""
Gate check for STEP 35:
Submitting feedback works via POST /v1/feedback;
a downvote with note lands in the table and appears in the eval-candidates list.
"""
client = TestClient(app)
fb_store = get_feedback_store()
fb_store.data_dir = tmp_path
fb_store.clear()
trace_mgr = get_trace_manager()
trace_mgr.trace_dir = tmp_path
trace_mgr.clear()
# Pre-populate trace for question 1
qid1 = "q_test_upvote"
trace_mgr.record_trace(
TraceRecord(
question_id=qid1,
query="What is the resignation notice period?",
route="LOOKUP",
sub_queries=["notice period resignation"],
retrieved_count=2,
reranked_count=2,
top_rerank_score=0.88,
total_sources=2,
refused=False,
provider="stub",
model="test-model",
total_tokens=45,
citations=[],
latencies_ms={"total_ms": 120.0},
)
)
# 1. Submit Upvote
resp_up = client.post(
"/v1/feedback",
json={
"question_id": qid1,
"score": 1,
"note": "Accurate and clear citation.",
},
)
assert resp_up.status_code == 200
assert resp_up.json()["status"] == "recorded"
# Pre-populate trace for question 2 (downvote candidate)
qid2 = "q_test_downvote"
trace_mgr.record_trace(
TraceRecord(
question_id=qid2,
query="How many unused leave days may be carried forward?",
route="LOOKUP",
sub_queries=["leave carry forward surplus"],
retrieved_count=3,
reranked_count=3,
top_rerank_score=0.75,
total_sources=2,
refused=False,
provider="stub",
model="test-model",
total_tokens=52,
citations=[],
latencies_ms={"total_ms": 140.0},
)
)
# 2. Submit Downvote with Note
resp_down = client.post(
"/v1/feedback",
json={
"question_id": qid2,
"score": -1,
"note": "Missed the executive waiver exception in Rule 2.1.4.",
},
)
assert resp_down.status_code == 200
assert resp_down.json()["status"] == "recorded"
# 3. Verify Candidates Endpoint
cand_resp = client.get("/v1/feedback/candidates")
assert cand_resp.status_code == 200
candidates = cand_resp.json()["candidates"]
# Only the downvote appears in the candidates list
assert len(candidates) == 1
cand = candidates[0]
assert cand["question"] == "How many unused leave days may be carried forward?"
assert cand["user_note"] == "Missed the executive waiver exception in Rule 2.1.4."
assert cand["type"] == "user_downvoted"
# 4. Verify Export CLI script outputs candidate JSON
export_out = tmp_path / "exported_candidates.json"
repo_root = Path(__file__).resolve().parent.parent.parent
script_path = repo_root / "scripts" / "export_eval_candidates.py"
assert script_path.exists()
cmd = [
sys.executable,
str(script_path),
"--output",
str(export_out),
"--data-dir",
str(tmp_path),
]
res = subprocess.run(cmd, capture_output=True, text=True, cwd=str(repo_root))
assert res.returncode == 0, res.stderr
assert export_out.exists()
with open(export_out, "r", encoding="utf-8") as f:
data = json.load(f)
assert data["total"] == 1
assert data["candidates"][0]["user_note"] == "Missed the executive waiver exception in Rule 2.1.4."