| { |
| "framework": "langchain-core", |
| "execution": "BayesianRetriever(BaseRetriever) -> BayesianRAG RunnableLambda -> grounded answer/abstention", |
| "clean": { |
| "n_cases": 8, |
| "passed": 8, |
| "accuracy": 1.0, |
| "rows": [ |
| { |
| "id": "c1", |
| "ok": true, |
| "answer": "The observatory opens to visitors at 9 AM every Saturday. [C1]", |
| "citations": [ |
| "C1" |
| ], |
| "abstained": false, |
| "top_doc": "C1" |
| }, |
| { |
| "id": "c2", |
| "ok": true, |
| "answer": "The chemistry safety briefing is held in Hall 7. [C2]", |
| "citations": [ |
| "C2" |
| ], |
| "abstained": false, |
| "top_doc": "C2" |
| }, |
| { |
| "id": "c3", |
| "ok": true, |
| "answer": "Library books may be borrowed for 21 days. [C3]", |
| "citations": [ |
| "C3" |
| ], |
| "abstained": false, |
| "top_doc": "C3" |
| }, |
| { |
| "id": "c4", |
| "ok": true, |
| "answer": "The robotics club meets on Thursday evenings. [C4]", |
| "citations": [ |
| "C4" |
| ], |
| "abstained": false, |
| "top_doc": "C4" |
| }, |
| { |
| "id": "c5", |
| "ok": true, |
| "answer": "Visitor parking is located beside Gate 3. [C5]", |
| "citations": [ |
| "C5" |
| ], |
| "abstained": false, |
| "top_doc": "C5" |
| }, |
| { |
| "id": "c6", |
| "ok": true, |
| "answer": "The campus shuttle begins service at 6:30 AM. [C6]", |
| "citations": [ |
| "C6" |
| ], |
| "abstained": false, |
| "top_doc": "C6" |
| }, |
| { |
| "id": "c7", |
| "ok": true, |
| "answer": "The data science seminar carries two academic credits. [C7]", |
| "citations": [ |
| "C7" |
| ], |
| "abstained": false, |
| "top_doc": "C7" |
| }, |
| { |
| "id": "c8", |
| "ok": true, |
| "answer": "The emergency assembly point is the North Lawn. [C8]", |
| "citations": [ |
| "C8" |
| ], |
| "abstained": false, |
| "top_doc": "C8" |
| } |
| ] |
| }, |
| "conflict": { |
| "n_cases": 18, |
| "passed": 18, |
| "accuracy": 1.0, |
| "rows": [ |
| { |
| "id": "q1", |
| "ok": true, |
| "answer": "All employee self-service tasks, including leave requests and payslips, are now handled in PeopleHub. [OFF-HR-PORTAL]", |
| "citations": [ |
| "OFF-HR-PORTAL" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-HR-PORTAL" |
| }, |
| { |
| "id": "q2", |
| "ok": true, |
| "answer": "The 2026 employee benefits policy grants 18 paid leave days per calendar year. [OFF-LEAVE]", |
| "citations": [ |
| "OFF-LEAVE" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-LEAVE" |
| }, |
| { |
| "id": "q3", |
| "ok": true, |
| "answer": "Under the active flexible-work policy, employees may work remotely up to two days per week with manager approval. [OFF-REMOTE]", |
| "citations": [ |
| "OFF-REMOTE" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-REMOTE" |
| }, |
| { |
| "id": "q4", |
| "ok": true, |
| "answer": "Payroll publishes salary slips by the fifth working day of each month. [OFF-PAYSLIP]", |
| "citations": [ |
| "OFF-PAYSLIP" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-PAYSLIP" |
| }, |
| { |
| "id": "q5", |
| "ok": true, |
| "answer": "The security baseline now requires WireGuard for the standard corporate VPN profile. [OFF-VPN]", |
| "citations": [ |
| "OFF-VPN" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-VPN" |
| }, |
| { |
| "id": "q6", |
| "ok": true, |
| "answer": "Production database backups have a retention period of 30 days. [OFF-BACKUP]", |
| "citations": [ |
| "OFF-BACKUP" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-BACKUP" |
| }, |
| { |
| "id": "q7", |
| "ok": true, |
| "answer": "The current gateway quota permits 120 requests per minute per token. [OFF-RATE]", |
| "citations": [ |
| "OFF-RATE" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-RATE" |
| }, |
| { |
| "id": "q8", |
| "ok": true, |
| "answer": "The production primary region is ap-south-1; disaster recovery remains in a separate region. [OFF-REGION]", |
| "citations": [ |
| "OFF-REGION" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-REGION" |
| }, |
| { |
| "id": "q9", |
| "ok": true, |
| "answer": "For domestic business travel, reimbursable meals are capped at INR 4,500 per day. [OFF-EXPENSE]", |
| "citations": [ |
| "OFF-EXPENSE" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-EXPENSE" |
| }, |
| { |
| "id": "q10", |
| "ok": true, |
| "answer": "Expense claims must be submitted within 15 days of the transaction date. [OFF-REIMB]", |
| "citations": [ |
| "OFF-REIMB" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-REIMB" |
| }, |
| { |
| "id": "q11", |
| "ok": true, |
| "answer": "Approved business travel must be booked through TripDesk. [OFF-TRAVEL]", |
| "citations": [ |
| "OFF-TRAVEL" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-TRAVEL" |
| }, |
| { |
| "id": "q12", |
| "ok": true, |
| "answer": "Procurement review is mandatory for purchases above INR 50,000. [OFF-PROC]", |
| "citations": [ |
| "OFF-PROC" |
| ], |
| "abstained": false, |
| "top_doc": "OFF-PROC" |
| }, |
| { |
| "id": "q13", |
| "ok": true, |
| "answer": "I do not know based on the provided documents.", |
| "citations": [], |
| "abstained": true, |
| "top_doc": "LEG-RATE" |
| }, |
| { |
| "id": "q14", |
| "ok": true, |
| "answer": "I do not know based on the provided documents.", |
| "citations": [], |
| "abstained": true, |
| "top_doc": "OFF-HR-PORTAL" |
| }, |
| { |
| "id": "q15", |
| "ok": true, |
| "answer": "I do not know based on the provided documents.", |
| "citations": [], |
| "abstained": true, |
| "top_doc": "OFF-HR-PORTAL" |
| }, |
| { |
| "id": "q16", |
| "ok": true, |
| "answer": "I do not know based on the provided documents.", |
| "citations": [], |
| "abstained": true, |
| "top_doc": "OFF-REIMB" |
| }, |
| { |
| "id": "q17", |
| "ok": true, |
| "answer": "I do not know based on the provided documents.", |
| "citations": [], |
| "abstained": true, |
| "top_doc": "OFF-LAB-A" |
| }, |
| { |
| "id": "q18", |
| "ok": true, |
| "answer": "I do not know based on the provided documents.", |
| "citations": [], |
| "abstained": true, |
| "top_doc": "OFF-OFFICE-A" |
| } |
| ] |
| } |
| } |
