Maria-1B-Instruct / eval_50_benchmark_dataset.py
Zahid2005's picture
Deploy eval_50_benchmark_dataset.py: Maria NHS Hybrid Architecture component
0327478 verified
Raw History Blame Contribute Delete
17.7 kB
"""
50-Case Comprehensive Benchmark Suite for Maria Hybrid Architecture
Evaluates:
- Guideline Retrieval Accuracy (Recall@1 & Recall@2)
- 2WW Cancer / Emergency Red-Flag Trigger Sensitivity
- BNF 1st-Line Prescribing Adherence
- End-to-End Point-of-Care Latency (ms)
- Maria-1B Neural Synthesis Generation Coherence
"""
CLINICAL_50_CASES = [
# ── 1-10: CARDIOVASCULAR & HYPERTENSION (NICE NG136 / CG95 / NG196) ──────
{
"id": 1,
"query": "A 48-year-old Caucasian male has confirmed clinic BP 162/102 and ABPM 154/96. No diabetes. What is the diagnosis and first-line drug?",
"expected_guideline": "NICE NG136",
"expected_prescribing": ["ramipril", "acei", "arb", "losartan", "candesartan"],
"is_red_flag": False
},
{
"id": 2,
"query": "A 62-year-old Afro-Caribbean female has clinic BP 168/104. What is the recommended Step 1 antihypertensive?",
"expected_guideline": "NICE NG136",
"expected_prescribing": ["amlodipine", "ccb", "calcium channel blocker"],
"is_red_flag": False
},
{
"id": 3,
"query": "A 54-year-old male with severe headache and clinic BP 210/125 has bilateral papilloedema on fundoscopy. What is the management?",
"expected_guideline": "NICE NG136",
"expected_prescribing": ["emergency", "hospital", "admission", "same-day"],
"is_red_flag": True
},
{
"id": 4,
"query": "A 72-year-old male presents with irregular pulse. ECG confirms Atrial Fibrillation. CHA2DS2-VASc score is 3. What anticoagulation?",
"expected_guideline": "NICE NG196",
"expected_prescribing": ["doac", "apixaban", "edoxaban", "rivaroxaban"],
"is_red_flag": False
},
{
"id": 5,
"query": "A 58-year-old female presents with central crushing chest pain radiating to left jaw, diaphoresis, and nausea for 25 minutes. Triage plan?",
"expected_guideline": "NICE CG95",
"expected_prescribing": ["999", "aspirin 300mg", "emergency", "ambulance", "gtn"],
"is_red_flag": True
},
{
"id": 6,
"query": "A 65-year-old male on Ramipril 10mg and Amlodipine 10mg still has clinic BP 155/98. What is the Step 3 medication under NG136?",
"expected_guideline": "NICE NG136",
"expected_prescribing": ["indapamide", "thiazide"],
"is_red_flag": False
},
{
"id": 7,
"query": "A 61-year-old female experiences retrosternal chest tightness brought on by walking uphill, resolving within 3 minutes of rest. Next referral?",
"expected_guideline": "NICE CG95",
"expected_prescribing": ["racpc", "rapid access chest pain", "ctca", "ct coronary"],
"is_red_flag": False
},
{
"id": 8,
"query": "A 70-year-old patient with AF has resting heart rate of 118 bpm. What is first-line rate control in primary care?",
"expected_guideline": "NICE NG196",
"expected_prescribing": ["bisoprolol", "beta-blocker", "diltiazem", "verapamil"],
"is_red_flag": False
},
{
"id": 9,
"query": "A 50-year-old diabetic patient has clinic BP 148/92 mmHg confirmed on ABPM. What is the target BP under NICE NG136?",
"expected_guideline": "NICE NG136",
"expected_prescribing": ["<140/90", "135/85"],
"is_red_flag": False
},
{
"id": 10,
"query": "A 67-year-old male with resistant hypertension on triple therapy has potassium 4.1 mmol/L. What Step 4 drug should be added?",
"expected_guideline": "NICE NG136",
"expected_prescribing": ["spironolactone"],
"is_red_flag": False
},
# ── 11-20: ENDOCRINOLOGY & DIABETES (NICE NG28) ──────────────────────────
{
"id": 11,
"query": "A 45-year-old asymptomatic male has routine HbA1c 52 mmol/mol. What is required to confirm Type 2 Diabetes?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["repeat", "second test", "2-4 weeks"],
"is_red_flag": False
},
{
"id": 12,
"query": "A newly diagnosed T2DM patient has established cardiovascular disease (prior MI). What dual first-line therapy does NICE NG28 advise?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["metformin", "sglt2", "dapagliflozin", "empagliflozin"],
"is_red_flag": False
},
{
"id": 13,
"query": "A 52-year-old with T2DM on Metformin monotherapy has HbA1c 62 mmol/mol. No CVD or CKD. What is the target HbA1c and intensification?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["53 mmol/mol", "dpp4", "sitagliptin", "sglt2", "sulfonylurea"],
"is_red_flag": False
},
{
"id": 14,
"query": "What routine baseline blood test and frequency is recommended for monitoring diabetic kidney disease under NICE NG28?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["acr", "albumin to creatinine", "egfr", "annual"],
"is_red_flag": False
},
{
"id": 15,
"query": "A patient with T2DM on Gliclazide experiences morning shakiness, sweating, and confusion with finger-prick blood glucose 3.2 mmol/L. Management?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["hypoglycaemia", "fast-acting carbohydrate", "glucose", "dose reduction"],
"is_red_flag": True
},
{
"id": 16,
"query": "A 39-year-old female has routine HbA1c 44 mmol/mol. How should this be classified and what primary care action?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["prediabetes", "impaired fasting glucose", "lifestyle", "prevention programme"],
"is_red_flag": False
},
{
"id": 17,
"query": "A diabetic patient on Empagliflozin reports nausea, abdominal pain, and deep rapid breathing. Normal glucose 11 mmol/L. What is the concern?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["euglycaemic dka", "diabetic ketoacidosis", "ketones", "emergency"],
"is_red_flag": True
},
{
"id": 18,
"query": "When can a GLP-1 receptor agonist be considered in adults with Type 2 Diabetes according to NICE NG28?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["bmi >=35", "triple therapy failed", "insulin"],
"is_red_flag": False
},
{
"id": 19,
"query": "A 60-year-old T2DM patient has eGFR drop to 28 mL/min/1.73m2. How should Metformin be adjusted?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["stop", "discontinue", "contraindicated <30"],
"is_red_flag": False
},
{
"id": 20,
"query": "What is the recommended HbA1c target for a T2DM patient managed with lifestyle alone or single drug with no hypo risk?",
"expected_guideline": "NICE NG28",
"expected_prescribing": ["48 mmol/mol", "6.5%"],
"is_red_flag": False
},
# ── 21-30: ONCOLOGY 2-WEEK WAIT RED FLAGS (NICE NG12) ────────────────────
{
"id": 21,
"query": "A 57-year-old female presents with difficulty swallowing solids (dysphagia) for 3 weeks and 4kg weight loss. How to triage?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "2-week wait", "urgent endoscopy", "oesophageal"],
"is_red_flag": True
},
{
"id": 22,
"query": "A 63-year-old male with a 30 pack-year smoking history notices painless gross visible haematuria for 2 days. Referral pathway?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "urology", "bladder", "cystoscopy"],
"is_red_flag": True
},
{
"id": 23,
"query": "A 66-year-old male reports 3 months of looser stools, rectal bleeding, and 5kg unintentional weight loss. What referral under NG12?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "colorectal", "colonoscopy", "lower gi"],
"is_red_flag": True
},
{
"id": 24,
"query": "A 52-year-old woman discovers a new, painless, firm, irregular lump in her right upper outer breast. Referral pathway?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "breast clinic", "urgent referral", "mammogram"],
"is_red_flag": True
},
{
"id": 25,
"query": "A 60-year-old smoker has a persistent cough for 4 weeks with two episodes of coughing up blood (haemoptysis). Management?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "chest x-ray", "urgent lung cancer", "respiratory"],
"is_red_flag": True
},
{
"id": 26,
"query": "A 58-year-old female presents with postmenopausal vaginal bleeding 3 years after her periods stopped. What urgent referral?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "gynaecology", "endometrial", "ultrasound"],
"is_red_flag": True
},
{
"id": 27,
"query": "A 64-year-old male presents with painless jaundice (yellow sclera and dark urine) and upper abdominal ache. What referral?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "pancreatic", "upper gi", "ct abdomen"],
"is_red_flag": True
},
{
"id": 28,
"query": "A 45-year-old male notices an existing back mole has grown, developed asymmetric borders, and multiple colours. Triage pathway?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "dermatology", "melanoma", "excision"],
"is_red_flag": True
},
{
"id": 29,
"query": "A 68-year-old male has routine bloods showing unexplained iron deficiency anaemia (Hb 88, ferritin 8). What NG12 referral?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "colorectal", "gastroscopy", "colonoscopy"],
"is_red_flag": True
},
{
"id": 30,
"query": "A 61-year-old male has an elevated age-adjusted PSA of 7.8 ng/mL with a firm nodular prostate on DRE. Referral pathway?",
"expected_guideline": "NICE NG12",
"expected_prescribing": ["2ww", "urology", "prostate", "multiparametric mri"],
"is_red_flag": True
},
# ── 31-40: RESPIRATORY & ASTHMA (NICE NG80) ──────────────────────────────
{
"id": 31,
"query": "A 24-year-old presents with intermittent wheeze, chest tightness, and dry cough worse at night. What objective diagnostic tests?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["fevo", "feno", "spirometry", "bronchodilator reversibility"],
"is_red_flag": False
},
{
"id": 32,
"query": "A confirmed adult asthmatic has daytime symptoms twice a week and wakes at night once a month. What is initial maintenance?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["low-dose ics", "saba", "inhaled corticosteroid"],
"is_red_flag": False
},
{
"id": 33,
"query": "An asthmatic patient on low-dose ICS remains uncontrolled with frequent rescue inhaler use. What is the Step 2 add-on?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["laba", "long-acting beta2 agonist", "formoterol", "salmeterol"],
"is_red_flag": False
},
{
"id": 34,
"query": "A 30-year-old asthmatic presents to clinic speaking in short phrases, respiratory rate 28, heart rate 115, PEFR 45% of best. Assessment?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["severe asthma", "nebulised salbutamol", "prednisolone", "oxygen", "hospital"],
"is_red_flag": True
},
{
"id": 35,
"query": "What dose and course duration of oral Prednisolone is recommended for an acute adult asthma exacerbation in primary care?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["40-50mg", "5 days"],
"is_red_flag": False
},
{
"id": 36,
"query": "When using MART (Maintenance and Reliever Therapy) in asthma, which inhaler combination is used for both maintenance and reliever?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["ics", "formoterol", "fostair", "symbicort"],
"is_red_flag": False
},
{
"id": 37,
"query": "An adult asthmatic on low-dose ICS + LABA is still symptomatic. What are the Step 3 options under NICE NG80?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["moderate-dose ics", "ltra", "montelukast"],
"is_red_flag": False
},
{
"id": 38,
"query": "A severe asthmatic arrives with silent chest, cyanosis, exhaustion, and PEFR <33% of predicted. What emergency action?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["life-threatening", "999", "ambulance", "oxygen", "hydrocortisone"],
"is_red_flag": True
},
{
"id": 39,
"query": "What FeNO cut-off value in parts per billion supports an asthma diagnosis in symptomatic adults under NICE guidelines?",
"expected_guideline": "NICE NG80",
"expected_prescribing": [">=40 ppb", "40"],
"is_red_flag": False
},
{
"id": 40,
"query": "What post-bronchodilator spirometry improvement is considered positive for asthma bronchodilator reversibility?",
"expected_guideline": "NICE NG80",
"expected_prescribing": ["12%", "200ml", "fev1"],
"is_red_flag": False
},
# ── 41-50: NEPHROLOGY, GASTROENTEROLOGY & INFECTIONS (NG203/CG184/NG109) ─
{
"id": 41,
"query": "A 28-year-old non-pregnant female presents with sudden onset dysuria, frequency, and cloudy urine for 24 hours. No fever. What antibiotic?",
"expected_guideline": "NICE NG109",
"expected_prescribing": ["nitrofurantoin", "trimethoprim", "3 days", "100mg mr bd"],
"is_red_flag": False
},
{
"id": 42,
"query": "A 26-year-old female with acute UTI has eGFR 35 mL/min/1.73m2. Why is Nitrofurantoin avoided and what alternative?",
"expected_guideline": "NICE NG109",
"expected_prescribing": ["ineffective in renal impairment", "trimethoprim", "cefalexin", "amoxicillin"],
"is_red_flag": False
},
{
"id": 43,
"query": "A 32-year-old female presents with acute dysuria, rigors, temperature 38.9Β°C, and severe right flank tenderness. What is diagnosis?",
"expected_guideline": "NICE NG109",
"expected_prescribing": ["acute pyelonephritis", "upper uti", "hospital admission", "cefalexin", "ciprofloxacin"],
"is_red_flag": True
},
{
"id": 44,
"query": "A 64-year-old male has confirmed eGFR 42 mL/min/1.73m2 and urine ACR 22 mg/mmol on repeat tests. What CKD stage?",
"expected_guideline": "NICE NG203",
"expected_prescribing": ["g3a a2", "ckd stage 3a", "microalbuminuria"],
"is_red_flag": False
},
{
"id": 45,
"query": "A non-diabetic CKD patient has urine ACR >=30 mg/mmol. What first-line medication is indicated for renal protection under NG203?",
"expected_guideline": "NICE NG203",
"expected_prescribing": ["acei", "arb", "ramipril", "losartan"],
"is_red_flag": False
},
{
"id": 46,
"query": "When should an adult patient with Chronic Kidney Disease be referred to secondary care nephrology according to NICE NG203?",
"expected_guideline": "NICE NG203",
"expected_prescribing": ["egfr <30", "acr >=70", "rapid decline >25%"],
"is_red_flag": False
},
{
"id": 47,
"query": "A 44-year-old male has heartburn and acid regurgitation for 6 weeks without alarm symptoms. What is Step 1 management under NICE CG184?",
"expected_guideline": "NICE CG184",
"expected_prescribing": ["ppi", "omeprazole 20mg", "lansoprazole 30mg", "4-8 weeks"],
"is_red_flag": False
},
{
"id": 48,
"query": "A 50-year-old female with dyspepsia tests positive on H. pylori stool antigen test. What 7-day eradication regimen is first-line?",
"expected_guideline": "NICE CG184",
"expected_prescribing": ["ppi", "amoxicillin 1g", "clarithromycin 500mg", "metronidazole"],
"is_red_flag": False
},
{
"id": 49,
"query": "A 48-year-old male on high-dose Naproxen presents with sudden haematemesis (vomiting blood) and black tarry stools (melaena). Plan?",
"expected_guideline": "NICE CG184",
"expected_prescribing": ["emergency", "hospital admission", "999", "upper gi bleed", "resuscitation"],
"is_red_flag": True
},
{
"id": 50,
"query": "A 31-year-old male with recurrent burning retrosternal pain has complete relief from OTC antacids. When should an endoscopy be considered?",
"expected_guideline": "NICE CG184",
"expected_prescribing": ["refractory symptoms", "alarm features", "2ww", "dysphagia"],
"is_red_flag": False
}
]
print(f"Loaded {len(CLINICAL_50_CASES)} diverse UK Primary Care evaluation cases across 8 major NICE guidelines.")