Download eval_50_benchmark_dataset.py from Zahid2005/Maria-1B-Instruct: direct link, hf CLI and curl.
- Browser
- Download file 17.7 kB
-
https://huggingface.co/Zahid2005/Maria-1B-Instruct/resolve/main/eval_50_benchmark_dataset.py
- Command line
-
hf download hf://Zahid2005/Maria-1B-Instruct/eval_50_benchmark_dataset.py
-
curl -L -o eval_50_benchmark_dataset.py https://huggingface.co/Zahid2005/Maria-1B-Instruct/resolve/main/eval_50_benchmark_dataset.py
17.7 kB
| """ | |
| 50-Case Comprehensive Benchmark Suite for Maria Hybrid Architecture | |
| Evaluates: | |
| - Guideline Retrieval Accuracy (Recall@1 & Recall@2) | |
| - 2WW Cancer / Emergency Red-Flag Trigger Sensitivity | |
| - BNF 1st-Line Prescribing Adherence | |
| - End-to-End Point-of-Care Latency (ms) | |
| - Maria-1B Neural Synthesis Generation Coherence | |
| """ | |
| CLINICAL_50_CASES = [ | |
| # ββ 1-10: CARDIOVASCULAR & HYPERTENSION (NICE NG136 / CG95 / NG196) ββββββ | |
| { | |
| "id": 1, | |
| "query": "A 48-year-old Caucasian male has confirmed clinic BP 162/102 and ABPM 154/96. No diabetes. What is the diagnosis and first-line drug?", | |
| "expected_guideline": "NICE NG136", | |
| "expected_prescribing": ["ramipril", "acei", "arb", "losartan", "candesartan"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 2, | |
| "query": "A 62-year-old Afro-Caribbean female has clinic BP 168/104. What is the recommended Step 1 antihypertensive?", | |
| "expected_guideline": "NICE NG136", | |
| "expected_prescribing": ["amlodipine", "ccb", "calcium channel blocker"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 3, | |
| "query": "A 54-year-old male with severe headache and clinic BP 210/125 has bilateral papilloedema on fundoscopy. What is the management?", | |
| "expected_guideline": "NICE NG136", | |
| "expected_prescribing": ["emergency", "hospital", "admission", "same-day"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 4, | |
| "query": "A 72-year-old male presents with irregular pulse. ECG confirms Atrial Fibrillation. CHA2DS2-VASc score is 3. What anticoagulation?", | |
| "expected_guideline": "NICE NG196", | |
| "expected_prescribing": ["doac", "apixaban", "edoxaban", "rivaroxaban"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 5, | |
| "query": "A 58-year-old female presents with central crushing chest pain radiating to left jaw, diaphoresis, and nausea for 25 minutes. Triage plan?", | |
| "expected_guideline": "NICE CG95", | |
| "expected_prescribing": ["999", "aspirin 300mg", "emergency", "ambulance", "gtn"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 6, | |
| "query": "A 65-year-old male on Ramipril 10mg and Amlodipine 10mg still has clinic BP 155/98. What is the Step 3 medication under NG136?", | |
| "expected_guideline": "NICE NG136", | |
| "expected_prescribing": ["indapamide", "thiazide"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 7, | |
| "query": "A 61-year-old female experiences retrosternal chest tightness brought on by walking uphill, resolving within 3 minutes of rest. Next referral?", | |
| "expected_guideline": "NICE CG95", | |
| "expected_prescribing": ["racpc", "rapid access chest pain", "ctca", "ct coronary"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 8, | |
| "query": "A 70-year-old patient with AF has resting heart rate of 118 bpm. What is first-line rate control in primary care?", | |
| "expected_guideline": "NICE NG196", | |
| "expected_prescribing": ["bisoprolol", "beta-blocker", "diltiazem", "verapamil"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 9, | |
| "query": "A 50-year-old diabetic patient has clinic BP 148/92 mmHg confirmed on ABPM. What is the target BP under NICE NG136?", | |
| "expected_guideline": "NICE NG136", | |
| "expected_prescribing": ["<140/90", "135/85"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 10, | |
| "query": "A 67-year-old male with resistant hypertension on triple therapy has potassium 4.1 mmol/L. What Step 4 drug should be added?", | |
| "expected_guideline": "NICE NG136", | |
| "expected_prescribing": ["spironolactone"], | |
| "is_red_flag": False | |
| }, | |
| # ββ 11-20: ENDOCRINOLOGY & DIABETES (NICE NG28) ββββββββββββββββββββββββββ | |
| { | |
| "id": 11, | |
| "query": "A 45-year-old asymptomatic male has routine HbA1c 52 mmol/mol. What is required to confirm Type 2 Diabetes?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["repeat", "second test", "2-4 weeks"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 12, | |
| "query": "A newly diagnosed T2DM patient has established cardiovascular disease (prior MI). What dual first-line therapy does NICE NG28 advise?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["metformin", "sglt2", "dapagliflozin", "empagliflozin"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 13, | |
| "query": "A 52-year-old with T2DM on Metformin monotherapy has HbA1c 62 mmol/mol. No CVD or CKD. What is the target HbA1c and intensification?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["53 mmol/mol", "dpp4", "sitagliptin", "sglt2", "sulfonylurea"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 14, | |
| "query": "What routine baseline blood test and frequency is recommended for monitoring diabetic kidney disease under NICE NG28?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["acr", "albumin to creatinine", "egfr", "annual"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 15, | |
| "query": "A patient with T2DM on Gliclazide experiences morning shakiness, sweating, and confusion with finger-prick blood glucose 3.2 mmol/L. Management?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["hypoglycaemia", "fast-acting carbohydrate", "glucose", "dose reduction"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 16, | |
| "query": "A 39-year-old female has routine HbA1c 44 mmol/mol. How should this be classified and what primary care action?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["prediabetes", "impaired fasting glucose", "lifestyle", "prevention programme"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 17, | |
| "query": "A diabetic patient on Empagliflozin reports nausea, abdominal pain, and deep rapid breathing. Normal glucose 11 mmol/L. What is the concern?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["euglycaemic dka", "diabetic ketoacidosis", "ketones", "emergency"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 18, | |
| "query": "When can a GLP-1 receptor agonist be considered in adults with Type 2 Diabetes according to NICE NG28?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["bmi >=35", "triple therapy failed", "insulin"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 19, | |
| "query": "A 60-year-old T2DM patient has eGFR drop to 28 mL/min/1.73m2. How should Metformin be adjusted?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["stop", "discontinue", "contraindicated <30"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 20, | |
| "query": "What is the recommended HbA1c target for a T2DM patient managed with lifestyle alone or single drug with no hypo risk?", | |
| "expected_guideline": "NICE NG28", | |
| "expected_prescribing": ["48 mmol/mol", "6.5%"], | |
| "is_red_flag": False | |
| }, | |
| # ββ 21-30: ONCOLOGY 2-WEEK WAIT RED FLAGS (NICE NG12) ββββββββββββββββββββ | |
| { | |
| "id": 21, | |
| "query": "A 57-year-old female presents with difficulty swallowing solids (dysphagia) for 3 weeks and 4kg weight loss. How to triage?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "2-week wait", "urgent endoscopy", "oesophageal"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 22, | |
| "query": "A 63-year-old male with a 30 pack-year smoking history notices painless gross visible haematuria for 2 days. Referral pathway?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "urology", "bladder", "cystoscopy"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 23, | |
| "query": "A 66-year-old male reports 3 months of looser stools, rectal bleeding, and 5kg unintentional weight loss. What referral under NG12?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "colorectal", "colonoscopy", "lower gi"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 24, | |
| "query": "A 52-year-old woman discovers a new, painless, firm, irregular lump in her right upper outer breast. Referral pathway?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "breast clinic", "urgent referral", "mammogram"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 25, | |
| "query": "A 60-year-old smoker has a persistent cough for 4 weeks with two episodes of coughing up blood (haemoptysis). Management?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "chest x-ray", "urgent lung cancer", "respiratory"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 26, | |
| "query": "A 58-year-old female presents with postmenopausal vaginal bleeding 3 years after her periods stopped. What urgent referral?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "gynaecology", "endometrial", "ultrasound"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 27, | |
| "query": "A 64-year-old male presents with painless jaundice (yellow sclera and dark urine) and upper abdominal ache. What referral?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "pancreatic", "upper gi", "ct abdomen"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 28, | |
| "query": "A 45-year-old male notices an existing back mole has grown, developed asymmetric borders, and multiple colours. Triage pathway?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "dermatology", "melanoma", "excision"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 29, | |
| "query": "A 68-year-old male has routine bloods showing unexplained iron deficiency anaemia (Hb 88, ferritin 8). What NG12 referral?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "colorectal", "gastroscopy", "colonoscopy"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 30, | |
| "query": "A 61-year-old male has an elevated age-adjusted PSA of 7.8 ng/mL with a firm nodular prostate on DRE. Referral pathway?", | |
| "expected_guideline": "NICE NG12", | |
| "expected_prescribing": ["2ww", "urology", "prostate", "multiparametric mri"], | |
| "is_red_flag": True | |
| }, | |
| # ββ 31-40: RESPIRATORY & ASTHMA (NICE NG80) ββββββββββββββββββββββββββββββ | |
| { | |
| "id": 31, | |
| "query": "A 24-year-old presents with intermittent wheeze, chest tightness, and dry cough worse at night. What objective diagnostic tests?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["fevo", "feno", "spirometry", "bronchodilator reversibility"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 32, | |
| "query": "A confirmed adult asthmatic has daytime symptoms twice a week and wakes at night once a month. What is initial maintenance?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["low-dose ics", "saba", "inhaled corticosteroid"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 33, | |
| "query": "An asthmatic patient on low-dose ICS remains uncontrolled with frequent rescue inhaler use. What is the Step 2 add-on?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["laba", "long-acting beta2 agonist", "formoterol", "salmeterol"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 34, | |
| "query": "A 30-year-old asthmatic presents to clinic speaking in short phrases, respiratory rate 28, heart rate 115, PEFR 45% of best. Assessment?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["severe asthma", "nebulised salbutamol", "prednisolone", "oxygen", "hospital"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 35, | |
| "query": "What dose and course duration of oral Prednisolone is recommended for an acute adult asthma exacerbation in primary care?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["40-50mg", "5 days"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 36, | |
| "query": "When using MART (Maintenance and Reliever Therapy) in asthma, which inhaler combination is used for both maintenance and reliever?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["ics", "formoterol", "fostair", "symbicort"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 37, | |
| "query": "An adult asthmatic on low-dose ICS + LABA is still symptomatic. What are the Step 3 options under NICE NG80?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["moderate-dose ics", "ltra", "montelukast"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 38, | |
| "query": "A severe asthmatic arrives with silent chest, cyanosis, exhaustion, and PEFR <33% of predicted. What emergency action?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["life-threatening", "999", "ambulance", "oxygen", "hydrocortisone"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 39, | |
| "query": "What FeNO cut-off value in parts per billion supports an asthma diagnosis in symptomatic adults under NICE guidelines?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": [">=40 ppb", "40"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 40, | |
| "query": "What post-bronchodilator spirometry improvement is considered positive for asthma bronchodilator reversibility?", | |
| "expected_guideline": "NICE NG80", | |
| "expected_prescribing": ["12%", "200ml", "fev1"], | |
| "is_red_flag": False | |
| }, | |
| # ββ 41-50: NEPHROLOGY, GASTROENTEROLOGY & INFECTIONS (NG203/CG184/NG109) β | |
| { | |
| "id": 41, | |
| "query": "A 28-year-old non-pregnant female presents with sudden onset dysuria, frequency, and cloudy urine for 24 hours. No fever. What antibiotic?", | |
| "expected_guideline": "NICE NG109", | |
| "expected_prescribing": ["nitrofurantoin", "trimethoprim", "3 days", "100mg mr bd"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 42, | |
| "query": "A 26-year-old female with acute UTI has eGFR 35 mL/min/1.73m2. Why is Nitrofurantoin avoided and what alternative?", | |
| "expected_guideline": "NICE NG109", | |
| "expected_prescribing": ["ineffective in renal impairment", "trimethoprim", "cefalexin", "amoxicillin"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 43, | |
| "query": "A 32-year-old female presents with acute dysuria, rigors, temperature 38.9Β°C, and severe right flank tenderness. What is diagnosis?", | |
| "expected_guideline": "NICE NG109", | |
| "expected_prescribing": ["acute pyelonephritis", "upper uti", "hospital admission", "cefalexin", "ciprofloxacin"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 44, | |
| "query": "A 64-year-old male has confirmed eGFR 42 mL/min/1.73m2 and urine ACR 22 mg/mmol on repeat tests. What CKD stage?", | |
| "expected_guideline": "NICE NG203", | |
| "expected_prescribing": ["g3a a2", "ckd stage 3a", "microalbuminuria"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 45, | |
| "query": "A non-diabetic CKD patient has urine ACR >=30 mg/mmol. What first-line medication is indicated for renal protection under NG203?", | |
| "expected_guideline": "NICE NG203", | |
| "expected_prescribing": ["acei", "arb", "ramipril", "losartan"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 46, | |
| "query": "When should an adult patient with Chronic Kidney Disease be referred to secondary care nephrology according to NICE NG203?", | |
| "expected_guideline": "NICE NG203", | |
| "expected_prescribing": ["egfr <30", "acr >=70", "rapid decline >25%"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 47, | |
| "query": "A 44-year-old male has heartburn and acid regurgitation for 6 weeks without alarm symptoms. What is Step 1 management under NICE CG184?", | |
| "expected_guideline": "NICE CG184", | |
| "expected_prescribing": ["ppi", "omeprazole 20mg", "lansoprazole 30mg", "4-8 weeks"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 48, | |
| "query": "A 50-year-old female with dyspepsia tests positive on H. pylori stool antigen test. What 7-day eradication regimen is first-line?", | |
| "expected_guideline": "NICE CG184", | |
| "expected_prescribing": ["ppi", "amoxicillin 1g", "clarithromycin 500mg", "metronidazole"], | |
| "is_red_flag": False | |
| }, | |
| { | |
| "id": 49, | |
| "query": "A 48-year-old male on high-dose Naproxen presents with sudden haematemesis (vomiting blood) and black tarry stools (melaena). Plan?", | |
| "expected_guideline": "NICE CG184", | |
| "expected_prescribing": ["emergency", "hospital admission", "999", "upper gi bleed", "resuscitation"], | |
| "is_red_flag": True | |
| }, | |
| { | |
| "id": 50, | |
| "query": "A 31-year-old male with recurrent burning retrosternal pain has complete relief from OTC antacids. When should an endoscopy be considered?", | |
| "expected_guideline": "NICE CG184", | |
| "expected_prescribing": ["refractory symptoms", "alarm features", "2ww", "dysphagia"], | |
| "is_red_flag": False | |
| } | |
| ] | |
| print(f"Loaded {len(CLINICAL_50_CASES)} diverse UK Primary Care evaluation cases across 8 major NICE guidelines.") | |