aparnamohankumar commited on
Commit
e76dbee
Β·
verified Β·
1 Parent(s): 763b295

Upload app.py

Browse files
Files changed (1) hide show
  1. src/app.py +444 -0
src/app.py ADDED
@@ -0,0 +1,444 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================
2
+ # DentaScribe β€” AI Dental Voice Assistant
3
+ # Role 6 β€” Demo & Presentation (Aparna)
4
+ # Streamlit front-end that runs the full pipeline end-to-end
5
+ # ============================================================
6
+
7
+ import streamlit as st
8
+ import tempfile
9
+ import os
10
+ import json
11
+ from datetime import datetime
12
+
13
+ # ── Page config ─────────────────────────────────────────────
14
+ st.set_page_config(
15
+ page_title="DentaScribe",
16
+ page_icon="🦷",
17
+ layout="wide",
18
+ )
19
+
20
+ # ── Custom CSS ───────────────────────────────────────────────
21
+ st.markdown("""
22
+ <style>
23
+ .main-header {
24
+ font-size: 2.8rem;
25
+ font-weight: 700;
26
+ color: #1a73e8;
27
+ text-align: center;
28
+ margin-bottom: 0.2rem;
29
+ }
30
+ .sub-header {
31
+ font-size: 1.1rem;
32
+ color: #555;
33
+ text-align: center;
34
+ margin-bottom: 2rem;
35
+ }
36
+ .step-box {
37
+ background: #f0f4ff;
38
+ border-left: 4px solid #1a73e8;
39
+ padding: 0.8rem 1.2rem;
40
+ border-radius: 6px;
41
+ margin-bottom: 0.6rem;
42
+ }
43
+ .entity-tag {
44
+ display: inline-block;
45
+ padding: 2px 10px;
46
+ border-radius: 12px;
47
+ font-size: 0.85rem;
48
+ font-weight: 600;
49
+ margin: 3px;
50
+ }
51
+ .tag-disease { background: #ffe0e0; color: #c0392b; }
52
+ .tag-chemical { background: #e0f0ff; color: #1565c0; }
53
+ .form-card {
54
+ background: #ffffff;
55
+ border: 1px solid #ddd;
56
+ border-radius: 10px;
57
+ padding: 1.5rem;
58
+ box-shadow: 0 2px 6px rgba(0,0,0,0.06);
59
+ }
60
+ .confidence-bar {
61
+ height: 10px;
62
+ border-radius: 5px;
63
+ background: linear-gradient(90deg, #1a73e8, #34a853);
64
+ }
65
+ </style>
66
+ """, unsafe_allow_html=True)
67
+
68
+ # ── Header ───────────────────────────────────────────────────
69
+ st.markdown('<div class="main-header">🦷 DentaScribe</div>', unsafe_allow_html=True)
70
+ st.markdown('<div class="sub-header">AI-powered voice assistant for dental clinics β€” speech β†’ NER β†’ auto-filled patient form</div>', unsafe_allow_html=True)
71
+
72
+ # ── Pipeline loader (cached so models load once) ─────────────
73
+ @st.cache_resource(show_spinner="Loading AI models β€” this takes ~30 s on first run…")
74
+ def load_models():
75
+ """
76
+ Load all three models into memory.
77
+ Role 1 β€” Andy : Whisper ASR
78
+ Role 2 β€” Ali : DistilBERT NER
79
+ Role 4 β€” Iva : DistilBERT Form Classifier
80
+ """
81
+ from pipeline import load_whisper, load_ner, load_form_classifier
82
+ whisper_processor, whisper_model = load_whisper()
83
+ ner_pipe = load_ner()
84
+ form_tokenizer, form_model = load_form_classifier()
85
+ return whisper_processor, whisper_model, ner_pipe, form_tokenizer, form_model
86
+
87
+
88
+ # ── Sidebar ──────────────────────────────────────────────────
89
+ with st.sidebar:
90
+ st.image("https://img.icons8.com/color/96/tooth.png", width=80)
91
+ st.markdown("### DentaScribe")
92
+ st.markdown("**Deep Learning Group Project**")
93
+ st.markdown("---")
94
+ st.markdown("**Pipeline**")
95
+ st.markdown("""
96
+ 1. πŸŽ™οΈ Audio Upload
97
+ 2. πŸ“ Whisper ASR *(Andy)*
98
+ 3. 🏷️ NER *(Ali)*
99
+ 4. πŸ—„οΈ Data Engineering *(Varsha)*
100
+ 5. πŸ“‹ Form Classifier *(Iva)*
101
+ 6. πŸ”— Integration *(Koroush)*
102
+ 7. πŸ–₯️ This Demo *(Aparna)*
103
+ """)
104
+ st.markdown("---")
105
+ st.markdown("**Patient ID**")
106
+ patient_id = st.text_input("Enter patient ID", value="P001", max_chars=20)
107
+ st.markdown("---")
108
+ st.markdown("**Input Mode**")
109
+ input_mode = st.radio(
110
+ "Choose how to provide input",
111
+ ["πŸŽ™οΈ Record Live", "πŸ“ Upload File", "πŸ“‹ Demo Transcript"],
112
+ index=1,
113
+ label_visibility="collapsed",
114
+ )
115
+ demo_mode = input_mode == "πŸ“‹ Demo Transcript"
116
+
117
+ DEMO_SCENARIOS = {
118
+ # ── cleaning_form ────────────────────────────────────────
119
+ "🦠 Gingivitis & Cleaning": (
120
+ "The patient presents with moderate gingivitis and early periodontitis. "
121
+ "We prescribed amoxicillin 500 mg three times daily for one week and "
122
+ "recommended ibuprofen for pain management. A thorough cleaning was performed."
123
+ ),
124
+ # ── root_canal_form ──────────────────────────────────────
125
+ "🦷 Root Canal": (
126
+ "Patient has severe pulpitis in the lower left molar with signs of apical abscess. "
127
+ "Root canal therapy was initiated today. We administered lidocaine for local anaesthesia "
128
+ "and prescribed metronidazole 400 mg twice daily for five days along with paracetamol "
129
+ "for post-procedure pain relief."
130
+ ),
131
+ # ── emergency_form ───────────────────────────────────────
132
+ "🚨 Emergency Visit": (
133
+ "Emergency visit for acute pericoronitis around the lower wisdom tooth. "
134
+ "Significant swelling and trismus observed. Patient was given a chlorhexidine "
135
+ "mouthwash prescription and amoxicillin-clavulanate 875 mg twice daily. "
136
+ "Extraction of the third molar is recommended next week."
137
+ ),
138
+ # ── extraction_form ──────────────────────────────────────
139
+ "πŸ”§ Tooth Extraction": (
140
+ "Extraction of the upper left second molar performed today due to severe dental caries "
141
+ "and irreversible pulpitis. Local anaesthesia with articaine was administered. "
142
+ "Patient prescribed amoxicillin 500 mg three times daily and ibuprofen 400 mg "
143
+ "for swelling. Post-extraction instructions provided."
144
+ ),
145
+ # ── filling_form ─────────────────────────────────────────
146
+ "πŸͺ₯ Cavity Filling": (
147
+ "Composite resin filling placed on the lower right first molar. "
148
+ "Patient presented with moderate dental caries extending to the dentine. "
149
+ "No signs of pulpitis detected. Fluoride varnish applied after the procedure. "
150
+ "Patient advised to avoid hard foods for 24 hours."
151
+ ),
152
+ # ── crown_bridge_form ────────────────────────────────────
153
+ "πŸ‘‘ Crown & Bridge": (
154
+ "The patient requires a porcelain crown on the upper right premolar following "
155
+ "a cracked tooth diagnosis. Impressions were taken today. Temporary crown placed "
156
+ "and patient prescribed ibuprofen 400 mg as needed for sensitivity. "
157
+ "Final crown fitting is scheduled in two weeks."
158
+ ),
159
+ # ── periodontal_form ─────────────────────────────────────
160
+ "🌿 Periodontal Treatment": (
161
+ "Deep periodontal scaling performed on all four quadrants. "
162
+ "Patient has chronic periodontitis with 5 to 7 mm pocket depths. "
163
+ "Doxycycline 100 mg once daily prescribed for two weeks. "
164
+ "Chlorhexidine gel applied subgingivally. Follow-up in six weeks."
165
+ ),
166
+ # ── orthodontic_form ─────────────────────────────────────
167
+ "😁 Orthodontic Consultation": (
168
+ "Initial orthodontic consultation for Class II malocclusion with moderate crowding. "
169
+ "Patient presents with dental fluorosis on the upper incisors. "
170
+ "Treatment plan includes fixed braces over 18 months. "
171
+ "Fluoride gel application recommended before bonding appointment."
172
+ ),
173
+ # ── whitening_form ───────────────────────────────────────
174
+ "✨ Teeth Whitening": (
175
+ "Patient requested professional teeth whitening for mild to moderate tooth discolouration. "
176
+ "In-office bleaching performed using hydrogen peroxide gel. "
177
+ "Mild dentinal hypersensitivity noted post-procedure. "
178
+ "Prescribed fluoride toothpaste and advised to avoid coffee and tea for 48 hours."
179
+ ),
180
+ # ── consultation_form ────────────────────────────────────
181
+ "πŸ“‹ General Consultation": (
182
+ "New patient consultation. Patient reports intermittent toothache and bleeding gums. "
183
+ "Examination reveals mild gingivitis and early signs of bruxism. "
184
+ "Dental radiographs taken. Recommended night guard for bruxism and "
185
+ "chlorhexidine mouthwash twice daily. Full treatment plan to be discussed at next visit."
186
+ ),
187
+ }
188
+
189
+ if "custom_transcript" not in st.session_state:
190
+ st.session_state["custom_transcript"] = list(DEMO_SCENARIOS.values())[0]
191
+
192
+ if demo_mode:
193
+ preset = st.selectbox(
194
+ "πŸ“‹ Load a demo scenario",
195
+ options=["β€” type your own β€”"] + list(DEMO_SCENARIOS.keys()),
196
+ )
197
+ if preset != "β€” type your own β€”":
198
+ st.session_state["custom_transcript"] = DEMO_SCENARIOS[preset]
199
+
200
+ SAMPLE_TRANSCRIPT = st.text_area(
201
+ "Transcript (edit freely)",
202
+ key="custom_transcript",
203
+ height=160,
204
+ )
205
+
206
+ # ── Main content ─────────────────────────────────────────────
207
+ col_left, col_right = st.columns([1, 1], gap="large")
208
+
209
+ with col_left:
210
+ audio_file = None
211
+ recorded_audio = None
212
+
213
+ if input_mode == "πŸŽ™οΈ Record Live":
214
+ st.markdown("### Step 1 β€” Record Live Audio")
215
+ recorded_audio = st.audio_input("πŸŽ™οΈ Press to record the consultation")
216
+ if recorded_audio is not None:
217
+ st.success("βœ… Recording captured β€” press play to review, then run the pipeline.")
218
+ elif input_mode == "πŸ“ Upload File":
219
+ st.markdown("### Step 1 β€” Upload Audio")
220
+ audio_file = st.file_uploader(
221
+ "Upload a WAV, MP3 or M4A recording of the dental consultation",
222
+ type=["wav", "mp3", "m4a"],
223
+ )
224
+ if audio_file is not None:
225
+ st.markdown("**πŸ”Š Preview β€” play to listen along while the model transcribes:**")
226
+ st.audio(audio_file)
227
+ else: # Demo Transcript
228
+ st.markdown("### Step 1 β€” Demo Transcript")
229
+ st.info("πŸ“Œ Demo mode: using built-in sample transcript β€” no audio needed.")
230
+
231
+ run_btn = st.button("β–Ά Run DentaScribe Pipeline", type="primary", use_container_width=True)
232
+
233
+ # ── Pipeline execution ───────────────────────────────────────
234
+ if run_btn:
235
+ # Pick the active audio source based on input mode
236
+ audio_source = recorded_audio if input_mode == "πŸŽ™οΈ Record Live" else audio_file
237
+ if input_mode != "πŸ“‹ Demo Transcript" and audio_source is None:
238
+ if input_mode == "πŸŽ™οΈ Record Live":
239
+ st.error("Please record audio first, or switch to Upload File / Demo Transcript.")
240
+ else:
241
+ st.error("Please upload an audio file or switch input mode.")
242
+ st.stop()
243
+
244
+ # Read bytes once so we can both transcribe and replay them in Results
245
+ audio_bytes = audio_source.getvalue() if audio_source is not None else None
246
+
247
+ # Load models
248
+ try:
249
+ whisper_processor, whisper_model, ner_pipe, form_tokenizer, form_model = load_models()
250
+ except Exception as e:
251
+ st.error(f"Model loading failed: {e}")
252
+ st.stop()
253
+
254
+ from pipeline import (
255
+ transcribe_audio, extract_entities,
256
+ classify_and_fill_form, save_to_filing_system,
257
+ )
258
+
259
+ # ── Step 1: Transcription ────────────────────────────────
260
+ with st.spinner("πŸŽ™οΈ Transcribing audio…"):
261
+ if input_mode == "πŸ“‹ Demo Transcript":
262
+ transcript = SAMPLE_TRANSCRIPT
263
+ st.success("βœ… Demo transcript loaded.")
264
+ else:
265
+ if input_mode == "πŸŽ™οΈ Record Live":
266
+ suffix = ".wav"
267
+ else:
268
+ suffix = "." + audio_source.name.split(".")[-1]
269
+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
270
+ tmp.write(audio_bytes)
271
+ tmp_path = tmp.name
272
+ try:
273
+ transcript = transcribe_audio(tmp_path, whisper_processor, whisper_model)
274
+ st.success("βœ… Transcription complete.")
275
+ finally:
276
+ os.unlink(tmp_path)
277
+
278
+ # ── Step 2: NER ──────────────────────────────────────────
279
+ with st.spinner("🏷️ Running Named Entity Recognition…"):
280
+ entities = extract_entities(transcript, ner_pipe)
281
+ st.success(f"βœ… {len(entities)} entities detected.")
282
+
283
+ # ── Step 3: Form Classification + Autofill ───────────────
284
+ with st.spinner("πŸ“‹ Classifying form & auto-filling fields…"):
285
+ form = classify_and_fill_form(transcript, entities, form_tokenizer, form_model)
286
+ st.success("βœ… Form classified and filled.")
287
+
288
+ # ── Step 4: Filing ───────────────────────────────────────
289
+ with st.spinner("πŸ—„οΈ Saving to filing system…"):
290
+ saved_path = save_to_filing_system(patient_id, form)
291
+ st.success(f"βœ… Saved β†’ `{saved_path}`")
292
+
293
+ # ────────────────────────────────────────────────────────
294
+ # RESULTS
295
+ # ────────────────────────────────────────────────────────
296
+ st.markdown("---")
297
+ st.markdown("## Results")
298
+
299
+ r1, r2, r3 = st.columns(3)
300
+ r1.metric("Form Type", form.get("form_type", "β€”").replace("_", " ").title())
301
+ r2.metric("Confidence", f"{form.get('confidence', 0)*100:.1f}%")
302
+ r3.metric("Entities Found", len(entities))
303
+
304
+ st.markdown("---")
305
+
306
+ tab1, tab2, tab3 = st.tabs(["πŸ“ Transcript & Entities", "πŸ“‹ Auto-filled Form", "πŸ—„οΈ Raw JSON"])
307
+
308
+ # Tab 1 β€” Transcript & NER
309
+ with tab1:
310
+ if audio_bytes is not None:
311
+ st.markdown("#### πŸ”Š Original Audio")
312
+ st.audio(audio_bytes)
313
+ st.caption("Play back the original recording while reading the transcript to spot ASR errors.")
314
+ st.markdown("#### Transcript")
315
+ st.info(transcript)
316
+
317
+ st.markdown("#### Detected Entities")
318
+ if entities:
319
+ html_tags = ""
320
+ for ent in entities:
321
+ label = ent.get("entity_group", ent.get("label", ""))
322
+ word = ent.get("word", "")
323
+ score = ent.get("score", 0)
324
+ if "Disease" in label or "DISEASE" in label:
325
+ css = "tag-disease"
326
+ icon = "🦠"
327
+ elif "Chemical" in label or "CHEM" in label:
328
+ css = "tag-chemical"
329
+ icon = "πŸ’Š"
330
+ else:
331
+ css = "tag-chemical"
332
+ icon = "🏷️"
333
+ html_tags += (
334
+ f'<span class="entity-tag {css}">'
335
+ f'{icon} {word} <small>({label} {score:.0%})</small>'
336
+ f'</span>'
337
+ )
338
+ st.markdown(html_tags, unsafe_allow_html=True)
339
+
340
+ # Entity table
341
+ st.markdown("#### Entity Details")
342
+ import pandas as pd
343
+ df = pd.DataFrame([
344
+ {
345
+ "Word": e.get("word", ""),
346
+ "Label": e.get("entity_group", e.get("label", "")),
347
+ "Confidence": f"{e.get('score', 0):.2%}",
348
+ }
349
+ for e in entities
350
+ ])
351
+ st.dataframe(df, use_container_width=True, hide_index=True)
352
+ else:
353
+ st.warning("No entities detected in transcript.")
354
+
355
+ # Tab 2 β€” Auto-filled form
356
+ with tab2:
357
+ st.markdown("#### Patient Form")
358
+ st.markdown('<div class="form-card">', unsafe_allow_html=True)
359
+
360
+ fc1, fc2 = st.columns(2)
361
+ with fc1:
362
+ st.markdown(f"**Patient ID:** `{patient_id}`")
363
+ st.markdown(f"**Form Type:** {form.get('form_type','β€”').replace('_',' ').title()}")
364
+ st.markdown(f"**Date / Time:** {form.get('timestamp', datetime.now().strftime('%Y-%m-%d %H:%M'))}")
365
+ st.markdown(f"**Status:** {form.get('status','β€”').replace('_',' ').title()}")
366
+ with fc2:
367
+ conf = form.get("confidence", 0)
368
+ st.markdown(f"**Classifier Confidence:** {conf*100:.1f}%")
369
+ st.markdown(
370
+ f'<div class="confidence-bar" style="width:{conf*100:.0f}%;"></div>',
371
+ unsafe_allow_html=True,
372
+ )
373
+
374
+ st.markdown("---")
375
+
376
+ fc3, fc4 = st.columns(2)
377
+ with fc3:
378
+ st.markdown("**Diagnosis / Condition**")
379
+ diag = form.get("diagnosis", [])
380
+ if diag:
381
+ for d in (diag if isinstance(diag, list) else [diag]):
382
+ st.markdown(f"- 🦠 {d}")
383
+ else:
384
+ st.markdown("_Not detected_")
385
+
386
+ with fc4:
387
+ st.markdown("**Medication / Treatment**")
388
+ meds = form.get("medication", [])
389
+ if meds:
390
+ for m in (meds if isinstance(meds, list) else [meds]):
391
+ st.markdown(f"- πŸ’Š {m}")
392
+ else:
393
+ st.markdown("_Not detected_")
394
+
395
+ st.markdown("---")
396
+ st.markdown("**Full Transcript (recorded)**")
397
+ st.text_area("", value=form.get("original_text", transcript), height=120, disabled=True)
398
+
399
+ st.markdown('</div>', unsafe_allow_html=True)
400
+
401
+ # Tab 3 β€” Raw JSON
402
+ with tab3:
403
+ st.markdown("#### Raw Form JSON (saved to filing system)")
404
+ st.json(form)
405
+ st.markdown("#### All Detected Entities (JSON)")
406
+ st.json(entities)
407
+
408
+ else:
409
+ # Landing state β€” show pipeline diagram
410
+ with col_right:
411
+ st.markdown("### How it works")
412
+ steps = [
413
+ ("πŸŽ™οΈ", "Step 1 β€” Speech Recognition", "Whisper (fine-tuned) converts dental audio to text *(Andy)*"),
414
+ ("🏷️", "Step 2 β€” Named Entity Recognition", "DistilBERT NER extracts diseases & medications *(Ali)*"),
415
+ ("πŸ“‹", "Step 3 β€” Form Classification", "DistilBERT classifier picks the right form type *(Iva)*"),
416
+ ("πŸ—„οΈ", "Step 4 β€” Filing System", "JSON form saved by patient ID + date *(Koroush)*"),
417
+ ]
418
+ for icon, title, desc in steps:
419
+ st.markdown(
420
+ f'<div class="step-box">'
421
+ f'<strong>{icon} {title}</strong><br><span style="color:#555">{desc}</span>'
422
+ f'</div>',
423
+ unsafe_allow_html=True,
424
+ )
425
+
426
+ st.markdown("---")
427
+ st.markdown("### Quick Start")
428
+ st.markdown("""
429
+ 1. Enter a **Patient ID** in the sidebar (e.g. `P001`)
430
+ 2. Upload a **WAV/MP3** recording of a dental consultation
431
+ β€” or tick **Demo Mode** to use the built-in sample
432
+ 3. Click **β–Ά Run DentaScribe Pipeline**
433
+ 4. See the transcript, detected entities, and the auto-filled patient form
434
+ """)
435
+
436
+ # ── Footer ───────────────────────────────────────────────────
437
+ st.markdown("---")
438
+ st.markdown(
439
+ "<div style='text-align:center;color:#999;font-size:0.8rem;'>"
440
+ "DentaScribe Β· Deep Learning Group Project Β· "
441
+ "Andy Β· Ali Β· Varsha Β· Iva Β· Koroush Β· Aparna"
442
+ "</div>",
443
+ unsafe_allow_html=True,
444
+ )