// src/webhook.ts — Production HealthBridge Webhook // Pipeline Architecture: // TEXT → K2 Think V2 (deep reasoning + openFDA) → final answer → WhatsApp // IMAGE → Gemma 4 (extract everything) → K2 Think V2 (research + reason) → final answer → WhatsApp // MEMORY → full conversation history so users can ask follow-up questions (like ChatGPT) import { sendMessage } from './services/whatsapp-baileys.js'; import { openrouterService } from './services/openrouter.js'; import { k2ThinkService } from './services/k2-think.js'; import { getOrCreateSession, addToConversationHistory, resetFlow } from './services/session.js'; import { checkRateLimit, sanitize } from './services/security.js'; import { checkEmergency, getEmergencyResponse } from './flows/emergency.js'; import { startTriage, continueTriage } from './flows/triage.js'; import { startProvider, continueProvider, handleSafeAccess } from './flows/provider.js'; import { handleReminderSetup, handleTaken } from './flows/reminder.js'; import { extractImageDetails, processPDF } from './services/vision.js'; import { normalizeDrugName } from './services/rxnorm.js'; import { checkInteractions, getDrugInfo } from './services/openfda.js'; import { WELCOME, RATE_LIMIT_MSG } from './data/messages.js'; import { Language } from './types/index.js'; import sharp from 'sharp'; import axios from 'axios'; // ——— Fetch Full-Quality Image (bypasses WhatsApp compression) ———————————————— async function fetchFullQualityImage(mediaId: string): Promise<{ base64: string, mimeType: string }> { const token = process.env.WHATSAPP_TOKEN; if (!token) throw new Error("Missing WHATSAPP_TOKEN in environment"); const urlRes = await axios.get( `https://graph.facebook.com/v19.0/${mediaId}`, { headers: { Authorization: `Bearer ${token}` } } ); const downloadUrl = urlRes.data.url; const mimeType = urlRes.data.mime_type; const imgRes = await axios.get(downloadUrl, { responseType: "arraybuffer", headers: { Authorization: `Bearer ${token}` }, }); return { base64: Buffer.from(imgRes.data).toString("base64"), mimeType: mimeType || "image/jpeg", }; } export async function handleWhatsApp(request: any, reply: any) { try { const body = request.body; const from: string = body.From || ''; const rawText: string = body.Body || ''; let mediaBuffer: Buffer | null = body.MediaBuffer || null; let mediaType: string = body.MediaType || ''; const mediaId: string = body.MediaId || ''; // Meta Production Webhook — fetch full resolution image using media_id if (mediaId && !mediaBuffer) { console.log(` 📸 Fetching full-quality original image from Meta for media_id: ${mediaId}`); try { const fullQuality = await fetchFullQualityImage(mediaId); mediaBuffer = Buffer.from(fullQuality.base64, 'base64'); mediaType = fullQuality.mimeType; } catch (err: any) { console.error(" ❌ Failed to fetch full-quality image:", err.message); } } if (!from) return reply.code(400).send({ error: 'Missing sender ID' }); const text = sanitize(rawText); // Rate limit if (!checkRateLimit(from)) { await sendMessage(from, RATE_LIMIT_MSG.en || '⏳ Too many messages. Please wait.'); return reply.code(200).send({ status: 'rate_limited' }); } const session = getOrCreateSession(from); // ===== 1. EMERGENCY OVERRIDE — instant, no LLM ===== const emergencyLang = checkEmergency(text); if (emergencyLang) { session.language = emergencyLang; await sendMessage(from, getEmergencyResponse(emergencyLang)); console.log(` 🚨 EMERGENCY [${emergencyLang}]`); return reply.code(200).send({ status: 'emergency' }); } // ===== 2. TAKEN acknowledgment ===== const lowerTrim = text.toLowerCase().trim(); if (['taken', 'tomado', '已服', 'लिया', 'تم', 'đã uống', '복용완료', 'ininom'].includes(lowerTrim)) { const response = handleTaken(from, session); await sendMessage(from, response); return reply.code(200).send({ status: 'taken' }); } // ===== 3. MEDIA — Images, PDFs, Documents ===== // Pipeline: Gemma 4 extracts → K2 thinks → final answer to WhatsApp if (mediaBuffer) { return await handleMedia(from, mediaBuffer, mediaType, text, session, reply); } // Empty text, no media — skip if (!text.trim()) return reply.code(200).send({ status: 'empty' }); // ===== 4. CONTINUE ACTIVE FLOW ===== if (session.currentFlow !== 'idle') { console.log(` 🔄 Flow: ${session.currentFlow} step ${session.triageStep}`); switch (session.currentFlow) { case 'triage': await continueTriage(from, text, session, sendMessage); break; case 'provider': case 'safe_access': await continueProvider(from, text, session, sendMessage); break; case 'reminder_setup': await handleReminderSetup(from, text, session, sendMessage); break; default: resetFlow(session); break; } return reply.code(200).send({ status: 'flow', flow: session.currentFlow }); } // ===== 5. DETECT INTENT ===== const intent = await openrouterService.detectIntent(text, session.language); console.log(` 🧠 Intent: ${intent.intent} [${intent.detectedLanguage}] (${intent.confidence})`); // Update language from detection const validLangs: Language[] = ['en', 'es', 'zh', 'hi', 'ar', 'vi', 'ko', 'tl', 'pt']; if (validLangs.includes(intent.detectedLanguage as Language)) { session.language = intent.detectedLanguage as Language; } // ===== 6. ROUTE BY INTENT — All text goes through K2 with memory ===== switch (intent.intent) { case 'emergency': const eLang = checkEmergency(text) || session.language; await sendMessage(from, getEmergencyResponse(eLang as Language)); break; case 'triage': await startTriage(from, text, session, sendMessage); break; case 'medication': await handleMedicationQuery(from, text, session); break; case 'find_provider': await startProvider(from, text, session, sendMessage); break; case 'safe_access': await handleSafeAccess(from, text, session, sendMessage); break; case 'insurance': // Route insurance through K2 for deep, warm explanation — with full history addToConversationHistory(session, 'user', text); const insuranceResp = await k2ThinkService.explainInsurance(text, session.language, session.conversationHistory); addToConversationHistory(session, 'assistant', insuranceResp); await sendMessage(from, insuranceResp); break; case 'greeting': const welcome = WELCOME[session.language] || WELCOME.en; await sendMessage(from, welcome); addToConversationHistory(session, 'assistant', welcome); break; case 'taken': const takenResp = handleTaken(from, session); await sendMessage(from, takenResp); break; default: // General AI conversation — K2 Think V2 with full conversation memory // User can ask follow-up questions about previous replies (like ChatGPT) addToConversationHistory(session, 'user', text); const aiResp = await k2ThinkService.chat(text, session.language, session.conversationHistory); addToConversationHistory(session, 'assistant', aiResp); await sendMessage(from, aiResp); break; } return reply.code(200).send({ status: 'processed', intent: intent.intent }); } catch (err: any) { console.error('❌ Webhook error:', err.message || err); return reply.code(500).send({ error: 'Internal server error' }); } } // ═══════════════════════════════════════════════════════════════════════════════ // IMAGE PIPELINE — The heart of the dual-model architecture // Step 1: Gemma 4 Vision → extract ALL details from the image // Step 2: K2 Think V2 → research, reason, and explain everything // Step 3: ONLY the final polished answer → WhatsApp // Step 4: Store in conversation memory → user can ask follow-ups // ═══════════════════════════════════════════════════════════════════════════════ async function handleMedia( from: string, buffer: Buffer, mimeType: string, captionText: string, session: any, reply: any ): Promise { // === PDF HANDLING === if (mimeType === 'application/pdf') { const pdf = await processPDF(buffer); if (pdf.text && pdf.confidence > 0.5) { const processingMsg = session.language === 'es' ? '📄 Analizando tu documento PDF...' : '📄 Analyzing your PDF document...'; await sendMessage(from, processingMsg); // Store the PDF content in conversation memory addToConversationHistory(session, 'user', `[User sent a PDF document] ${captionText || ''}`); // Send PDF text directly to K2 for deep analysis const explanation = await k2ThinkService.analyzeImageExtraction( pdf.text, 'pdf_document', session.language, session.conversationHistory ); addToConversationHistory(session, 'assistant', explanation); await sendMessage(from, explanation); return reply.code(200).send({ status: 'pdf_analyzed' }); } } // === IMAGE HANDLING — Full Gemma → K2 Pipeline === try { const processingMsg = session.language === 'es' ? '🔍 Analizando tu imagen con IA... un momento por favor' : '🔍 Analyzing your image with AI... one moment please'; await sendMessage(from, processingMsg); // ——— STEP 1: GEMMA 4 VISION EXTRACTION ——— // Send image to Gemma → extracts every possible detail console.log(' 👁️ Step 1: Sending image to Gemma 4 Vision for extraction...'); const gemmaResult = await withTimeout( extractImageDetails(buffer), 45000, { extraction: '', imageType: 'unknown', confidence: 0 } ); console.log(` 👁️ Gemma extracted: ${gemmaResult.extraction.length} chars, type: ${gemmaResult.imageType}, confidence: ${gemmaResult.confidence}`); // If Gemma couldn't extract anything useful if (!gemmaResult.extraction || gemmaResult.confidence < 0.2) { const retryMsg = session.language === 'es' ? '😕 No pude leer bien esa imagen. Intenta:\n\n1️⃣ Buena iluminación frontal\n2️⃣ Acercarte al texto/etiqueta\n3️⃣ Sin movimiento (imagen nítida)\n4️⃣ Enviar como *Documento* (📎 → Documento)\n\n¡O describe lo que ves y te ayudo!' : '😕 I couldn\'t read that image clearly. Try:\n\n1️⃣ Good front lighting\n2️⃣ Closer to the text/label\n3️⃣ No motion blur (hold steady)\n4️⃣ Send as *Document* (📎 → Document)\n\nOr describe what you see and I\'ll help!'; await sendMessage(from, retryMsg); return reply.code(200).send({ status: 'image_unclear' }); } // Store the image context in conversation memory so user can ask follow-ups const imageContext = `[User sent an image: ${gemmaResult.imageType}] ${captionText || ''}`; addToConversationHistory(session, 'user', imageContext); // ——— STEP 2: K2 THINK V2 DEEP REASONING ——— // Send Gemma's extraction to K2 → K2 researches, reasons, verifies, explains console.log(' 🧠 Step 2: Sending to K2 Think V2 for deep reasoning...'); // If medications were found, also get FDA data for K2 let fdaEnrichment = ''; if (['medication', 'prescription'].includes(gemmaResult.imageType)) { fdaEnrichment = await getFDAEnrichment(gemmaResult.extraction); } const enrichedExtraction = fdaEnrichment ? `${gemmaResult.extraction}\n\n=== FDA DATABASE VERIFICATION ===\n${fdaEnrichment}` : gemmaResult.extraction; const k2Response = await k2ThinkService.analyzeImageExtraction( enrichedExtraction, gemmaResult.imageType, session.language, session.conversationHistory ); // ——— STEP 3: SEND ONLY FINAL ANSWER TO WHATSAPP ——— // K2's polished response goes directly to the user — no thinking traces console.log(` ✅ Step 3: Sending final answer (${k2Response.length} chars) to WhatsApp`); // Store K2's response in memory so user can ask follow-up questions addToConversationHistory(session, 'assistant', k2Response); await sendMessage(from, k2Response); // Add follow-up invitation const followUpMsg = session.language === 'es' ? '\n💬 ¿Tienes alguna pregunta sobre lo que te mostré? ¡Solo pregunta!' : '\n💬 Have any questions about what I just explained? Just ask!'; await sendMessage(from, followUpMsg); return reply.code(200).send({ status: 'image_analyzed', type: gemmaResult.imageType }); } catch (err) { console.error(' ❌ Image pipeline error:', err); const errorMsg = session.language === 'es' ? '❌ No pude procesar esa imagen. ¿Podrías enviarla de nuevo, preferiblemente como archivo (📎 → Documento)?' : '❌ I couldn\'t process that image. Could you send it again, preferably as a file (📎 → Document)?'; await sendMessage(from, errorMsg); } return reply.code(200).send({ status: 'media_processed' }); } // ═══════════════════════════════════════════════════════════════════════════════ // MEDICATION QUERY — Text-based, K2 Think V2 + openFDA with memory // ═══════════════════════════════════════════════════════════════════════════════ async function handleMedicationQuery(from: string, text: string, session: any): Promise { addToConversationHistory(session, 'user', text); // Try to extract drug names from the message const drugMatches = text.match(/\b([A-Z][a-z]+(?:in|ol|pine|ide|ate|one|fen|pam|lam|tin|ril|tan|mab)\b)/gi) || []; let fdaContext = ''; for (const drugName of drugMatches.slice(0, 3)) { const rxcui = await normalizeDrugName(drugName); if (rxcui) { const info = await getDrugInfo(rxcui); fdaContext += `\n[FDA DATA for ${drugName}]: Purpose: ${info.purpose}. Generic: ${info.genericName}. Warnings: ${info.warnings.join('; ')}`; } } // If user has existing medications, check interactions if (drugMatches.length > 0 && session.medications?.length > 0) { const existingRxcuis = session.medications.map((m: any) => m.rxcui); for (const drugName of drugMatches) { const rxcui = await normalizeDrugName(drugName); if (rxcui) { const interactions = await checkInteractions([...existingRxcuis, rxcui]); if (interactions.length > 0) { for (const ix of interactions) { fdaContext += `\n[FDA INTERACTION WARNING]: ${ix.drug1} + ${ix.drug2}: ${ix.description}. Deaths: ${ix.deathCount}`; } } } } } // Route through K2 Think V2 — with full conversation history for memory const response = await k2ThinkService.analyzeMedication(text, fdaContext, session.language, session.conversationHistory); addToConversationHistory(session, 'assistant', response); await sendMessage(from, response); } // ═══════════════════════════════════════════════════════════════════════════════ // FDA ENRICHMENT — Extract drug names from Gemma text and look up FDA data // ═══════════════════════════════════════════════════════════════════════════════ async function getFDAEnrichment(extractionText: string): Promise { // Try to extract drug names from the vision extraction const drugMatches = extractionText.match(/\b([A-Z][a-z]+(?:in|ol|pine|ide|ate|one|fen|pam|lam|tin|ril|tan|mab)\b)/gi) || []; // Also look for common drug names that might not match the suffix pattern const commonDrugs = ['aspirin', 'tylenol', 'advil', 'motrin', 'aleve', 'prilosec', 'nexium', 'lipitor', 'zoloft', 'lexapro', 'metformin', 'insulin', 'lisinopril', 'amoxicillin', 'azithromycin', 'prednisone', 'albuterol', 'ibuprofen', 'acetaminophen', 'omeprazole', 'losartan', 'amlodipine', 'atorvastatin']; const lower = extractionText.toLowerCase(); const additionalMatches = commonDrugs.filter(d => lower.includes(d)); const allDrugs = [...new Set([...drugMatches, ...additionalMatches])].slice(0, 5); if (allDrugs.length === 0) return ''; let fdaData = ''; const rxcuis: string[] = []; for (const drugName of allDrugs) { try { const rxcui = await normalizeDrugName(drugName); if (rxcui) { rxcuis.push(rxcui); const info = await getDrugInfo(rxcui); fdaData += `\n[FDA: ${drugName}] Purpose: ${info.purpose}. Generic: ${info.genericName}. Warnings: ${info.warnings.join('; ')}`; } } catch {} } // Check interactions between all found drugs if (rxcuis.length >= 2) { try { const interactions = await checkInteractions(rxcuis); for (const ix of interactions) { fdaData += `\n[FDA INTERACTION]: ${ix.drug1} + ${ix.drug2}: ${ix.description}. Severity: ${ix.severity}. Deaths: ${ix.deathCount || 0}`; } } catch {} } return fdaData; } // ═══════════════════════════════════════════════════════════════════════════════ // UTILITIES // ═══════════════════════════════════════════════════════════════════════════════ async function withTimeout(promise: Promise, timeoutMs: number, fallback: T): Promise { let timeoutHandle: NodeJS.Timeout | null = null; try { const timeoutPromise = new Promise((resolve) => { timeoutHandle = setTimeout(() => resolve(fallback), timeoutMs); }); return await Promise.race([promise, timeoutPromise]); } finally { if (timeoutHandle) clearTimeout(timeoutHandle); } }