healthbridge / src /webhook.ts
Preshak Bhattarai
Initial commit for HF Space
ff7df3e
Raw History Blame Contribute Delete
19.2 kB
// src/webhook.ts β€” Production HealthBridge Webhook
// Pipeline Architecture:
// TEXT β†’ K2 Think V2 (deep reasoning + openFDA) β†’ final answer β†’ WhatsApp
// IMAGE β†’ Gemma 4 (extract everything) β†’ K2 Think V2 (research + reason) β†’ final answer β†’ WhatsApp
// MEMORY β†’ full conversation history so users can ask follow-up questions (like ChatGPT)
import { sendMessage } from './services/whatsapp-baileys.js';
import { openrouterService } from './services/openrouter.js';
import { k2ThinkService } from './services/k2-think.js';
import { getOrCreateSession, addToConversationHistory, resetFlow } from './services/session.js';
import { checkRateLimit, sanitize } from './services/security.js';
import { checkEmergency, getEmergencyResponse } from './flows/emergency.js';
import { startTriage, continueTriage } from './flows/triage.js';
import { startProvider, continueProvider, handleSafeAccess } from './flows/provider.js';
import { handleReminderSetup, handleTaken } from './flows/reminder.js';
import { extractImageDetails, processPDF } from './services/vision.js';
import { normalizeDrugName } from './services/rxnorm.js';
import { checkInteractions, getDrugInfo } from './services/openfda.js';
import { WELCOME, RATE_LIMIT_MSG } from './data/messages.js';
import { Language } from './types/index.js';
import sharp from 'sharp';
import axios from 'axios';
// β€”β€”β€” Fetch Full-Quality Image (bypasses WhatsApp compression) β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”β€”
async function fetchFullQualityImage(mediaId: string): Promise<{ base64: string, mimeType: string }> {
const token = process.env.WHATSAPP_TOKEN;
if (!token) throw new Error("Missing WHATSAPP_TOKEN in environment");
const urlRes = await axios.get(
`https://graph.facebook.com/v19.0/${mediaId}`,
{ headers: { Authorization: `Bearer ${token}` } }
);
const downloadUrl = urlRes.data.url;
const mimeType = urlRes.data.mime_type;
const imgRes = await axios.get(downloadUrl, {
responseType: "arraybuffer",
headers: { Authorization: `Bearer ${token}` },
});
return {
base64: Buffer.from(imgRes.data).toString("base64"),
mimeType: mimeType || "image/jpeg",
};
}
export async function handleWhatsApp(request: any, reply: any) {
try {
const body = request.body;
const from: string = body.From || '';
const rawText: string = body.Body || '';
let mediaBuffer: Buffer | null = body.MediaBuffer || null;
let mediaType: string = body.MediaType || '';
const mediaId: string = body.MediaId || '';
// Meta Production Webhook β€” fetch full resolution image using media_id
if (mediaId && !mediaBuffer) {
console.log(` πŸ“Έ Fetching full-quality original image from Meta for media_id: ${mediaId}`);
try {
const fullQuality = await fetchFullQualityImage(mediaId);
mediaBuffer = Buffer.from(fullQuality.base64, 'base64');
mediaType = fullQuality.mimeType;
} catch (err: any) {
console.error(" ❌ Failed to fetch full-quality image:", err.message);
}
}
if (!from) return reply.code(400).send({ error: 'Missing sender ID' });
const text = sanitize(rawText);
// Rate limit
if (!checkRateLimit(from)) {
await sendMessage(from, RATE_LIMIT_MSG.en || '⏳ Too many messages. Please wait.');
return reply.code(200).send({ status: 'rate_limited' });
}
const session = getOrCreateSession(from);
// ===== 1. EMERGENCY OVERRIDE β€” instant, no LLM =====
const emergencyLang = checkEmergency(text);
if (emergencyLang) {
session.language = emergencyLang;
await sendMessage(from, getEmergencyResponse(emergencyLang));
console.log(` 🚨 EMERGENCY [${emergencyLang}]`);
return reply.code(200).send({ status: 'emergency' });
}
// ===== 2. TAKEN acknowledgment =====
const lowerTrim = text.toLowerCase().trim();
if (['taken', 'tomado', '已服', 'ΰ€²ΰ€Ώΰ€―ΰ€Ύ', 'ΨͺΩ…', 'Δ‘Γ£ uα»‘ng', 'λ³΅μš©μ™„λ£Œ', 'ininom'].includes(lowerTrim)) {
const response = handleTaken(from, session);
await sendMessage(from, response);
return reply.code(200).send({ status: 'taken' });
}
// ===== 3. MEDIA β€” Images, PDFs, Documents =====
// Pipeline: Gemma 4 extracts β†’ K2 thinks β†’ final answer to WhatsApp
if (mediaBuffer) {
return await handleMedia(from, mediaBuffer, mediaType, text, session, reply);
}
// Empty text, no media β€” skip
if (!text.trim()) return reply.code(200).send({ status: 'empty' });
// ===== 4. CONTINUE ACTIVE FLOW =====
if (session.currentFlow !== 'idle') {
console.log(` πŸ”„ Flow: ${session.currentFlow} step ${session.triageStep}`);
switch (session.currentFlow) {
case 'triage':
await continueTriage(from, text, session, sendMessage);
break;
case 'provider':
case 'safe_access':
await continueProvider(from, text, session, sendMessage);
break;
case 'reminder_setup':
await handleReminderSetup(from, text, session, sendMessage);
break;
default:
resetFlow(session);
break;
}
return reply.code(200).send({ status: 'flow', flow: session.currentFlow });
}
// ===== 5. DETECT INTENT =====
const intent = await openrouterService.detectIntent(text, session.language);
console.log(` 🧠 Intent: ${intent.intent} [${intent.detectedLanguage}] (${intent.confidence})`);
// Update language from detection
const validLangs: Language[] = ['en', 'es', 'zh', 'hi', 'ar', 'vi', 'ko', 'tl', 'pt'];
if (validLangs.includes(intent.detectedLanguage as Language)) {
session.language = intent.detectedLanguage as Language;
}
// ===== 6. ROUTE BY INTENT β€” All text goes through K2 with memory =====
switch (intent.intent) {
case 'emergency':
const eLang = checkEmergency(text) || session.language;
await sendMessage(from, getEmergencyResponse(eLang as Language));
break;
case 'triage':
await startTriage(from, text, session, sendMessage);
break;
case 'medication':
await handleMedicationQuery(from, text, session);
break;
case 'find_provider':
await startProvider(from, text, session, sendMessage);
break;
case 'safe_access':
await handleSafeAccess(from, text, session, sendMessage);
break;
case 'insurance':
// Route insurance through K2 for deep, warm explanation β€” with full history
addToConversationHistory(session, 'user', text);
const insuranceResp = await k2ThinkService.explainInsurance(text, session.language, session.conversationHistory);
addToConversationHistory(session, 'assistant', insuranceResp);
await sendMessage(from, insuranceResp);
break;
case 'greeting':
const welcome = WELCOME[session.language] || WELCOME.en;
await sendMessage(from, welcome);
addToConversationHistory(session, 'assistant', welcome);
break;
case 'taken':
const takenResp = handleTaken(from, session);
await sendMessage(from, takenResp);
break;
default:
// General AI conversation β€” K2 Think V2 with full conversation memory
// User can ask follow-up questions about previous replies (like ChatGPT)
addToConversationHistory(session, 'user', text);
const aiResp = await k2ThinkService.chat(text, session.language, session.conversationHistory);
addToConversationHistory(session, 'assistant', aiResp);
await sendMessage(from, aiResp);
break;
}
return reply.code(200).send({ status: 'processed', intent: intent.intent });
} catch (err: any) {
console.error('❌ Webhook error:', err.message || err);
return reply.code(500).send({ error: 'Internal server error' });
}
}
// ═══════════════════════════════════════════════════════════════════════════════
// IMAGE PIPELINE β€” The heart of the dual-model architecture
// Step 1: Gemma 4 Vision β†’ extract ALL details from the image
// Step 2: K2 Think V2 β†’ research, reason, and explain everything
// Step 3: ONLY the final polished answer β†’ WhatsApp
// Step 4: Store in conversation memory β†’ user can ask follow-ups
// ═══════════════════════════════════════════════════════════════════════════════
async function handleMedia(
from: string,
buffer: Buffer,
mimeType: string,
captionText: string,
session: any,
reply: any
): Promise<any> {
// === PDF HANDLING ===
if (mimeType === 'application/pdf') {
const pdf = await processPDF(buffer);
if (pdf.text && pdf.confidence > 0.5) {
const processingMsg = session.language === 'es'
? 'πŸ“„ Analizando tu documento PDF...'
: 'πŸ“„ Analyzing your PDF document...';
await sendMessage(from, processingMsg);
// Store the PDF content in conversation memory
addToConversationHistory(session, 'user', `[User sent a PDF document] ${captionText || ''}`);
// Send PDF text directly to K2 for deep analysis
const explanation = await k2ThinkService.analyzeImageExtraction(
pdf.text, 'pdf_document', session.language, session.conversationHistory
);
addToConversationHistory(session, 'assistant', explanation);
await sendMessage(from, explanation);
return reply.code(200).send({ status: 'pdf_analyzed' });
}
}
// === IMAGE HANDLING β€” Full Gemma β†’ K2 Pipeline ===
try {
const processingMsg = session.language === 'es'
? 'πŸ” Analizando tu imagen con IA... un momento por favor'
: 'πŸ” Analyzing your image with AI... one moment please';
await sendMessage(from, processingMsg);
// β€”β€”β€” STEP 1: GEMMA 4 VISION EXTRACTION β€”β€”β€”
// Send image to Gemma β†’ extracts every possible detail
console.log(' πŸ‘οΈ Step 1: Sending image to Gemma 4 Vision for extraction...');
const gemmaResult = await withTimeout(
extractImageDetails(buffer),
45000,
{ extraction: '', imageType: 'unknown', confidence: 0 }
);
console.log(` πŸ‘οΈ Gemma extracted: ${gemmaResult.extraction.length} chars, type: ${gemmaResult.imageType}, confidence: ${gemmaResult.confidence}`);
// If Gemma couldn't extract anything useful
if (!gemmaResult.extraction || gemmaResult.confidence < 0.2) {
const retryMsg = session.language === 'es'
? 'πŸ˜• No pude leer bien esa imagen. Intenta:\n\n1️⃣ Buena iluminaciΓ³n frontal\n2️⃣ Acercarte al texto/etiqueta\n3️⃣ Sin movimiento (imagen nΓ­tida)\n4️⃣ Enviar como *Documento* (πŸ“Ž β†’ Documento)\n\nΒ‘O describe lo que ves y te ayudo!'
: 'πŸ˜• I couldn\'t read that image clearly. Try:\n\n1️⃣ Good front lighting\n2️⃣ Closer to the text/label\n3️⃣ No motion blur (hold steady)\n4️⃣ Send as *Document* (πŸ“Ž β†’ Document)\n\nOr describe what you see and I\'ll help!';
await sendMessage(from, retryMsg);
return reply.code(200).send({ status: 'image_unclear' });
}
// Store the image context in conversation memory so user can ask follow-ups
const imageContext = `[User sent an image: ${gemmaResult.imageType}] ${captionText || ''}`;
addToConversationHistory(session, 'user', imageContext);
// β€”β€”β€” STEP 2: K2 THINK V2 DEEP REASONING β€”β€”β€”
// Send Gemma's extraction to K2 β†’ K2 researches, reasons, verifies, explains
console.log(' 🧠 Step 2: Sending to K2 Think V2 for deep reasoning...');
// If medications were found, also get FDA data for K2
let fdaEnrichment = '';
if (['medication', 'prescription'].includes(gemmaResult.imageType)) {
fdaEnrichment = await getFDAEnrichment(gemmaResult.extraction);
}
const enrichedExtraction = fdaEnrichment
? `${gemmaResult.extraction}\n\n=== FDA DATABASE VERIFICATION ===\n${fdaEnrichment}`
: gemmaResult.extraction;
const k2Response = await k2ThinkService.analyzeImageExtraction(
enrichedExtraction,
gemmaResult.imageType,
session.language,
session.conversationHistory
);
// β€”β€”β€” STEP 3: SEND ONLY FINAL ANSWER TO WHATSAPP β€”β€”β€”
// K2's polished response goes directly to the user β€” no thinking traces
console.log(` βœ… Step 3: Sending final answer (${k2Response.length} chars) to WhatsApp`);
// Store K2's response in memory so user can ask follow-up questions
addToConversationHistory(session, 'assistant', k2Response);
await sendMessage(from, k2Response);
// Add follow-up invitation
const followUpMsg = session.language === 'es'
? '\nπŸ’¬ ΒΏTienes alguna pregunta sobre lo que te mostrΓ©? Β‘Solo pregunta!'
: '\nπŸ’¬ Have any questions about what I just explained? Just ask!';
await sendMessage(from, followUpMsg);
return reply.code(200).send({ status: 'image_analyzed', type: gemmaResult.imageType });
} catch (err) {
console.error(' ❌ Image pipeline error:', err);
const errorMsg = session.language === 'es'
? '❌ No pude procesar esa imagen. ΒΏPodrΓ­as enviarla de nuevo, preferiblemente como archivo (πŸ“Ž β†’ Documento)?'
: '❌ I couldn\'t process that image. Could you send it again, preferably as a file (πŸ“Ž β†’ Document)?';
await sendMessage(from, errorMsg);
}
return reply.code(200).send({ status: 'media_processed' });
}
// ═══════════════════════════════════════════════════════════════════════════════
// MEDICATION QUERY β€” Text-based, K2 Think V2 + openFDA with memory
// ═══════════════════════════════════════════════════════════════════════════════
async function handleMedicationQuery(from: string, text: string, session: any): Promise<void> {
addToConversationHistory(session, 'user', text);
// Try to extract drug names from the message
const drugMatches = text.match(/\b([A-Z][a-z]+(?:in|ol|pine|ide|ate|one|fen|pam|lam|tin|ril|tan|mab)\b)/gi) || [];
let fdaContext = '';
for (const drugName of drugMatches.slice(0, 3)) {
const rxcui = await normalizeDrugName(drugName);
if (rxcui) {
const info = await getDrugInfo(rxcui);
fdaContext += `\n[FDA DATA for ${drugName}]: Purpose: ${info.purpose}. Generic: ${info.genericName}. Warnings: ${info.warnings.join('; ')}`;
}
}
// If user has existing medications, check interactions
if (drugMatches.length > 0 && session.medications?.length > 0) {
const existingRxcuis = session.medications.map((m: any) => m.rxcui);
for (const drugName of drugMatches) {
const rxcui = await normalizeDrugName(drugName);
if (rxcui) {
const interactions = await checkInteractions([...existingRxcuis, rxcui]);
if (interactions.length > 0) {
for (const ix of interactions) {
fdaContext += `\n[FDA INTERACTION WARNING]: ${ix.drug1} + ${ix.drug2}: ${ix.description}. Deaths: ${ix.deathCount}`;
}
}
}
}
}
// Route through K2 Think V2 β€” with full conversation history for memory
const response = await k2ThinkService.analyzeMedication(text, fdaContext, session.language, session.conversationHistory);
addToConversationHistory(session, 'assistant', response);
await sendMessage(from, response);
}
// ═══════════════════════════════════════════════════════════════════════════════
// FDA ENRICHMENT β€” Extract drug names from Gemma text and look up FDA data
// ═══════════════════════════════════════════════════════════════════════════════
async function getFDAEnrichment(extractionText: string): Promise<string> {
// Try to extract drug names from the vision extraction
const drugMatches = extractionText.match(/\b([A-Z][a-z]+(?:in|ol|pine|ide|ate|one|fen|pam|lam|tin|ril|tan|mab)\b)/gi) || [];
// Also look for common drug names that might not match the suffix pattern
const commonDrugs = ['aspirin', 'tylenol', 'advil', 'motrin', 'aleve', 'prilosec', 'nexium',
'lipitor', 'zoloft', 'lexapro', 'metformin', 'insulin', 'lisinopril',
'amoxicillin', 'azithromycin', 'prednisone', 'albuterol', 'ibuprofen',
'acetaminophen', 'omeprazole', 'losartan', 'amlodipine', 'atorvastatin'];
const lower = extractionText.toLowerCase();
const additionalMatches = commonDrugs.filter(d => lower.includes(d));
const allDrugs = [...new Set([...drugMatches, ...additionalMatches])].slice(0, 5);
if (allDrugs.length === 0) return '';
let fdaData = '';
const rxcuis: string[] = [];
for (const drugName of allDrugs) {
try {
const rxcui = await normalizeDrugName(drugName);
if (rxcui) {
rxcuis.push(rxcui);
const info = await getDrugInfo(rxcui);
fdaData += `\n[FDA: ${drugName}] Purpose: ${info.purpose}. Generic: ${info.genericName}. Warnings: ${info.warnings.join('; ')}`;
}
} catch {}
}
// Check interactions between all found drugs
if (rxcuis.length >= 2) {
try {
const interactions = await checkInteractions(rxcuis);
for (const ix of interactions) {
fdaData += `\n[FDA INTERACTION]: ${ix.drug1} + ${ix.drug2}: ${ix.description}. Severity: ${ix.severity}. Deaths: ${ix.deathCount || 0}`;
}
} catch {}
}
return fdaData;
}
// ═══════════════════════════════════════════════════════════════════════════════
// UTILITIES
// ═══════════════════════════════════════════════════════════════════════════════
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, fallback: T): Promise<T> {
let timeoutHandle: NodeJS.Timeout | null = null;
try {
const timeoutPromise = new Promise<T>((resolve) => {
timeoutHandle = setTimeout(() => resolve(fallback), timeoutMs);
});
return await Promise.race([promise, timeoutPromise]);
} finally {
if (timeoutHandle) clearTimeout(timeoutHandle);
}
}