import { Request, Response } from 'express'; import { asyncHandler } from '../../utils/asyncHandler'; import { createWorker } from 'tesseract.js'; import { processExtractedText } from './ocr.utils'; import { HuggingFaceService } from '../../services/hugging-face.service'; import sharp from 'sharp'; // Helper to get image dimensions from buffer (basic JPEG/PNG support) async function getImageDimensions(buffer: Buffer): Promise<{ width: number; height: number }> { try { const metadata = await sharp(buffer).metadata(); return { width: metadata.width || 0, height: metadata.height || 0 }; } catch (error) { console.error('Error getting image dimensions:', error); return { width: 0, height: 0 }; } } // Function to call Python OCR server async function runPythonOCR(imageBase64: string): Promise { try { const pythonServiceUrl = process.env.PYTHON_OCR_URL || 'http://localhost:5001/ocr'; const response = await fetch(pythonServiceUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ imageBase64 }) }); if (!response.ok) { throw new Error(`Python server error: ${response.status} ${response.statusText}`); } const result = await response.json(); if (result.success) { return result; } else { throw new Error(result.error || 'Unknown OCR error from Python server'); } } catch (error: any) { console.error('Failed to communicate with Python OCR server:', error); throw error; // Rethrow to trigger fallback } } // Function to run Tesseract.js OCR as fallback async function runTesseractOCR(imageBase64: string): Promise { console.log('Falling back to Tesseract.js OCR...'); let worker; try { // Create worker with English language worker = await createWorker('eng'); // Recognize text from base64 image // Tesseract.js accepts base64 strings directly (with or without data URI prefix) const { data: { text } } = await worker.recognize(imageBase64); await worker.terminate(); return text; } catch (error) { console.error('Tesseract.js OCR failed:', error); if (worker) { await worker.terminate(); } throw new Error('Tesseract.js OCR failed'); } } // Function to run Hugging Face extraction async function runHuggingFaceExtraction(imageBase64: string): Promise { console.log('Attempting Hugging Face Donut extraction...'); const result = await HuggingFaceService.extractFromImage(imageBase64); if (result.success && result.data) { return result.data; } throw new Error(result.error || 'Hugging Face extraction failed'); } // Using any for req and res to resolve Property 'status' missing errors export const processImage = asyncHandler(async (req: any, res: any) => { try { const { imageBase64, useHuggingFace = false } = req.body; if (!imageBase64) { return res.status(400).json({ error: 'Image data is required' }); } console.log(`Processing image...`); let text = ''; let processedData: any = null; let engine = 'paddleocr-python'; // High-priority: If user explicitly wants Hugging Face or if we want to try it first const hfApiKey = process.env.HUGGING_FACE_API_KEY; const isHfConfigured = hfApiKey && hfApiKey !== 'YOUR_HF_TOKEN_HERE'; if (useHuggingFace || isHfConfigured) { try { console.log('Attempting Hugging Face Hybrid Extraction...'); const result = await HuggingFaceService.extractFromImage(imageBase64); if (result.success && result.data) { processedData = result.data; text = result.rawText || ''; engine = 'huggingface-hybrid-mistral'; console.log('Hugging Face successful'); } } catch (hfError: any) { console.warn('Hugging Face failed, falling back to local OCR:', hfError.message); } } if (!processedData) { try { // 1. Try Python PaddleOCR Server console.log('Attempting PaddleOCR...'); const ocrResult = await runPythonOCR(imageBase64); text = ocrResult.text; console.log('PaddleOCR successful'); } catch (pythonError: any) { console.warn('PaddleOCR failed, attempting fallback to Tesseract.js:', pythonError.message); // 2. Fallback to Tesseract.js try { text = await runTesseractOCR(imageBase64); engine = 'tesseract.js-fallback'; console.log('Tesseract.js successful'); } catch (tesseractError: any) { console.error('All OCR methods failed'); throw new Error('All OCR methods failed. Please check server logs.'); } } if (!text || text.trim().length === 0) { throw new Error('OCR returned empty output'); } // Option to refine text extraction using LLM const { useHuggingFaceLLM = false } = req.body; if ((useHuggingFaceLLM || isHfConfigured) && text) { try { console.log('Refining extraction with Hugging Face LLM...'); const llmResult = await HuggingFaceService.parseText(text); if (llmResult.success && llmResult.data) { processedData = llmResult.data; engine += ` + hf-llm-refinement`; console.log('LLM refinement successful'); } } catch (llmError: any) { console.warn('LLM refinement failed, using regex extraction:', llmError.message); } } // Fallback to regex-based extraction if LLM refinement wasn't used or failed if (!processedData) { processedData = processExtractedText(text); } } res.status(200).json({ success: true, text: text || 'Handled by structured model', data: processedData, engine: engine }); } catch (error: any) { console.error('OCR processing error (outer catch):', error); // Final fallback response if everything fails res.status(200).json({ success: false, // Mark as false so frontend knows it failed text: 'OCR processing failed - please try with a clearer image', data: { storeName: 'Unknown Store', totalAmount: 0, items: [] }, engine: 'failed', error: error.message }); } });