generated-bill / backend /src /modules /ocr /ocr.controller.ts
narasimha01tm's picture
Deploy SmartSpend backend
4bf0aa2
Raw History Blame Contribute Delete
7.14 kB
import { Request, Response } from 'express';
import { asyncHandler } from '../../utils/asyncHandler';
import { createWorker } from 'tesseract.js';
import { processExtractedText } from './ocr.utils';
import { HuggingFaceService } from '../../services/hugging-face.service';
import sharp from 'sharp';
// Helper to get image dimensions from buffer (basic JPEG/PNG support)
async function getImageDimensions(buffer: Buffer): Promise<{ width: number; height: number }> {
try {
const metadata = await sharp(buffer).metadata();
return {
width: metadata.width || 0,
height: metadata.height || 0
};
} catch (error) {
console.error('Error getting image dimensions:', error);
return { width: 0, height: 0 };
}
}
// Function to call Python OCR server
async function runPythonOCR(imageBase64: string): Promise<any> {
try {
const pythonServiceUrl = process.env.PYTHON_OCR_URL || 'http://localhost:5001/ocr';
const response = await fetch(pythonServiceUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ imageBase64 })
});
if (!response.ok) {
throw new Error(`Python server error: ${response.status} ${response.statusText}`);
}
const result = await response.json();
if (result.success) {
return result;
} else {
throw new Error(result.error || 'Unknown OCR error from Python server');
}
} catch (error: any) {
console.error('Failed to communicate with Python OCR server:', error);
throw error; // Rethrow to trigger fallback
}
}
// Function to run Tesseract.js OCR as fallback
async function runTesseractOCR(imageBase64: string): Promise<string> {
console.log('Falling back to Tesseract.js OCR...');
let worker;
try {
// Create worker with English language
worker = await createWorker('eng');
// Recognize text from base64 image
// Tesseract.js accepts base64 strings directly (with or without data URI prefix)
const { data: { text } } = await worker.recognize(imageBase64);
await worker.terminate();
return text;
} catch (error) {
console.error('Tesseract.js OCR failed:', error);
if (worker) {
await worker.terminate();
}
throw new Error('Tesseract.js OCR failed');
}
}
// Function to run Hugging Face extraction
async function runHuggingFaceExtraction(imageBase64: string): Promise<any> {
console.log('Attempting Hugging Face Donut extraction...');
const result = await HuggingFaceService.extractFromImage(imageBase64);
if (result.success && result.data) {
return result.data;
}
throw new Error(result.error || 'Hugging Face extraction failed');
}
// Using any for req and res to resolve Property 'status' missing errors
export const processImage = asyncHandler(async (req: any, res: any) => {
try {
const { imageBase64, useHuggingFace = false } = req.body;
if (!imageBase64) {
return res.status(400).json({ error: 'Image data is required' });
}
console.log(`Processing image...`);
let text = '';
let processedData: any = null;
let engine = 'paddleocr-python';
// High-priority: If user explicitly wants Hugging Face or if we want to try it first
const hfApiKey = process.env.HUGGING_FACE_API_KEY;
const isHfConfigured = hfApiKey && hfApiKey !== 'YOUR_HF_TOKEN_HERE';
if (useHuggingFace || isHfConfigured) {
try {
console.log('Attempting Hugging Face Hybrid Extraction...');
const result = await HuggingFaceService.extractFromImage(imageBase64);
if (result.success && result.data) {
processedData = result.data;
text = result.rawText || '';
engine = 'huggingface-hybrid-mistral';
console.log('Hugging Face successful');
}
} catch (hfError: any) {
console.warn('Hugging Face failed, falling back to local OCR:', hfError.message);
}
}
if (!processedData) {
try {
// 1. Try Python PaddleOCR Server
console.log('Attempting PaddleOCR...');
const ocrResult = await runPythonOCR(imageBase64);
text = ocrResult.text;
console.log('PaddleOCR successful');
} catch (pythonError: any) {
console.warn('PaddleOCR failed, attempting fallback to Tesseract.js:', pythonError.message);
// 2. Fallback to Tesseract.js
try {
text = await runTesseractOCR(imageBase64);
engine = 'tesseract.js-fallback';
console.log('Tesseract.js successful');
} catch (tesseractError: any) {
console.error('All OCR methods failed');
throw new Error('All OCR methods failed. Please check server logs.');
}
}
if (!text || text.trim().length === 0) {
throw new Error('OCR returned empty output');
}
// Option to refine text extraction using LLM
const { useHuggingFaceLLM = false } = req.body;
if ((useHuggingFaceLLM || isHfConfigured) && text) {
try {
console.log('Refining extraction with Hugging Face LLM...');
const llmResult = await HuggingFaceService.parseText(text);
if (llmResult.success && llmResult.data) {
processedData = llmResult.data;
engine += ` + hf-llm-refinement`;
console.log('LLM refinement successful');
}
} catch (llmError: any) {
console.warn('LLM refinement failed, using regex extraction:', llmError.message);
}
}
// Fallback to regex-based extraction if LLM refinement wasn't used or failed
if (!processedData) {
processedData = processExtractedText(text);
}
}
res.status(200).json({
success: true,
text: text || 'Handled by structured model',
data: processedData,
engine: engine
});
} catch (error: any) {
console.error('OCR processing error (outer catch):', error);
// Final fallback response if everything fails
res.status(200).json({
success: false, // Mark as false so frontend knows it failed
text: 'OCR processing failed - please try with a clearer image',
data: {
storeName: 'Unknown Store',
totalAmount: 0,
items: []
},
engine: 'failed',
error: error.message
});
}
});