File size: 7,143 Bytes
4bf0aa2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import { Request, Response } from 'express';
import { asyncHandler } from '../../utils/asyncHandler';
import { createWorker } from 'tesseract.js';
import { processExtractedText } from './ocr.utils';
import { HuggingFaceService } from '../../services/hugging-face.service';
import sharp from 'sharp';

// Helper to get image dimensions from buffer (basic JPEG/PNG support)
async function getImageDimensions(buffer: Buffer): Promise<{ width: number; height: number }> {
    try {
        const metadata = await sharp(buffer).metadata();
        return {
            width: metadata.width || 0,
            height: metadata.height || 0
        };
    } catch (error) {
        console.error('Error getting image dimensions:', error);
        return { width: 0, height: 0 };
    }
}

// Function to call Python OCR server
async function runPythonOCR(imageBase64: string): Promise<any> {
    try {
        const pythonServiceUrl = process.env.PYTHON_OCR_URL || 'http://localhost:5001/ocr';
        const response = await fetch(pythonServiceUrl, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json'
            },
            body: JSON.stringify({ imageBase64 })
        });

        if (!response.ok) {
            throw new Error(`Python server error: ${response.status} ${response.statusText}`);
        }

        const result = await response.json();
        if (result.success) {
            return result;
        } else {
            throw new Error(result.error || 'Unknown OCR error from Python server');
        }
    } catch (error: any) {
        console.error('Failed to communicate with Python OCR server:', error);
        throw error; // Rethrow to trigger fallback
    }
}

// Function to run Tesseract.js OCR as fallback
async function runTesseractOCR(imageBase64: string): Promise<string> {
    console.log('Falling back to Tesseract.js OCR...');
    let worker;
    try {
        // Create worker with English language
        worker = await createWorker('eng');

        // Recognize text from base64 image
        // Tesseract.js accepts base64 strings directly (with or without data URI prefix)
        const { data: { text } } = await worker.recognize(imageBase64);

        await worker.terminate();
        return text;
    } catch (error) {
        console.error('Tesseract.js OCR failed:', error);
        if (worker) {
            await worker.terminate();
        }
        throw new Error('Tesseract.js OCR failed');
    }
}

// Function to run Hugging Face extraction
async function runHuggingFaceExtraction(imageBase64: string): Promise<any> {
    console.log('Attempting Hugging Face Donut extraction...');
    const result = await HuggingFaceService.extractFromImage(imageBase64);
    if (result.success && result.data) {
        return result.data;
    }
    throw new Error(result.error || 'Hugging Face extraction failed');
}

// Using any for req and res to resolve Property 'status' missing errors
export const processImage = asyncHandler(async (req: any, res: any) => {
    try {
        const { imageBase64, useHuggingFace = false } = req.body;

        if (!imageBase64) {
            return res.status(400).json({ error: 'Image data is required' });
        }

        console.log(`Processing image...`);
        let text = '';
        let processedData: any = null;
        let engine = 'paddleocr-python';

        // High-priority: If user explicitly wants Hugging Face or if we want to try it first
        const hfApiKey = process.env.HUGGING_FACE_API_KEY;
        const isHfConfigured = hfApiKey && hfApiKey !== 'YOUR_HF_TOKEN_HERE';

        if (useHuggingFace || isHfConfigured) {
            try {
                console.log('Attempting Hugging Face Hybrid Extraction...');
                const result = await HuggingFaceService.extractFromImage(imageBase64);
                if (result.success && result.data) {
                    processedData = result.data;
                    text = result.rawText || '';
                    engine = 'huggingface-hybrid-mistral';
                    console.log('Hugging Face successful');
                }
            } catch (hfError: any) {
                console.warn('Hugging Face failed, falling back to local OCR:', hfError.message);
            }
        }

        if (!processedData) {
            try {
                // 1. Try Python PaddleOCR Server
                console.log('Attempting PaddleOCR...');
                const ocrResult = await runPythonOCR(imageBase64);
                text = ocrResult.text;
                console.log('PaddleOCR successful');

            } catch (pythonError: any) {
                console.warn('PaddleOCR failed, attempting fallback to Tesseract.js:', pythonError.message);

                // 2. Fallback to Tesseract.js
                try {
                    text = await runTesseractOCR(imageBase64);
                    engine = 'tesseract.js-fallback';
                    console.log('Tesseract.js successful');
                } catch (tesseractError: any) {
                    console.error('All OCR methods failed');
                    throw new Error('All OCR methods failed. Please check server logs.');
                }
            }

            if (!text || text.trim().length === 0) {
                throw new Error('OCR returned empty output');
            }

            // Option to refine text extraction using LLM
            const { useHuggingFaceLLM = false } = req.body;
            if ((useHuggingFaceLLM || isHfConfigured) && text) {
                try {
                    console.log('Refining extraction with Hugging Face LLM...');
                    const llmResult = await HuggingFaceService.parseText(text);
                    if (llmResult.success && llmResult.data) {
                        processedData = llmResult.data;
                        engine += ` + hf-llm-refinement`;
                        console.log('LLM refinement successful');
                    }
                } catch (llmError: any) {
                    console.warn('LLM refinement failed, using regex extraction:', llmError.message);
                }
            }

            // Fallback to regex-based extraction if LLM refinement wasn't used or failed
            if (!processedData) {
                processedData = processExtractedText(text);
            }
        }

        res.status(200).json({
            success: true,
            text: text || 'Handled by structured model',
            data: processedData,
            engine: engine
        });

    } catch (error: any) {
        console.error('OCR processing error (outer catch):', error);

        // Final fallback response if everything fails
        res.status(200).json({
            success: false, // Mark as false so frontend knows it failed
            text: 'OCR processing failed - please try with a clearer image',
            data: {
                storeName: 'Unknown Store',
                totalAmount: 0,
                items: []
            },
            engine: 'failed',
            error: error.message
        });
    }
});