File size: 14,502 Bytes
fd842a4
 
5234643
fd842a4
5234643
fd842a4
 
 
 
5234643
 
 
fd842a4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5234643
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fd842a4
5234643
 
 
 
 
 
 
fd842a4
5234643
 
 
 
 
 
fd842a4
5234643
 
 
 
 
 
 
 
 
fd842a4
 
 
5234643
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fd842a4
5234643
 
 
 
 
 
 
fd842a4
5234643
 
 
 
 
 
fd842a4
5234643
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
from __future__ import annotations

import base64
import json
import logging
from pathlib import Path
from typing import Any


logger = logging.getLogger(__name__)


EXTRACTION_INSTRUCTIONS = (
    "You extract purchase receipt information from a single document. "
    "Return one JSON object that matches the schema exactly. "
    "Use null for missing values, empty arrays for missing line items, and numbers for amounts. "
    "Prefer the receipt's own values over guesses. "
    "Extract only these fields: fecha, fecha_vencimiento, cuit_proveedor, description_proveedor, moneda, subtotal, taxes, total. "
    "If the document is not a purchase receipt, set document_type to unknown but still capture any purchase-related hints you can find. "
    "Do not add markdown, prose, or extra keys."
)


def build_user_prompt(file_name: str, source_type: str) -> str:
    return (
        f"File name: {file_name}\n\n"
        f"Source type: {source_type}\n"
        "Important rules:\n"
        "- Read the full document before answering.\n"
        "- Copy amounts as numbers, not strings.\n"
        "- Use ISO 8601 for dates when possible.\n"
        "- Extract only: fecha, fecha_vencimiento, cuit_proveedor, description_proveedor, moneda, subtotal, taxes, total.\n"
        "- If a field cannot be found confidently, set it to null.\n"
        "- Keep output compact and do not include extra fields."
    )


def build_schema() -> dict[str, Any]:
    return {
        "name": "purchase_receipt_extraction",
        "schema": {
            "type": "object",
            "additionalProperties": False,
            "properties": {
                "source_file": {"type": "string"},
                "document_type": {"type": "string", "enum": ["purchase_receipt", "unknown"]},
                "fecha": {"type": ["string", "null"], "description": "ISO 8601 date when possible."},
                "fecha_vencimiento": {"type": ["string", "null"], "description": "ISO 8601 date when possible."},
                "cuit_proveedor": {"type": ["string", "null"]},
                "description_proveedor": {"type": ["string", "null"]},
                "moneda": {"type": ["string", "null"]},
                "subtotal": {"type": ["number", "null"]},
                "taxes": {"type": ["number", "null"]},
                "total": {"type": ["number", "null"]},
                "confidence": {"type": ["number", "null"], "minimum": 0, "maximum": 1},
            },
            "required": [
                "source_file",
                "document_type",
                "fecha",
                "fecha_vencimiento",
                "cuit_proveedor",
                "description_proveedor",
                "moneda",
                "subtotal",
                "taxes",
                "total",
                "confidence",
            ],
        },
        "strict": True,
    }


def extract_receipt_json(client: Any, model: str, file_path: Path, text: str) -> dict[str, Any]:
    logger.debug("extract_receipt_json called: file=%s, model=%s, text_length=%s", file_path.name, model, len(text))
    
    try:
        logger.debug("Sending text extraction request to OpenAI for %s", file_path.name)
        response = client.responses.create(
            model=model,
            instructions=EXTRACTION_INSTRUCTIONS,
            input=f"{build_user_prompt(file_path.name, 'text')}\n\nDocument text:\n{text[:20000]}",
            text={
                "format": {
                    "type": "json_schema",
                    "name": "purchase_receipt_extraction",
                    "schema": build_schema()["schema"],
                    "strict": True,
                }
            },
        )
        
        logger.debug("Received response from OpenAI for %s", file_path.name)

        output_text = getattr(response, "output_text", None)
        if output_text:
            logger.debug("Parsing output_text from response")
            result = json.loads(output_text)
            logger.debug("Successfully parsed JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        if response.output and response.output[0].content:
            logger.debug("Parsing response.output content")
            result = json.loads(response.output[0].content[0].text)
            logger.debug("Successfully parsed JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        logger.error("No JSON returned for %s - no output_text and no response.output", file_path.name)
        raise RuntimeError(f"No JSON returned for {file_path.name}")
    
    except json.JSONDecodeError as e:
        logger.error("JSON decode error for %s: %s", file_path.name, e)
        raise
    except Exception as e:
        logger.error("Error in extract_receipt_json for %s: %s", file_path.name, e)
        raise


def extract_receipt_json_from_image(client: Any, model: str, file_path: Path, image_data_uri: str) -> dict[str, Any]:
    logger.debug("extract_receipt_json_from_image called: file=%s, model=%s, image_size=%s bytes", 
                file_path.name, model, len(image_data_uri))
    
    try:
        logger.debug("Sending image extraction request to OpenAI for %s", file_path.name)
        response = client.responses.create(
            model=model,
            instructions=EXTRACTION_INSTRUCTIONS,
            input=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "input_text",
                            "text": build_user_prompt(file_path.name, "image"),
                        },
                        {
                            "type": "input_image",
                            "image_url": image_data_uri,
                            "detail": "high",
                        },
                    ],
                }
            ],
            text={
                "format": {
                    "type": "json_schema",
                    "name": "purchase_receipt_extraction",
                    "schema": build_schema()["schema"],
                    "strict": True,
                }
            },
        )
        
        logger.debug("Received response from OpenAI for image %s", file_path.name)

        output_text = getattr(response, "output_text", None)
        if output_text:
            logger.debug("Parsing output_text from image response")
            result = json.loads(output_text)
            logger.debug("Successfully parsed image JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        if response.output and response.output[0].content:
            logger.debug("Parsing response.output content from image")
            result = json.loads(response.output[0].content[0].text)
            logger.debug("Successfully parsed image JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        logger.error("No JSON returned for image %s - no output_text and no response.output", file_path.name)
        raise RuntimeError(f"No JSON returned for {file_path.name}")
    
    except json.JSONDecodeError as e:
        logger.error("JSON decode error for image %s: %s", file_path.name, e)
        raise
    except Exception as e:
        logger.error("Error in extract_receipt_json_from_image for %s: %s", file_path.name, e)
        raise


def extract_receipt_json_from_pdf(client: Any, model: str, file_path: Path) -> dict[str, Any]:
    """Upload PDF to OpenAI Files API and extract structured data."""
    logger.debug("extract_receipt_json_from_pdf called: file=%s, model=%s", file_path.name, model)
    
    file_id = None
    try:
        # 1. Upload PDF to Files API
        logger.debug("Uploading PDF to OpenAI Files API: %s", file_path.name)
        with open(file_path, "rb") as pdf_file:
            uploaded_file = client.files.create(
                file=pdf_file,
                purpose="user_data"
            )
            file_id = uploaded_file.id
            logger.debug("PDF uploaded successfully, file_id: %s", file_id)
        
        # 2. Send file_id to model for processing
        logger.debug("Sending PDF to OpenAI via Files API for %s", file_path.name)
        response = client.responses.create(
            model=model,
            instructions=EXTRACTION_INSTRUCTIONS,
            input=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "input_file",
                            "file_id": file_id
                        },
                        {
                            "type": "input_text",
                            "text": build_user_prompt(file_path.name, "pdf")
                        }
                    ]
                }
            ],
            text={
                "format": {
                    "type": "json_schema",
                    "name": "purchase_receipt_extraction",
                    "schema": build_schema()["schema"],
                    "strict": True,
                }
            },
        )
        
        logger.debug("Received response from OpenAI for PDF %s", file_path.name)

        output_text = getattr(response, "output_text", None)
        if output_text:
            logger.debug("Parsing output_text from PDF response")
            result = json.loads(output_text)
            logger.debug("Successfully parsed PDF JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        if response.output and response.output[0].content:
            logger.debug("Parsing response.output content from PDF")
            result = json.loads(response.output[0].content[0].text)
            logger.debug("Successfully parsed PDF JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        logger.error("No JSON returned for PDF %s - no output_text and no response.output", file_path.name)
        raise RuntimeError(f"No JSON returned for {file_path.name}")
    
    except json.JSONDecodeError as e:
        logger.error("JSON decode error for PDF %s: %s", file_path.name, e)
        raise
    except Exception as e:
        logger.error("Error in extract_receipt_json_from_pdf for %s: %s", file_path.name, e)
        raise
    finally:
        # Clean up uploaded file
        if file_id:
            try:
                logger.debug("Deleting uploaded file: %s", file_id)
                client.files.delete(file_id)
            except Exception as e:
                logger.warning("Failed to delete file %s: %s", file_id, e)


def extract_receipt_json_from_document(client: Any, model: str, file_path: Path) -> dict[str, Any]:
    """Send document type to OpenAI by extracting text (DOCX, TXT, etc)."""
    logger.debug("extract_receipt_json_from_document called: file=%s, model=%s", file_path.name, model)
    
    try:
        logger.debug("Extracting text from document: %s", file_path.name)
        
        # Extract text based on file type
        if file_path.suffix.lower() == ".txt":
            text = file_path.read_text(encoding="utf-8", errors="ignore")
            logger.debug("Read TXT file: %s characters", len(text))
        
        elif file_path.suffix.lower() == ".docx":
            from docx import Document
            document = Document(str(file_path))
            paragraphs = [paragraph.text for paragraph in document.paragraphs]
            text = "\n".join(paragraphs)
            logger.debug("Extracted DOCX file: %s characters", len(text))
        
        else:
            logger.warning("Unsupported document format: %s, attempting generic text read", file_path.suffix)
            text = file_path.read_text(encoding="utf-8", errors="ignore")
            logger.debug("Read document file: %s characters", len(text))
        
        # Send extracted text to OpenAI
        logger.debug("Sending extracted document text to OpenAI for %s", file_path.name)
        response = client.responses.create(
            model=model,
            instructions=EXTRACTION_INSTRUCTIONS,
            input=f"{build_user_prompt(file_path.name, 'document')}\n\nDocument text:\n{text[:20000]}",
            text={
                "format": {
                    "type": "json_schema",
                    "name": "purchase_receipt_extraction",
                    "schema": build_schema()["schema"],
                    "strict": True,
                }
            },
        )
        
        logger.debug("Received response from OpenAI for document %s", file_path.name)

        output_text = getattr(response, "output_text", None)
        if output_text:
            logger.debug("Parsing output_text from document response")
            result = json.loads(output_text)
            logger.debug("Successfully parsed document JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        if response.output and response.output[0].content:
            logger.debug("Parsing response.output content from document")
            result = json.loads(response.output[0].content[0].text)
            logger.debug("Successfully parsed document JSON: document_type=%s, total=%s, confidence=%s", 
                        result.get("document_type"), result.get("total"), result.get("confidence"))
            return result

        logger.error("No JSON returned for document %s - no output_text and no response.output", file_path.name)
        raise RuntimeError(f"No JSON returned for {file_path.name}")
    
    except json.JSONDecodeError as e:
        logger.error("JSON decode error for document %s: %s", file_path.name, e)
        raise
    except Exception as e:
        logger.error("Error in extract_receipt_json_from_document for %s: %s", file_path.name, e)
        raise