"""Main orchestration agent for DocuParse AI. Pipeline: 1. File validation 2. Text extraction (PDF / OCR / TXT) 3. Document classification 4. LLM semantic extraction (Groq) 5. Pydantic schema validation 6. Normalisation (dates, money, quantities) 7. Deterministic validation 8. Confidence calculation 9. Structured output assembly Design principle: LLM = semantic extraction Python = deterministic verification """ from __future__ import annotations import logging import time from pathlib import Path from typing import Any, Callable, Dict, List, Optional from .classifier import classify_document from .confidence import calculate_confidence from .extraction import extract_text_from_file from .llm_provider import get_provider from .normalizer import ( detect_currency_from_string, format_decimal, normalise_date, normalise_money, normalise_quantity, ) from .schemas import ( ConfidenceScore, DocumentExtraction, ExtractionMetadata, FieldProvenance, LineItem, LLMRawExtraction, ) from .utils import pretty_json, truncate_text, validate_file from .validator import run_all_validations logger = logging.getLogger(__name__) StatusCallback = Callable[[str], None] def _no_op(msg: str) -> None: pass def _parse_llm_line_items(raw_items: List[Dict]) -> List[LineItem]: """Parse raw LLM line item dicts into LineItem objects with normalised decimals.""" items = [] for raw in raw_items: if not isinstance(raw, dict): continue item = LineItem( item_id=raw.get("item_id"), description=str(raw.get("description") or ""), quantity=raw.get("quantity"), unit_price=raw.get("unit_price"), tax=raw.get("tax"), line_total=raw.get("line_total"), ) item.quantity_decimal = normalise_quantity(item.quantity) item.unit_price_decimal = normalise_money(item.unit_price) item.tax_decimal = normalise_money(item.tax) item.line_total_decimal = normalise_money(item.line_total) items.append(item) return items def _build_provenance( raw: LLMRawExtraction, source_text: str, ) -> List[FieldProvenance]: """Build field provenance records for key fields.""" provenance_fields = [ ("document_id", raw.document_id), ("vendor", raw.vendor), ("document_date", raw.document_date), ("subtotal", raw.subtotal), ("tax", raw.tax), ("total", raw.total), ] records = [] for field, value in provenance_fields: if value is None: records.append(FieldProvenance(field=field, value=None, confidence=0.0)) continue search_str = str(value) idx = source_text.find(search_str) if idx != -1: start = max(0, idx - 30) end = min(len(source_text), idx + len(search_str) + 30) snippet = source_text[start:end].strip().replace("\n", " ") records.append(FieldProvenance( field=field, value=value, source_text=snippet, confidence=0.95, )) else: records.append(FieldProvenance( field=field, value=value, source_text=None, confidence=0.70, )) return records def process_document( file_path: str, status_callback: StatusCallback = _no_op, ) -> Dict[str, Any]: """ Main pipeline: process a document file and return structured extraction result. Args: file_path: Path to document file (PDF, image, or TXT) status_callback: Optional callable for progress updates Returns: Dictionary with full structured extraction result """ start_time = time.time() filename = Path(file_path).name # Step 1: Validate file status_callback("Checking document...") try: validate_file(file_path) except ValueError as e: return _error_response(str(e), filename) # Step 2: Extract text status_callback("Extracting text from document...") try: document_text, extraction_method, page_count = extract_text_from_file(file_path) except Exception as e: logger.exception("Text extraction failed") return _error_response(f"Text extraction failed: {e}", filename) status_callback(f"Text extracted ({extraction_method}, {len(document_text)} chars)") # Step 3: Classify document status_callback("Classifying document type...") doc_type, classification_confidence = classify_document(document_text) status_callback(f"Classified as '{doc_type}' (confidence: {classification_confidence:.0%})") # Step 4: LLM extraction status_callback("Running AI field extraction...") try: provider = get_provider() truncated_text = truncate_text(document_text, max_chars=8000) raw_dict = provider.extract_document(truncated_text, doc_type) except ValueError as e: return _error_response(str(e), filename) except Exception as e: logger.exception("LLM extraction error") return _error_response(f"AI extraction failed: {e}", filename) # Step 5: Parse into Pydantic schema status_callback("Validating extracted schema...") try: raw = LLMRawExtraction(**raw_dict) except Exception as e: logger.warning("Pydantic parsing failed: %s. Attempting recovery.", e) try: allowed_keys = set(LLMRawExtraction.model_fields.keys()) safe_dict = {k: v for k, v in raw_dict.items() if k in allowed_keys} raw = LLMRawExtraction(**safe_dict) except Exception as e2: return _error_response(f"Schema validation failed: {e2}", filename) if raw.document_type in (None, "unknown") and doc_type != "unknown": raw.document_type = doc_type # Step 6: Normalise status_callback("Normalising extracted data...") detected_currency = raw.currency if not detected_currency: for val in [raw.total, raw.subtotal, raw.tax]: if val: detected_currency = detect_currency_from_string(str(val)) if detected_currency: break subtotal_decimal = normalise_money(raw.subtotal) tax_decimal = normalise_money(raw.tax) discount_decimal = normalise_money(raw.discount) shipping_decimal = normalise_money(raw.shipping) total_decimal = normalise_money(raw.total) norm_doc_date = normalise_date(raw.document_date) norm_due_date = normalise_date(raw.due_date) line_items = _parse_llm_line_items(raw.line_items) provenance = _build_provenance(raw, document_text) # Step 7: Assemble extraction object extraction = DocumentExtraction( document_type=raw.document_type or "unknown", document_id=raw.document_id, vendor=raw.vendor, customer=raw.customer, document_date=norm_doc_date, due_date=norm_due_date, currency=detected_currency, subtotal=format_decimal(subtotal_decimal), tax=format_decimal(tax_decimal), discount=format_decimal(discount_decimal), shipping=format_decimal(shipping_decimal), total=format_decimal(total_decimal), payment_method=raw.payment_method, purchase_order_number=raw.purchase_order_number, line_items=line_items, provenance=provenance, subtotal_decimal=subtotal_decimal, tax_decimal=tax_decimal, discount_decimal=discount_decimal, shipping_decimal=shipping_decimal, total_decimal=total_decimal, ) # Step 8: Deterministic validation status_callback("Running deterministic validation...") extraction.validation = run_all_validations(extraction) status_callback(f"Validation complete: {extraction.validation.overall_status}") # Step 9: Confidence scoring overall_conf, field_conf, conf_explanation = calculate_confidence( extraction, classification_confidence ) extraction.confidence = ConfidenceScore( overall=overall_conf, fields=field_conf, explanation=conf_explanation, ) # Step 10: Metadata elapsed = time.time() - start_time extraction.metadata = ExtractionMetadata( source_filename=filename, extraction_method=extraction_method, page_count=page_count, processing_time_seconds=elapsed, document_type_confidence=classification_confidence, model_used=provider.model if hasattr(provider, "model") else "unknown", raw_confidence_notes=raw.raw_confidence_notes or "", ) status_callback("JSON generated successfully") return extraction.to_output_dict() def _error_response(message: str, filename: str = "") -> Dict[str, Any]: """Return a structured error response.""" logger.error("Processing error: %s", message) return { "error": True, "error_message": message, "document_type": "unknown", "document_id": None, "vendor": None, "customer": None, "document_date": None, "due_date": None, "currency": None, "subtotal": None, "tax": None, "discount": None, "shipping": None, "total": None, "payment_method": None, "purchase_order_number": None, "line_items": [], "validation": { "is_valid": False, "overall_status": "INVALID", "checks": [], "warnings": [], "errors": [message], }, "confidence": {"overall": 0.0, "fields": {}, "explanation": ""}, "metadata": { "source_filename": filename, "extraction_method": "none", "page_count": 0, "processing_time_seconds": 0.0, "document_type_confidence": 0.0, "model_used": "none", }, "provenance": [], }