DocuPurse / src /agent.py
Zayeemk's picture
Initial commit: DocuParse AI complete implementation
26b8deb
Raw History Blame Contribute Delete
10 kB
"""Main orchestration agent for DocuParse AI.
Pipeline:
1. File validation
2. Text extraction (PDF / OCR / TXT)
3. Document classification
4. LLM semantic extraction (Groq)
5. Pydantic schema validation
6. Normalisation (dates, money, quantities)
7. Deterministic validation
8. Confidence calculation
9. Structured output assembly
Design principle:
LLM = semantic extraction
Python = deterministic verification
"""
from __future__ import annotations
import logging
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from .classifier import classify_document
from .confidence import calculate_confidence
from .extraction import extract_text_from_file
from .llm_provider import get_provider
from .normalizer import (
detect_currency_from_string,
format_decimal,
normalise_date,
normalise_money,
normalise_quantity,
)
from .schemas import (
ConfidenceScore,
DocumentExtraction,
ExtractionMetadata,
FieldProvenance,
LineItem,
LLMRawExtraction,
)
from .utils import pretty_json, truncate_text, validate_file
from .validator import run_all_validations
logger = logging.getLogger(__name__)
StatusCallback = Callable[[str], None]
def _no_op(msg: str) -> None:
pass
def _parse_llm_line_items(raw_items: List[Dict]) -> List[LineItem]:
"""Parse raw LLM line item dicts into LineItem objects with normalised decimals."""
items = []
for raw in raw_items:
if not isinstance(raw, dict):
continue
item = LineItem(
item_id=raw.get("item_id"),
description=str(raw.get("description") or ""),
quantity=raw.get("quantity"),
unit_price=raw.get("unit_price"),
tax=raw.get("tax"),
line_total=raw.get("line_total"),
)
item.quantity_decimal = normalise_quantity(item.quantity)
item.unit_price_decimal = normalise_money(item.unit_price)
item.tax_decimal = normalise_money(item.tax)
item.line_total_decimal = normalise_money(item.line_total)
items.append(item)
return items
def _build_provenance(
raw: LLMRawExtraction,
source_text: str,
) -> List[FieldProvenance]:
"""Build field provenance records for key fields."""
provenance_fields = [
("document_id", raw.document_id),
("vendor", raw.vendor),
("document_date", raw.document_date),
("subtotal", raw.subtotal),
("tax", raw.tax),
("total", raw.total),
]
records = []
for field, value in provenance_fields:
if value is None:
records.append(FieldProvenance(field=field, value=None, confidence=0.0))
continue
search_str = str(value)
idx = source_text.find(search_str)
if idx != -1:
start = max(0, idx - 30)
end = min(len(source_text), idx + len(search_str) + 30)
snippet = source_text[start:end].strip().replace("\n", " ")
records.append(FieldProvenance(
field=field,
value=value,
source_text=snippet,
confidence=0.95,
))
else:
records.append(FieldProvenance(
field=field,
value=value,
source_text=None,
confidence=0.70,
))
return records
def process_document(
file_path: str,
status_callback: StatusCallback = _no_op,
) -> Dict[str, Any]:
"""
Main pipeline: process a document file and return structured extraction result.
Args:
file_path: Path to document file (PDF, image, or TXT)
status_callback: Optional callable for progress updates
Returns:
Dictionary with full structured extraction result
"""
start_time = time.time()
filename = Path(file_path).name
# Step 1: Validate file
status_callback("Checking document...")
try:
validate_file(file_path)
except ValueError as e:
return _error_response(str(e), filename)
# Step 2: Extract text
status_callback("Extracting text from document...")
try:
document_text, extraction_method, page_count = extract_text_from_file(file_path)
except Exception as e:
logger.exception("Text extraction failed")
return _error_response(f"Text extraction failed: {e}", filename)
status_callback(f"Text extracted ({extraction_method}, {len(document_text)} chars)")
# Step 3: Classify document
status_callback("Classifying document type...")
doc_type, classification_confidence = classify_document(document_text)
status_callback(f"Classified as '{doc_type}' (confidence: {classification_confidence:.0%})")
# Step 4: LLM extraction
status_callback("Running AI field extraction...")
try:
provider = get_provider()
truncated_text = truncate_text(document_text, max_chars=8000)
raw_dict = provider.extract_document(truncated_text, doc_type)
except ValueError as e:
return _error_response(str(e), filename)
except Exception as e:
logger.exception("LLM extraction error")
return _error_response(f"AI extraction failed: {e}", filename)
# Step 5: Parse into Pydantic schema
status_callback("Validating extracted schema...")
try:
raw = LLMRawExtraction(**raw_dict)
except Exception as e:
logger.warning("Pydantic parsing failed: %s. Attempting recovery.", e)
try:
allowed_keys = set(LLMRawExtraction.model_fields.keys())
safe_dict = {k: v for k, v in raw_dict.items() if k in allowed_keys}
raw = LLMRawExtraction(**safe_dict)
except Exception as e2:
return _error_response(f"Schema validation failed: {e2}", filename)
if raw.document_type in (None, "unknown") and doc_type != "unknown":
raw.document_type = doc_type
# Step 6: Normalise
status_callback("Normalising extracted data...")
detected_currency = raw.currency
if not detected_currency:
for val in [raw.total, raw.subtotal, raw.tax]:
if val:
detected_currency = detect_currency_from_string(str(val))
if detected_currency:
break
subtotal_decimal = normalise_money(raw.subtotal)
tax_decimal = normalise_money(raw.tax)
discount_decimal = normalise_money(raw.discount)
shipping_decimal = normalise_money(raw.shipping)
total_decimal = normalise_money(raw.total)
norm_doc_date = normalise_date(raw.document_date)
norm_due_date = normalise_date(raw.due_date)
line_items = _parse_llm_line_items(raw.line_items)
provenance = _build_provenance(raw, document_text)
# Step 7: Assemble extraction object
extraction = DocumentExtraction(
document_type=raw.document_type or "unknown",
document_id=raw.document_id,
vendor=raw.vendor,
customer=raw.customer,
document_date=norm_doc_date,
due_date=norm_due_date,
currency=detected_currency,
subtotal=format_decimal(subtotal_decimal),
tax=format_decimal(tax_decimal),
discount=format_decimal(discount_decimal),
shipping=format_decimal(shipping_decimal),
total=format_decimal(total_decimal),
payment_method=raw.payment_method,
purchase_order_number=raw.purchase_order_number,
line_items=line_items,
provenance=provenance,
subtotal_decimal=subtotal_decimal,
tax_decimal=tax_decimal,
discount_decimal=discount_decimal,
shipping_decimal=shipping_decimal,
total_decimal=total_decimal,
)
# Step 8: Deterministic validation
status_callback("Running deterministic validation...")
extraction.validation = run_all_validations(extraction)
status_callback(f"Validation complete: {extraction.validation.overall_status}")
# Step 9: Confidence scoring
overall_conf, field_conf, conf_explanation = calculate_confidence(
extraction, classification_confidence
)
extraction.confidence = ConfidenceScore(
overall=overall_conf,
fields=field_conf,
explanation=conf_explanation,
)
# Step 10: Metadata
elapsed = time.time() - start_time
extraction.metadata = ExtractionMetadata(
source_filename=filename,
extraction_method=extraction_method,
page_count=page_count,
processing_time_seconds=elapsed,
document_type_confidence=classification_confidence,
model_used=provider.model if hasattr(provider, "model") else "unknown",
raw_confidence_notes=raw.raw_confidence_notes or "",
)
status_callback("JSON generated successfully")
return extraction.to_output_dict()
def _error_response(message: str, filename: str = "") -> Dict[str, Any]:
"""Return a structured error response."""
logger.error("Processing error: %s", message)
return {
"error": True,
"error_message": message,
"document_type": "unknown",
"document_id": None,
"vendor": None,
"customer": None,
"document_date": None,
"due_date": None,
"currency": None,
"subtotal": None,
"tax": None,
"discount": None,
"shipping": None,
"total": None,
"payment_method": None,
"purchase_order_number": None,
"line_items": [],
"validation": {
"is_valid": False,
"overall_status": "INVALID",
"checks": [],
"warnings": [],
"errors": [message],
},
"confidence": {"overall": 0.0, "fields": {}, "explanation": ""},
"metadata": {
"source_filename": filename,
"extraction_method": "none",
"page_count": 0,
"processing_time_seconds": 0.0,
"document_type_confidence": 0.0,
"model_used": "none",
},
"provenance": [],
}