Download src/agent.py from Zayeemk/DocuPurse: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/Zayeemk/DocuPurse/resolve/main/src/agent.py
- Command line
-
hf download hf://spaces/Zayeemk/DocuPurse/src/agent.py
-
curl -L -o agent.py https://huggingface.co/spaces/Zayeemk/DocuPurse/resolve/main/src/agent.py
10 kB
| """Main orchestration agent for DocuParse AI. | |
| Pipeline: | |
| 1. File validation | |
| 2. Text extraction (PDF / OCR / TXT) | |
| 3. Document classification | |
| 4. LLM semantic extraction (Groq) | |
| 5. Pydantic schema validation | |
| 6. Normalisation (dates, money, quantities) | |
| 7. Deterministic validation | |
| 8. Confidence calculation | |
| 9. Structured output assembly | |
| Design principle: | |
| LLM = semantic extraction | |
| Python = deterministic verification | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import time | |
| from pathlib import Path | |
| from typing import Any, Callable, Dict, List, Optional | |
| from .classifier import classify_document | |
| from .confidence import calculate_confidence | |
| from .extraction import extract_text_from_file | |
| from .llm_provider import get_provider | |
| from .normalizer import ( | |
| detect_currency_from_string, | |
| format_decimal, | |
| normalise_date, | |
| normalise_money, | |
| normalise_quantity, | |
| ) | |
| from .schemas import ( | |
| ConfidenceScore, | |
| DocumentExtraction, | |
| ExtractionMetadata, | |
| FieldProvenance, | |
| LineItem, | |
| LLMRawExtraction, | |
| ) | |
| from .utils import pretty_json, truncate_text, validate_file | |
| from .validator import run_all_validations | |
| logger = logging.getLogger(__name__) | |
| StatusCallback = Callable[[str], None] | |
| def _no_op(msg: str) -> None: | |
| pass | |
| def _parse_llm_line_items(raw_items: List[Dict]) -> List[LineItem]: | |
| """Parse raw LLM line item dicts into LineItem objects with normalised decimals.""" | |
| items = [] | |
| for raw in raw_items: | |
| if not isinstance(raw, dict): | |
| continue | |
| item = LineItem( | |
| item_id=raw.get("item_id"), | |
| description=str(raw.get("description") or ""), | |
| quantity=raw.get("quantity"), | |
| unit_price=raw.get("unit_price"), | |
| tax=raw.get("tax"), | |
| line_total=raw.get("line_total"), | |
| ) | |
| item.quantity_decimal = normalise_quantity(item.quantity) | |
| item.unit_price_decimal = normalise_money(item.unit_price) | |
| item.tax_decimal = normalise_money(item.tax) | |
| item.line_total_decimal = normalise_money(item.line_total) | |
| items.append(item) | |
| return items | |
| def _build_provenance( | |
| raw: LLMRawExtraction, | |
| source_text: str, | |
| ) -> List[FieldProvenance]: | |
| """Build field provenance records for key fields.""" | |
| provenance_fields = [ | |
| ("document_id", raw.document_id), | |
| ("vendor", raw.vendor), | |
| ("document_date", raw.document_date), | |
| ("subtotal", raw.subtotal), | |
| ("tax", raw.tax), | |
| ("total", raw.total), | |
| ] | |
| records = [] | |
| for field, value in provenance_fields: | |
| if value is None: | |
| records.append(FieldProvenance(field=field, value=None, confidence=0.0)) | |
| continue | |
| search_str = str(value) | |
| idx = source_text.find(search_str) | |
| if idx != -1: | |
| start = max(0, idx - 30) | |
| end = min(len(source_text), idx + len(search_str) + 30) | |
| snippet = source_text[start:end].strip().replace("\n", " ") | |
| records.append(FieldProvenance( | |
| field=field, | |
| value=value, | |
| source_text=snippet, | |
| confidence=0.95, | |
| )) | |
| else: | |
| records.append(FieldProvenance( | |
| field=field, | |
| value=value, | |
| source_text=None, | |
| confidence=0.70, | |
| )) | |
| return records | |
| def process_document( | |
| file_path: str, | |
| status_callback: StatusCallback = _no_op, | |
| ) -> Dict[str, Any]: | |
| """ | |
| Main pipeline: process a document file and return structured extraction result. | |
| Args: | |
| file_path: Path to document file (PDF, image, or TXT) | |
| status_callback: Optional callable for progress updates | |
| Returns: | |
| Dictionary with full structured extraction result | |
| """ | |
| start_time = time.time() | |
| filename = Path(file_path).name | |
| # Step 1: Validate file | |
| status_callback("Checking document...") | |
| try: | |
| validate_file(file_path) | |
| except ValueError as e: | |
| return _error_response(str(e), filename) | |
| # Step 2: Extract text | |
| status_callback("Extracting text from document...") | |
| try: | |
| document_text, extraction_method, page_count = extract_text_from_file(file_path) | |
| except Exception as e: | |
| logger.exception("Text extraction failed") | |
| return _error_response(f"Text extraction failed: {e}", filename) | |
| status_callback(f"Text extracted ({extraction_method}, {len(document_text)} chars)") | |
| # Step 3: Classify document | |
| status_callback("Classifying document type...") | |
| doc_type, classification_confidence = classify_document(document_text) | |
| status_callback(f"Classified as '{doc_type}' (confidence: {classification_confidence:.0%})") | |
| # Step 4: LLM extraction | |
| status_callback("Running AI field extraction...") | |
| try: | |
| provider = get_provider() | |
| truncated_text = truncate_text(document_text, max_chars=8000) | |
| raw_dict = provider.extract_document(truncated_text, doc_type) | |
| except ValueError as e: | |
| return _error_response(str(e), filename) | |
| except Exception as e: | |
| logger.exception("LLM extraction error") | |
| return _error_response(f"AI extraction failed: {e}", filename) | |
| # Step 5: Parse into Pydantic schema | |
| status_callback("Validating extracted schema...") | |
| try: | |
| raw = LLMRawExtraction(**raw_dict) | |
| except Exception as e: | |
| logger.warning("Pydantic parsing failed: %s. Attempting recovery.", e) | |
| try: | |
| allowed_keys = set(LLMRawExtraction.model_fields.keys()) | |
| safe_dict = {k: v for k, v in raw_dict.items() if k in allowed_keys} | |
| raw = LLMRawExtraction(**safe_dict) | |
| except Exception as e2: | |
| return _error_response(f"Schema validation failed: {e2}", filename) | |
| if raw.document_type in (None, "unknown") and doc_type != "unknown": | |
| raw.document_type = doc_type | |
| # Step 6: Normalise | |
| status_callback("Normalising extracted data...") | |
| detected_currency = raw.currency | |
| if not detected_currency: | |
| for val in [raw.total, raw.subtotal, raw.tax]: | |
| if val: | |
| detected_currency = detect_currency_from_string(str(val)) | |
| if detected_currency: | |
| break | |
| subtotal_decimal = normalise_money(raw.subtotal) | |
| tax_decimal = normalise_money(raw.tax) | |
| discount_decimal = normalise_money(raw.discount) | |
| shipping_decimal = normalise_money(raw.shipping) | |
| total_decimal = normalise_money(raw.total) | |
| norm_doc_date = normalise_date(raw.document_date) | |
| norm_due_date = normalise_date(raw.due_date) | |
| line_items = _parse_llm_line_items(raw.line_items) | |
| provenance = _build_provenance(raw, document_text) | |
| # Step 7: Assemble extraction object | |
| extraction = DocumentExtraction( | |
| document_type=raw.document_type or "unknown", | |
| document_id=raw.document_id, | |
| vendor=raw.vendor, | |
| customer=raw.customer, | |
| document_date=norm_doc_date, | |
| due_date=norm_due_date, | |
| currency=detected_currency, | |
| subtotal=format_decimal(subtotal_decimal), | |
| tax=format_decimal(tax_decimal), | |
| discount=format_decimal(discount_decimal), | |
| shipping=format_decimal(shipping_decimal), | |
| total=format_decimal(total_decimal), | |
| payment_method=raw.payment_method, | |
| purchase_order_number=raw.purchase_order_number, | |
| line_items=line_items, | |
| provenance=provenance, | |
| subtotal_decimal=subtotal_decimal, | |
| tax_decimal=tax_decimal, | |
| discount_decimal=discount_decimal, | |
| shipping_decimal=shipping_decimal, | |
| total_decimal=total_decimal, | |
| ) | |
| # Step 8: Deterministic validation | |
| status_callback("Running deterministic validation...") | |
| extraction.validation = run_all_validations(extraction) | |
| status_callback(f"Validation complete: {extraction.validation.overall_status}") | |
| # Step 9: Confidence scoring | |
| overall_conf, field_conf, conf_explanation = calculate_confidence( | |
| extraction, classification_confidence | |
| ) | |
| extraction.confidence = ConfidenceScore( | |
| overall=overall_conf, | |
| fields=field_conf, | |
| explanation=conf_explanation, | |
| ) | |
| # Step 10: Metadata | |
| elapsed = time.time() - start_time | |
| extraction.metadata = ExtractionMetadata( | |
| source_filename=filename, | |
| extraction_method=extraction_method, | |
| page_count=page_count, | |
| processing_time_seconds=elapsed, | |
| document_type_confidence=classification_confidence, | |
| model_used=provider.model if hasattr(provider, "model") else "unknown", | |
| raw_confidence_notes=raw.raw_confidence_notes or "", | |
| ) | |
| status_callback("JSON generated successfully") | |
| return extraction.to_output_dict() | |
| def _error_response(message: str, filename: str = "") -> Dict[str, Any]: | |
| """Return a structured error response.""" | |
| logger.error("Processing error: %s", message) | |
| return { | |
| "error": True, | |
| "error_message": message, | |
| "document_type": "unknown", | |
| "document_id": None, | |
| "vendor": None, | |
| "customer": None, | |
| "document_date": None, | |
| "due_date": None, | |
| "currency": None, | |
| "subtotal": None, | |
| "tax": None, | |
| "discount": None, | |
| "shipping": None, | |
| "total": None, | |
| "payment_method": None, | |
| "purchase_order_number": None, | |
| "line_items": [], | |
| "validation": { | |
| "is_valid": False, | |
| "overall_status": "INVALID", | |
| "checks": [], | |
| "warnings": [], | |
| "errors": [message], | |
| }, | |
| "confidence": {"overall": 0.0, "fields": {}, "explanation": ""}, | |
| "metadata": { | |
| "source_filename": filename, | |
| "extraction_method": "none", | |
| "page_count": 0, | |
| "processing_time_seconds": 0.0, | |
| "document_type_confidence": 0.0, | |
| "model_used": "none", | |
| }, | |
| "provenance": [], | |
| } | |