DocuPurse / src /confidence.py
Zayeemk's picture
Initial commit: DocuParse AI complete implementation
26b8deb
Raw History Blame Contribute Delete
3.78 kB
"""Confidence scoring for DocuParse AI extractions.
Confidence is calculated from transparent heuristics - NOT from LLM self-reported confidence.
Signals:
- Field extraction completeness
- Document classification confidence
- Pydantic validation success
- Arithmetic reconciliation results
- Date validation success
"""
from __future__ import annotations
from typing import Dict, Tuple
from .schemas import DocumentExtraction
_FIELD_WEIGHTS: Dict[str, float] = {
"document_id": 0.10,
"vendor": 0.10,
"customer": 0.05,
"document_date": 0.10,
"due_date": 0.05,
"currency": 0.05,
"subtotal": 0.08,
"tax": 0.07,
"total": 0.10,
"line_items": 0.15,
"payment_method": 0.05,
"discount": 0.03,
"shipping": 0.03,
"purchase_order_number": 0.04,
}
_BONUS_RECONCILIATION_PASS = 0.10
_PENALTY_RECONCILIATION_FAIL = 0.15
_BONUS_DATE_VALID = 0.05
_PENALTY_DATE_INVALID = 0.05
def _field_present(extraction: DocumentExtraction, field: str) -> bool:
if field == "line_items":
return len(extraction.line_items) > 0
value = getattr(extraction, field, None)
return value is not None and str(value).strip() != ""
def _field_confidence(extraction: DocumentExtraction, field: str) -> float:
"""Individual field confidence score."""
if not _field_present(extraction, field):
return 0.0
base = 0.80
if field in ("subtotal", "tax", "total", "discount", "shipping"):
decimal_attr = f"{field}_decimal"
if getattr(extraction, decimal_attr, None) is not None:
base = 0.95
elif field == "document_date":
if extraction.document_date:
from .normalizer import normalise_date
if normalise_date(extraction.document_date):
base = 0.99
elif field == "line_items":
items = extraction.line_items
all_have_total = all(item.line_total_decimal is not None for item in items)
base = 0.95 if all_have_total else 0.80
return base
def calculate_confidence(
extraction: DocumentExtraction,
classification_confidence: float,
) -> Tuple[float, Dict[str, float], str]:
"""
Calculate overall and per-field confidence.
Returns (overall, field_scores, explanation).
"""
field_scores: Dict[str, float] = {}
weighted_sum = 0.0
total_weight = sum(_FIELD_WEIGHTS.values())
for field, weight in _FIELD_WEIGHTS.items():
score = _field_confidence(extraction, field)
field_scores[field] = score
weighted_sum += score * weight
base_confidence = weighted_sum / total_weight
base_confidence = base_confidence * 0.8 + classification_confidence * 0.2
total_check = next(
(c for c in extraction.validation.checks if c.name == "total_reconciliation"),
None,
)
if total_check is not None:
if total_check.passed:
base_confidence += _BONUS_RECONCILIATION_PASS
else:
base_confidence -= _PENALTY_RECONCILIATION_FAIL
date_check = next(
(c for c in extraction.validation.checks if c.name == "date_validation"),
None,
)
if date_check is not None:
if date_check.passed:
base_confidence += _BONUS_DATE_VALID
else:
base_confidence -= _PENALTY_DATE_INVALID
overall = max(0.0, min(1.0, base_confidence))
fields_extracted = len([f for f in _FIELD_WEIGHTS if _field_present(extraction, f)])
explanation = (
f"Confidence calculated from: field completeness ({fields_extracted}/"
f"{len(_FIELD_WEIGHTS)} fields extracted), classification confidence "
f"({classification_confidence:.0%}), and deterministic validation results."
)
return overall, field_scores, explanation