DocuPurse / src /normalizer.py
Zayeemk's picture
Initial commit: DocuParse AI complete implementation
26b8deb
Raw History Blame Contribute Delete
4.26 kB
"""Deterministic normalisation of extracted monetary and date values."""
from __future__ import annotations
import re
from decimal import Decimal, InvalidOperation
from typing import Optional
import logging
from dateutil import parser as dateutil_parser
from dateutil.parser import ParserError
logger = logging.getLogger(__name__)
# Currency symbol -> ISO 4217
_SYMBOL_MAP = {
"\u20b9": "INR",
"$": "USD",
"\u20ac": "EUR",
"\u00a3": "GBP",
"\u00a5": "JPY",
"\u20a9": "KRW",
"Fr": "CHF",
"A$": "AUD",
"C$": "CAD",
}
_CURRENCY_CODE_RE = re.compile(
r"^(USD|INR|EUR|GBP|JPY|AUD|CAD|CHF|CNY|KRW|SGD|AED|SAR)\s*",
re.IGNORECASE,
)
def detect_currency_from_string(value: str) -> Optional[str]:
"""Detect ISO currency code from a monetary string."""
v = value.strip()
for symbol, code in _SYMBOL_MAP.items():
if symbol in v:
return code
m = _CURRENCY_CODE_RE.match(v)
if m:
return m.group(1).upper()
return None
def normalise_money(value: Optional[str]) -> Optional[Decimal]:
"""
Convert a raw monetary string to Decimal.
Handles: $1,250.00 INR1,25,000.50 EUR 1.250,50 1250 (1250.00)
Returns None if the string cannot be parsed.
"""
if value is None:
return None
v = str(value).strip()
# Remove currency symbols and known currency codes
for symbol in sorted(_SYMBOL_MAP.keys(), key=len, reverse=True):
v = v.replace(symbol, "")
v = _CURRENCY_CODE_RE.sub("", v).strip()
# Remove surrounding parentheses (negative amounts)
negative = False
if v.startswith("(") and v.endswith(")"):
v = v[1:-1]
negative = True
if v.startswith("-"):
v = v[1:]
negative = True
v = v.strip()
# Handle "Rs." / "Rs " abbreviation for Indian Rupee
v = v.replace("Rs.", "").replace("Rs ", "").replace("Rs", "").strip()
v = v.strip()
# Detect European format: e.g. 1.250,50
european_re = re.compile(r"^\d{1,3}(\.\d{3})+(,\d+)?$")
if european_re.match(v):
v = v.replace(".", "").replace(",", ".")
else:
# Standard: remove thousands separators (commas)
v = v.replace(",", "")
try:
d = Decimal(v)
return -d if negative else d
except InvalidOperation:
logger.warning("Could not parse monetary value: %s", value)
return None
def format_decimal(value: Optional[Decimal], decimal_places: int = 2) -> Optional[str]:
"""Format a Decimal to a fixed-precision string."""
if value is None:
return None
return f"{value:.{decimal_places}f}"
def normalise_date(value: Optional[str]) -> Optional[str]:
"""
Normalise a date string to ISO 8601 YYYY-MM-DD format.
Returns None if the date cannot be reliably parsed.
"""
if value is None:
return None
v = str(value).strip()
if not v:
return None
# Already ISO
iso_re = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$")
m = iso_re.match(v)
if m:
y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
if 1900 <= y <= 2100 and 1 <= mo <= 12 and 1 <= d <= 31:
return v
# dd/mm/yyyy or dd-mm-yyyy (common Indian format)
dmy_re = re.compile(r"^(\d{1,2})[-/](\d{1,2})[-/](\d{4})$")
m = dmy_re.match(v)
if m:
day, month, year = int(m.group(1)), int(m.group(2)), int(m.group(3))
if 1 <= month <= 12 and 1 <= day <= 31 and 1900 <= year <= 2100:
return f"{year:04d}-{month:02d}-{day:02d}"
# Textual dates: "18 August 2026", "Aug 18, 2026"
try:
parsed = dateutil_parser.parse(v, dayfirst=True)
year = parsed.year
if 1900 <= year <= 2100:
return parsed.strftime("%Y-%m-%d")
except (ParserError, OverflowError, ValueError):
pass
logger.warning("Could not parse date: %s", value)
return None
def normalise_quantity(value: Optional[str]) -> Optional[Decimal]:
"""Parse a quantity string to Decimal."""
if value is None:
return None
v = str(value).strip().replace(",", "")
try:
return Decimal(v)
except InvalidOperation:
logger.warning("Could not parse quantity: %s", value)
return None