File size: 4,257 Bytes
26b8deb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """Deterministic normalisation of extracted monetary and date values."""
from __future__ import annotations
import re
from decimal import Decimal, InvalidOperation
from typing import Optional
import logging
from dateutil import parser as dateutil_parser
from dateutil.parser import ParserError
logger = logging.getLogger(__name__)
# Currency symbol -> ISO 4217
_SYMBOL_MAP = {
"\u20b9": "INR",
"$": "USD",
"\u20ac": "EUR",
"\u00a3": "GBP",
"\u00a5": "JPY",
"\u20a9": "KRW",
"Fr": "CHF",
"A$": "AUD",
"C$": "CAD",
}
_CURRENCY_CODE_RE = re.compile(
r"^(USD|INR|EUR|GBP|JPY|AUD|CAD|CHF|CNY|KRW|SGD|AED|SAR)\s*",
re.IGNORECASE,
)
def detect_currency_from_string(value: str) -> Optional[str]:
"""Detect ISO currency code from a monetary string."""
v = value.strip()
for symbol, code in _SYMBOL_MAP.items():
if symbol in v:
return code
m = _CURRENCY_CODE_RE.match(v)
if m:
return m.group(1).upper()
return None
def normalise_money(value: Optional[str]) -> Optional[Decimal]:
"""
Convert a raw monetary string to Decimal.
Handles: $1,250.00 INR1,25,000.50 EUR 1.250,50 1250 (1250.00)
Returns None if the string cannot be parsed.
"""
if value is None:
return None
v = str(value).strip()
# Remove currency symbols and known currency codes
for symbol in sorted(_SYMBOL_MAP.keys(), key=len, reverse=True):
v = v.replace(symbol, "")
v = _CURRENCY_CODE_RE.sub("", v).strip()
# Remove surrounding parentheses (negative amounts)
negative = False
if v.startswith("(") and v.endswith(")"):
v = v[1:-1]
negative = True
if v.startswith("-"):
v = v[1:]
negative = True
v = v.strip()
# Handle "Rs." / "Rs " abbreviation for Indian Rupee
v = v.replace("Rs.", "").replace("Rs ", "").replace("Rs", "").strip()
v = v.strip()
# Detect European format: e.g. 1.250,50
european_re = re.compile(r"^\d{1,3}(\.\d{3})+(,\d+)?$")
if european_re.match(v):
v = v.replace(".", "").replace(",", ".")
else:
# Standard: remove thousands separators (commas)
v = v.replace(",", "")
try:
d = Decimal(v)
return -d if negative else d
except InvalidOperation:
logger.warning("Could not parse monetary value: %s", value)
return None
def format_decimal(value: Optional[Decimal], decimal_places: int = 2) -> Optional[str]:
"""Format a Decimal to a fixed-precision string."""
if value is None:
return None
return f"{value:.{decimal_places}f}"
def normalise_date(value: Optional[str]) -> Optional[str]:
"""
Normalise a date string to ISO 8601 YYYY-MM-DD format.
Returns None if the date cannot be reliably parsed.
"""
if value is None:
return None
v = str(value).strip()
if not v:
return None
# Already ISO
iso_re = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$")
m = iso_re.match(v)
if m:
y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
if 1900 <= y <= 2100 and 1 <= mo <= 12 and 1 <= d <= 31:
return v
# dd/mm/yyyy or dd-mm-yyyy (common Indian format)
dmy_re = re.compile(r"^(\d{1,2})[-/](\d{1,2})[-/](\d{4})$")
m = dmy_re.match(v)
if m:
day, month, year = int(m.group(1)), int(m.group(2)), int(m.group(3))
if 1 <= month <= 12 and 1 <= day <= 31 and 1900 <= year <= 2100:
return f"{year:04d}-{month:02d}-{day:02d}"
# Textual dates: "18 August 2026", "Aug 18, 2026"
try:
parsed = dateutil_parser.parse(v, dayfirst=True)
year = parsed.year
if 1900 <= year <= 2100:
return parsed.strftime("%Y-%m-%d")
except (ParserError, OverflowError, ValueError):
pass
logger.warning("Could not parse date: %s", value)
return None
def normalise_quantity(value: Optional[str]) -> Optional[Decimal]:
"""Parse a quantity string to Decimal."""
if value is None:
return None
v = str(value).strip().replace(",", "")
try:
return Decimal(v)
except InvalidOperation:
logger.warning("Could not parse quantity: %s", value)
return None
|