File size: 7,225 Bytes
a72d1cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 | """Türkçeye özgü metin, sayı, tarih ve birim normalizasyonu.
Üç sayı ayrıştırma kipi desteklenir (ablasyon deneyi için):
- "tr" : doğru Türkçe yorum -> '1.234,56' = 1234.56, '%4,5' = 4.5, '(1.200)' = -1200
- "en" : yaygın naif yaklaşım (İngilizce yerel ayar) -> '.' ondalık, ',' binlik kabul edilir;
'1.234' = 1.234 (1000 kat hata), '4,5' = 45 (10 kat hata), '1.234,56' -> ayrıştırılamaz
- "text" : hiç ayrıştırma yok; hücreler metin olarak saklanır (SQLite SUM('1.234,56') = 1.234 !)
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass
from datetime import date
# ------------------------------------------------------------------ metin
_TR_ASCII = str.maketrans("çğıöşüÇĞİÖŞÜâîûÂÎÛ", "cgiosuCGIOSUaiuAIU")
def tr_lower(s: str) -> str:
"""Türkçe kurallarına uygun küçük harf: 'İ'->'i', 'I'->'ı'."""
if s is None:
return s
return str(s).replace("I", "ı").replace("İ", "i").lower()
def tr_upper(s: str) -> str:
return str(s).replace("i", "İ").replace("ı", "I").upper()
def fold(s: str) -> str:
"""Karşılaştırma için: Türkçe küçük harf + aksan/şapka kaldırma + ASCII'ye katlama.
'İSTANBUL', 'Istanbul', 'istanbul', 'ıstanbul' -> 'istanbul'."""
if s is None:
return ""
s = tr_lower(str(s)).translate(_TR_ASCII)
s = unicodedata.normalize("NFKD", s)
s = "".join(ch for ch in s if not unicodedata.combining(ch))
return re.sub(r"\s+", " ", s).strip()
def slugify(s: str, maxlen: int = 48) -> str:
s = fold(s)
s = s.replace("%", " yuzde ").replace("²", "2").replace("³", "3")
s = re.sub(r"[^a-z0-9]+", "_", s).strip("_")
return s[:maxlen].strip("_") or "sutun"
def clean_cell(s) -> str:
if s is None:
return ""
s = str(s).replace(" ", " ").replace("−", "-").replace("–", "-")
return re.sub(r"\s+", " ", s).strip()
# ------------------------------------------------------------------ sayılar
NULL_TOKENS = {"", "-", "--", "—", "...", "…", "n/a", "na", "yok", "x", ".."}
_RE_TR_THOUSANDS = re.compile(r"^\d{1,3}(\.\d{3})+(,\d+)?$") # 1.234 | 1.234,56
_RE_TR_PLAIN = re.compile(r"^\d+(,\d+)?$") # 1234 | 1234,56
_RE_EN_THOUSANDS = re.compile(r"^\d{1,3}(,\d{3})+(\.\d+)?$") # 1,234.56
_RE_EN_PLAIN = re.compile(r"^\d+\.\d+$") # 3.14
_CURRENCY = re.compile(r"(₺|\bTL\b|\bTRY\b|\bUSD\b|\$|€|\bEUR\b)", re.I)
@dataclass
class ParsedNumber:
value: float | None
is_percent: bool = False
ok: bool = True
def parse_number(raw, mode: str = "tr") -> ParsedNumber:
"""Bir hücreyi sayıya çevirmeyi dener. ok=False ise hücre sayı değildir."""
s = clean_cell(raw)
if fold(s) in NULL_TOKENS:
return ParsedNumber(None, ok=True)
if mode == "text":
return ParsedNumber(None, ok=False)
is_pct = "%" in s
s = s.replace("%", "")
s = _CURRENCY.sub("", s).strip()
neg = False
if s.startswith("(") and s.endswith(")"):
neg, s = True, s[1:-1].strip()
if s.startswith("-") or s.startswith("+"):
neg, s = s.startswith("-"), s[1:].strip()
s = s.replace(" ", "")
if not s or not re.fullmatch(r"[\d.,]+", s):
return ParsedNumber(None, ok=False)
val = None
if mode == "tr":
if _RE_TR_THOUSANDS.match(s) or _RE_TR_PLAIN.match(s):
val = float(s.replace(".", "").replace(",", "."))
elif _RE_EN_THOUSANDS.match(s): # belgede İngilizce biçim varsa yine de tanı
val = float(s.replace(",", ""))
elif _RE_EN_PLAIN.match(s): # '3.5' gibi tek nokta, 3 hane değil -> ondalık
val = float(s)
elif mode == "en": # naif: Python float() mantığı
try:
val = float(s.replace(",", ""))
except ValueError:
val = None
if val is None:
return ParsedNumber(None, ok=False)
return ParsedNumber(-val if neg else val, is_percent=is_pct)
def format_tr(x, dec: int | None = None) -> str:
"""Sonuçları kullanıcıya Türkçe biçimde gösterir."""
if x is None:
return "—"
if isinstance(x, bool):
return str(x)
if isinstance(x, int) or (isinstance(x, float) and x.is_integer() and dec is None):
s = f"{int(x):,}"
return s.replace(",", ".")
if isinstance(x, float):
d = 2 if dec is None else dec
s = f"{x:,.{d}f}"
return s.replace(",", "§").replace(".", ",").replace("§", ".")
return str(x)
# ------------------------------------------------------------------ tarihler
AYLAR = {"ocak": 1, "subat": 2, "mart": 3, "nisan": 4, "mayis": 5, "haziran": 6, "temmuz": 7,
"agustos": 8, "eylul": 9, "ekim": 10, "kasim": 11, "aralik": 12,
"oca": 1, "sub": 2, "mar": 3, "nis": 4, "may": 5, "haz": 6, "tem": 7, "agu": 8,
"eyl": 9, "eki": 10, "kas": 11, "ara": 12}
_RE_DMY = re.compile(r"^(\d{1,2})[./-](\d{1,2})[./-](\d{4})$")
_RE_YMD = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$")
_RE_MON_Y = re.compile(r"^([a-z]+)\s*(\d{4})$")
_RE_Y_Q = re.compile(r"^(\d{4})\s*[/-]?\s*(q|ç|c)\s*([1-4])$")
def parse_date(raw) -> str | None:
"""'31.12.2024' -> '2024-12-31', 'Ocak 2024' -> '2024-01', '2024/Ç1' -> '2024-Q1'."""
s = fold(clean_cell(raw))
if not s:
return None
m = _RE_DMY.match(s)
if m:
d, mo, y = map(int, m.groups())
try:
return date(y, mo, d).isoformat()
except ValueError:
return None
m = _RE_YMD.match(s)
if m:
return s
m = _RE_MON_Y.match(s)
if m and m.group(1) in AYLAR:
return f"{m.group(2)}-{AYLAR[m.group(1)]:02d}"
m = _RE_Y_Q.match(s)
if m:
return f"{m.group(1)}-Q{m.group(3)}"
return None
_RE_DATE_RANGE = re.compile(r"(\d{1,2}[./]\d{1,2}[./]?(\d{4})?)\s*[-–]\s*(\d{1,2}[./]\d{1,2}[./](\d{4}))")
_RE_YEAR = re.compile(r"\b(19\d{2}|20\d{2})\b")
# ------------------------------------------------------------------ birimler
UNIT_PATTERNS = [
(r"\bmilyar\s*(tl|₺)\b", "milyar TL", 1e9),
(r"\bmilyon\s*(tl|₺)\b", "milyon TL", 1e6),
(r"\bbin\s*(tl|₺)\b", "bin TL", 1e3),
(r"\b(tl|₺|türk lirası)\b", "TL", 1.0),
(r"\bbin\s*mwh\b", "bin MWh", 1e3),
(r"\bmwh\b", "MWh", 1.0),
(r"\bgwh\b", "GWh", 1e3),
(r"%|\byuzde\b", "%", 1.0),
(r"\bkisi\b", "kişi", 1.0),
(r"\bton\b", "ton", 1.0),
(r"\bm2\b|m²", "m²", 1.0),
(r"\bm3\b|m³", "m³", 1.0),
]
def detect_unit(text: str) -> tuple[str | None, float]:
"""Başlık veya tablo adı içindeki parantezli birimi bulur. ('bin TL', 1000.0)"""
if not text:
return None, 1.0
parts = re.findall(r"\(([^()]*)\)", text) or [text]
for part in parts:
f = fold(part)
for pat, name, mult in UNIT_PATTERNS:
if re.search(pat, f):
return name, mult
return None, 1.0
def strip_units(text: str) -> str:
"""'Ücret (TL)' -> 'Ücret'; '(-)' gibi işaretleri de kaldırır."""
out = re.sub(r"\((?:[^()]*?)\)", " ", text)
return re.sub(r"\s+", " ", out).strip()
|