Mach-Boom / frontend_en.py
maniac-11111's picture
Mach Boom
01860b0
Raw History Blame Contribute Delete
19.1 kB
"""English text front end: numbers, dates, times, money, units, codes and abbreviations as words."""
import re
ONES = "zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen " \
"seventeen eighteen nineteen".split()
TENS = "_ _ twenty thirty forty fifty sixty seventy eighty ninety".split()
ORD_IRREG = {"one": "first", "two": "second", "three": "third", "five": "fifth", "eight": "eighth",
"nine": "ninth", "twelve": "twelfth"}
MONTHS = "January February March April May June July August September October November December".split()
DIGIT = {str(i): ONES[i] for i in range(10)}
FRAC = {(1, 2): "one half", (1, 3): "one third", (2, 3): "two thirds", (1, 4): "one quarter", (3, 4): "three quarters",
(1, 5): "one fifth", (1, 8): "one eighth", (1, 10): "one tenth"}
def cardinal(n):
n = int(n)
if n < 0:
return "minus " + cardinal(-n)
if n < 20:
return ONES[n]
if n < 100:
return TENS[n // 10] + ("" if n % 10 == 0 else "-" + ONES[n % 10])
if n < 1000:
return ONES[n // 100] + " hundred" + ("" if n % 100 == 0 else " " + cardinal(n % 100))
for div, name in ((10 ** 12, "trillion"), (10 ** 9, "billion"), (10 ** 6, "million"), (1000, "thousand")):
if n >= div:
rest = n % div
return cardinal(n // div) + " " + name + ("" if rest == 0 else " " + cardinal(rest))
return str(n)
def ordinal(n):
w = cardinal(n)
head, sep, last = w.rpartition(" ") if " " in w else ("", "", w)
hy_head, hy, tail = last.rpartition("-") if "-" in last else ("", "", last)
if tail in ORD_IRREG:
tail = ORD_IRREG[tail]
elif tail.endswith("y"):
tail = tail[:-1] + "ieth"
else:
tail = tail + "th"
last = hy_head + hy + tail
return head + sep + last
def year(y):
y = int(y)
if 2000 <= y <= 2009:
return "two thousand" + ("" if y == 2000 else " " + ONES[y - 2000])
if 1100 <= y <= 2099 and y % 100 != 0:
hi, lo = divmod(y, 100)
return cardinal(hi) + " " + (("oh " + ONES[lo]) if lo < 10 else cardinal(lo))
if 1100 <= y <= 2099:
return cardinal(y // 100) + " hundred"
return cardinal(y)
def digits(s):
return " ".join(DIGIT[c] for c in s if c.isdigit())
def decimal(s):
s = s.replace(",", "")
neg = s.startswith("-") or s.startswith("−")
s = s.lstrip("-−")
if "." in s:
a, b = s.split(".", 1)
out = (cardinal(a) if a else "zero") + " point " + digits(b)
else:
out = cardinal(s)
return ("minus " if neg else "") + out
def number(s):
raw = s.replace(",", "")
if "." in raw:
return decimal(s)
if (len(raw) >= 5 and "," not in s) or (len(raw) > 1 and raw.startswith("0")):
return digits(raw)
return cardinal(raw)
def spell(tok):
out = []
for c in tok:
if c.isdigit():
out.append(DIGIT[c])
elif c.isalpha():
out.append(c.upper())
return " ".join(out)
EXT_WORDS = {"zip", "html", "json", "yaml", "java", "py"}
TLD_WORDS = {"com", "org", "net", "edu", "gov", "io", "co", "uk", "ai", "app", "dev"}
def _url_piece(p):
if not p:
return ""
if p.isdigit():
return digits(p)
if re.fullmatch(r"[A-Za-z]+", p):
return p
return " ".join(_url_piece(x) for x in re.findall(r"[A-Za-z]+|\d+", p))
def say_path(s, is_file=False):
out = []
for part in re.split(r"([./_\-:@])", s):
if part == ".":
out.append("dot")
elif part == "/":
out.append("slash")
elif part == "_":
out.append("underscore")
elif part == "-":
out.append("dash")
elif part == ":":
out.append("colon")
elif part == "@":
out.append("at")
elif part:
out.append(_url_piece(part))
return " ".join(x for x in out if x)
def r_email(m):
user, dom = m.group(1), m.group(2)
return say_path(user) + " at " + say_path(dom)
def r_url(m):
s = m.group(0)
s = re.sub(r"^https?://", lambda x: "", s)
return say_path(s.rstrip("./"))
def r_file(m):
name, ext = m.group(1), m.group(2)
e = ext.lower() if ext.lower() in EXT_WORDS else spell(ext)
return say_path(name) + " dot " + e
def r_version(m):
pre, ver = m.group(1), m.group(2)
lead = "version " if pre and pre.lower() == "v" else (pre if pre else "")
return lead + " point ".join(cardinal(p) for p in ver.split("."))
def r_phone(m):
s = m.group(0)
plus = "plus " if s.strip().startswith("+") else ""
groups = re.findall(r"\d+", s)
return plus + ", ".join(digits(g) for g in groups)
def _minutes(mm):
mm = int(mm)
return "oh " + ONES[mm] if 0 < mm < 10 else cardinal(mm)
def r_time(m):
h, mm, ap = int(m.group(1)), m.group(2), m.group(3)
ap_s = ""
if ap:
ap_s = " a m" if ap.lower().startswith("a") else " p m"
if int(mm) == 0:
if ap:
return cardinal(h) + ap_s
if h > 12 or h == 0:
return (cardinal(h) if h else "zero") + " hundred"
return cardinal(h) + " o'clock"
hour = ("oh " + ONES[h]) if (m.group(1).startswith("0") and not ap and h != 0) else cardinal(h)
if h == 0:
hour = "zero"
return hour + " " + _minutes(mm) + ap_s
def r_duration(m):
h, mm = int(m.group(1)), m.group(2)
s = cardinal(h) + (" hour" if h == 1 else " hours")
if mm:
s += " " + cardinal(int(mm)) + (" minutes" if m.group(3) else "")
return s
def _date_words(d, mo, y=None, us=False):
s = (MONTHS[mo - 1] + " " + ordinal(d)) if us else ("the " + ordinal(d) + " of " + MONTHS[mo - 1])
if y is not None:
yy = int(y)
if len(y) == 2:
yy += 2000 if yy < 50 else 1900
s += ", " + year(yy)
return s
def r_slash_date(m):
a, b, y = int(m.group(1)), int(m.group(2)), m.group(3)
if a > 12 and b <= 12:
return _date_words(a, b, y)
if b > 12 and a <= 12:
return _date_words(b, a, y, us=True)
if a <= 12 and b <= 12 and a >= 1 and b >= 1:
return _date_words(a, b, y)
return m.group(0)
def r_written_md(m):
mo = MONTHS.index(m.group(1).capitalize()) + 1
s = m.group(1) + " " + ordinal(int(m.group(2)))
if m.group(3):
s += ", " + year(m.group(3))
return s
def r_written_dm(m):
s = "the " + ordinal(int(m.group(1))) + " of " + m.group(2)
if m.group(3):
s += " " + year(m.group(3))
return s
CUR = {"$": ("dollar", "dollars", "cent", "cents"), "€": ("euro", "euros", "cent", "cents"),
"£": ("pound", "pounds", "penny", "pence"), "¥": ("yen", "yen", "", "")}
SCALE = {"k": "thousand", "m": "million", "mn": "million", "b": "billion", "bn": "billion"}
def r_currency(m):
sym, amt, scale = m.group(1), m.group(2), (m.group(3) or "").lower()
one, many, c1, cn = CUR[sym]
if scale:
return decimal(amt) + " " + SCALE[scale] + " " + many
raw = amt.replace(",", "")
whole, _, frac = raw.partition(".")
w = int(whole or 0)
s = cardinal(w) + " " + (one if w == 1 else many)
if frac and int(frac) and c1:
f = int(frac.ljust(2, "0")[:2])
s += " and " + cardinal(f) + " " + (c1 if f == 1 else cn)
return s
def r_percent(m):
return decimal(m.group(1)) + " percent"
def r_temp(m):
unit = {"C": " degrees Celsius", "F": " degrees Fahrenheit"}.get((m.group(2) or "").upper(), " degrees")
return decimal(m.group(1)) + unit
def _yearish(x):
return re.fullmatch(r"1[5-9]\d\d|20\d\d", x) is not None
def r_range(m):
a, b, unit = m.group(1), m.group(2), m.group(3)
if _yearish(a) and _yearish(b):
s = year(a) + " to " + year(b)
else:
s = number(a) + " to " + number(b)
if unit:
s += " " + UNITS[unit.lower()][1]
return s
def r_hash(m):
return "number " + number(m.group(1))
def r_ordinal(m):
return ordinal(int(m.group(1).replace(",", "")))
UNITS = {"mm": ("millimeter", "millimeters"), "cm": ("centimeter", "centimeters"), "m": ("meter", "meters"),
"km": ("kilometer", "kilometers"), "kg": ("kilogram", "kilograms"), "g": ("gram", "grams"),
"mg": ("milligram", "milligrams"), "l": ("liter", "liters"), "ml": ("milliliter", "milliliters"),
"lb": ("pound", "pounds"), "lbs": ("pound", "pounds"), "oz": ("ounce", "ounces"),
"ft": ("foot", "feet"), "in": ("inch", "inches"), "mi": ("mile", "miles"),
"km/h": ("kilometer per hour", "kilometers per hour"), "kph": ("kilometer per hour", "kilometers per hour"),
"mph": ("mile per hour", "miles per hour"), "kwh": ("kilowatt hour", "kilowatt hours"),
"kw": ("kilowatt", "kilowatts"), "w": ("watt", "watts"), "v": ("volt", "volts"), "mah": ("milliamp hour", "milliamp hours"),
"mAh": ("milliamp hour", "milliamp hours"), "ha": ("hectare", "hectares"),
"gb": ("gigabyte", "gigabytes"), "mb": ("megabyte", "megabytes"), "tb": ("terabyte", "terabytes"),
"kb": ("kilobyte", "kilobytes"), "ghz": ("gigahertz", "gigahertz"), "mhz": ("megahertz", "megahertz"),
"hz": ("hertz", "hertz"), "bar": ("bar", "bar"), "psi": ("p s i", "p s i"), "ms": ("millisecond", "milliseconds"),
"s": ("second", "seconds"), "sec": ("second", "seconds"), "min": ("minute", "minutes"),
"mins": ("minute", "minutes"), "h": ("hour", "hours"), "hr": ("hour", "hours"), "hrs": ("hour", "hours"),
"sq ft": ("square foot", "square feet"), "m2": ("square meter", "square meters"),
"m²": ("square meter", "square meters"), "°": ("degree", "degrees")}
_UNIT_RE = "|".join(sorted((re.escape(u) for u in UNITS), key=len, reverse=True))
def r_unit(m):
num, unit = m.group(1), m.group(2)
one, many = UNITS[unit.lower()]
val = num.replace(",", "")
return decimal(num) + " " + (one if val in ("1", "1.0") else many)
ABBR = [(r"\bMr\.", "Mister"), (r"\bMrs\.", "Missus"), (r"\bMs\.", "Miz"), (r"\bDr\.(?=\s+[A-Z])", "Doctor"),
(r"\bProf\.", "Professor"), (r"\bSt\.(?=\s+[A-Z])", "Saint"), (r"(?<=[a-z] )St\.", "Street"),
(r"(?<=[a-z] )St\b\.?", "Street"), (r"\bAve\.?(?=\W|$)", "Avenue"), (r"\bRd\.(?=\W|$)", "Road"),
(r"\bBlvd\.?", "Boulevard"), (r"\bApt\.?(?=\s)", "apartment"), (r"\b[Nn]o\.(?=\s*\d)", "number"),
(r"\betc\.", "et cetera"), (r"\be\.g\.", "for example"), (r"\bi\.e\.", "that is"), (r"\bvs\.?(?=\s)", "versus"),
(r"\bapprox\.", "approximately"), (r"\bJr\.", "Junior"), (r"\bSr\.", "Senior"), (r"\bDept\.", "department"),
(r"\bLtd\.?", "Limited"), (r"\bInc\.", "Incorporated"), (r"\bCo\.(?=\s|$)", "Company"),
(r"\bmin\.(?=\s)", "minutes"), (r"\bhrs?\.(?=\s)", "hours"), (r"\bft\.(?=\s)", "feet"),
(r"\bTel\.?(?=\s*[:\d])", "telephone"), (r"\bext\.(?=\s*\d)", "extension"), (r"\bMt\.(?=\s+[A-Z])", "Mount"),
(r"\b(a)\.m\.", r"A M"), (r"\b(p)\.m\.", r"P M"),
(r"\bref\.\s+no\.", "reference number"), (r"\bSgt\.", "Sergeant"), (r"\bCapt\.", "Captain"),
(r"\bLt\.", "Lieutenant"), (r"\b[Ff]ig\.(?=\s*\d)", "figure"), (r"\bCtrl\s?\+\s?", "control plus "),
(r"\bCmd\s?\+\s?", "command plus "), (r"\bAlt\s?\+\s?", "alt plus "), (r"\bShift\s?\+\s?", "shift plus "),
(r"\bMon\.", "Monday"), (r"\bTue\.", "Tuesday"), (r"\bWed\.", "Wednesday"), (r"\bThu\.", "Thursday"),
(r"\bFri\.", "Friday"), (r"\bSat\.", "Saturday"), (r"\bSun\.", "Sunday"), (r"\bASAP\b", "as soon as possible"),
(r"\bFAQ\b", "F A Q"), (r"\bETA\b", "E T A"), (r"\bID\b", "I D"), (r"\bOK\b", "okay")]
MATH = [("±", " plus or minus "), ("≈", " approximately "), ("≤", " less than or equal to "),
("≥", " greater than or equal to "), ("≠", " not equal to "), ("×", " times "), ("÷", " divided by "),
("√", " square root of "), ("π", " pi "), ("∞", " infinity "), ("½", " one half "), ("¼", " one quarter "),
("¾", " three quarters ")]
def r_code(m):
tok = m.group(0)
parts = [p for p in re.split(r"[-_/.]", tok) if p]
out = []
for p in parts:
if re.fullmatch(r"[A-Z]{4,}", p) and re.search(r"[AEIOU]", p) and len(p) <= 8:
out.append(p.lower())
elif re.fullmatch(r"[1-9]\d?", p) and out and re.fullmatch(r"[a-z]+", out[-1]):
out.append(cardinal(int(p)))
else:
chunks = re.findall(r"[A-Za-z]+|\d+", p)
out.append(" ".join(spell(c) if c.isalpha() else digits(c) for c in chunks))
return ", ".join(out)
def normalize(text):
t = " " + text.replace("−", "-").replace("–", "-").replace("—", ", ").replace("’", "'") + " "
end = text.strip()[-1:] if text.strip()[-1:] in ".!?" else ""
t = re.sub(r"\b(\d{4})-(\d{2})-(\d{2})\b", lambda m: _date_words(int(m.group(3)), int(m.group(2)), m.group(1))
if 1 <= int(m.group(2)) <= 12 and 1 <= int(m.group(3)) <= 31 else m.group(0), t)
t = re.sub(r"\b(\d{1,2}:\d{2})\s?-\s?(\d{1,2}:\d{2})\b", r"\1 to \2", t)
t = re.sub(r"\b(\d)/(\d{1,2})(?=\s+of\b)", lambda m: FRAC.get((int(m.group(1)), int(m.group(2))),
cardinal(m.group(1)) + " over " + cardinal(m.group(2))), t)
t = re.sub(r"(\d+(?:\.\d+)?)-(inch|foot|meter|metre|mile|liter|litre|hour|minute|day|week|month|year|page|"
r"person|seat|door|piece|core|bit|megapixel)\b", lambda m: decimal(m.group(1)) + " " + m.group(2), t)
t = re.sub(r"(?<!\S)(?:[A-Z0-9]{1,6} ){2,}[A-Z0-9]{1,6}(?=[\s.,;!?]|$)",
lambda m: ", ".join(spell(g) for g in m.group(0).split())
if re.search(r"[A-Z]", m.group(0)) and len(re.sub(r"\D", "", m.group(0))) >= 6 else m.group(0), t)
t = re.sub(r"([\w.+-]+)@([\w-]+(?:\.[\w-]+)+)", r_email, t)
t = re.sub(r"\bhttps?://[^\s,]+|\bwww\.[^\s,]+|\b(?:[\w-]+\.)+(?:com|org|net|io|ai|dev|app|co\.uk|edu|gov)\b(?:/[^\s,]*)?",
lambda m: r_url(m) + ("." if m.group(0).endswith(".") else ""), t)
t = re.sub(r"\b([\w\-]+(?:\.[\w\-]+)*)\.([A-Za-z]{2,4}\d?)\b(?=[\s,;:!?)]|\.\s|\.$)",
lambda m: r_file(m) if not re.fullmatch(r"\d+(?:\.\d+)*", m.group(1)) and (
"_" in m.group(1) or re.search(r"\d", m.group(1)) or m.group(2).lower() in
{"pdf", "jpg", "jpeg", "png", "gif", "csv", "txt", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
"zip", "mp3", "mp4", "mov", "wav", "json", "html", "py", "js", "exe", "svg", "md", "log", "tar",
"gz", "iso", "dmg", "apk", "heic", "webp", "key", "pages", "xml", "yaml", "yml", "sh", "ics"})
else m.group(0), t)
t = re.sub(r"\b(v|V|[Vv]ersion |Python |iOS |Android |macOS |Windows |release |build )?(\d+\.\d+(?:\.\d+)+)\b",
lambda m: r_version(m) if (m.group(1) or m.group(2).count(".") >= 2) else m.group(0), t)
t = re.sub(r"(?<![\w-])(v|V)(\d+(?:\.\d+)?)\b", lambda m: "version " + " point ".join(cardinal(p) for p in m.group(2).split(".")), t)
t = re.sub(r"\b(Python |iOS |Android |macOS |[Vv]ersion )(\d+)\.(\d+)\b",
lambda m: m.group(1) + cardinal(m.group(2)) + " point " + cardinal(m.group(3)), t)
t = re.sub(r"(?<![\w/\-])\+?\(?\d{2,5}\)?(?:[ \-.]\d{2,5}){1,4}(?![\w/:]|-\w)",
lambda m: r_phone(m) if len(re.sub(r"\D", "", m.group(0))) >= 7 and not re.fullmatch(
r"(1[5-9]\d\d|20\d\d)\s?-\s?(1[5-9]\d\d|20\d\d)", m.group(0).strip()) else m.group(0), t)
t = re.sub(r"(?<![\d:])(\d{1,2}):(\d)(?![\d:])", lambda m: cardinal(m.group(1)) + " to " + cardinal(m.group(2)), t)
t = re.sub(r"\b(\d{1,2}):(\d{2})\s*([AaPp]\.?\s?[Mm]\.?)?(?=\W|$)", r_time, t)
t = re.sub(r"\b(\d{1,2})\s?(?:h|hr|hrs)\s?(\d{1,2})?\s?(min|m)?\b(?!\w)", r_duration, t)
t = re.sub(r"\b(\d{1,2})/(\d{1,2})(?:/(\d{4}|\d{2}))?\b", r_slash_date, t)
t = re.sub(rf"\b(\d{{1,2}})\s?-\s?(\d{{1,2}}) ({'|'.join(MONTHS)})\b",
lambda m: f"the {ordinal(int(m.group(1)))} to the {ordinal(int(m.group(2)))} of {m.group(3)}", t)
mon = "|".join(MONTHS) + "|" + "|".join(m[:3] for m in MONTHS)
t = re.sub(rf"\b({mon})\.? (\d{{1,2}})(?:st|nd|rd|th)?(?:,? (\d{{4}}))?\b",
lambda m: r_written_md(type("M", (), {"group": lambda s, i: (
MONTHS[[x[:3] for x in MONTHS].index(m.group(1)[:3])] if i == 1 else m.group(i))})()), t)
t = re.sub(rf"\b(\d{{1,2}})(?:st|nd|rd|th)? (?:of )?({'|'.join(MONTHS)})(?:,? (\d{{4}}))?\b", r_written_dm, t)
t = re.sub(r"\b(?:in|since|from|of|by|until|year|circa|founded in) (1[1-9]\d\d|20\d\d)\b",
lambda m: m.group(0)[:-4] + year(m.group(1)), t)
t = re.sub(r"\b(US|A|C|NZ|HK)?\$\s?(\d[\d,]*(?:\.\d+)?) (thousand|million|billion|trillion)\b",
lambda m: decimal(m.group(2)) + " " + m.group(3) + (" " + spell(m.group(1)) if m.group(1) else "") + " dollars", t)
t = re.sub(r"\b(US|A|C|NZ|HK)\$(?=\s?\d)", lambda m: spell(m.group(1)) + " $", t)
t = re.sub(r"(-?)([$€£¥])\s?(\d[\d,]*(?:\.\d+)?)(?:\s?(bn|mn|[kKmMbB])\b)?",
lambda m: ("minus " if m.group(1) else "") + r_currency(type("M", (), {"group": lambda _s, i: m.group(i + 1)})()), t)
t = re.sub(r"(-?\d+(?:\.\d+)?)\s?%", r_percent, t)
t = re.sub(r"(-?\d+(?:\.\d+)?)\s?°\s?([CF])?\b", r_temp, t)
t = re.sub(r"#\s?(\d+(?:-\d+)+)\b", lambda m: "number " + ", ".join(digits(g) for g in m.group(1).split("-")), t)
t = re.sub(rf"(?<![\w.\-])(\d[\d,]*(?:\.\d+)?)\s?-\s?(\d[\d,]*(?:\.\d+)?)(?:\s?({_UNIT_RE})(?![\w/]))?(?![\w.]*\d|-\w)",
r_range, t)
t = re.sub(r"#\s?(\d[\d,]*)", r_hash, t)
t = re.sub(r"#(?=[A-Za-z])", "number ", t)
t = re.sub(r"\b(\d[\d,]*)(?:st|nd|rd|th)\b", r_ordinal, t)
t = re.sub(rf"(?<![\w.])(\d[\d,]*(?:\.\d+)?)\s?({_UNIT_RE})(?![\w/])", r_unit, t)
for pat, rep in ABBR:
t = re.sub(pat, rep, t)
for a, b in MATH:
t = t.replace(a, b)
t = re.sub(r"(?<=\d)\s?[x×]\s?(?=\d)", " by ", t)
t = re.sub(r"(?<=\s)\+(?=\s|\d)", "plus ", t)
t = re.sub(r"(?<=\s)=(?=\s)", "equals", t)
t = re.sub(r"(?<=\d)\s?/\s?(?=\d)", " over ", t)
t = re.sub(r"\s<\s", " less than ", t)
t = re.sub(r"\s>\s", " greater than ", t)
t = re.sub(r"(?<![\w-])(\d{1,3})([A-Z])(?![\w-])", lambda m: cardinal(m.group(1)) + " " + m.group(2), t)
t = re.sub(r"\b(?=[A-Za-z0-9\-]*\d)(?=[A-Za-z0-9\-]*[A-Za-z])[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*\b", r_code, t)
t = re.sub(r"\b[A-Z]{2,}(?:-[A-Z0-9]+)+\b", r_code, t)
t = re.sub(r"\d[\d,]*(?:\.\d+)?", lambda m: number(m.group(0)), t)
t = t.replace("&", " and ").replace("@", " at ").replace("_", " underscore ")
t = re.sub(r"(?<=\w)/(?=\w)", " slash ", t)
t = re.sub(r"\s+", " ", t).strip()
t = re.sub(r"\s+([,.;:!?])", r"\1", t)
t = re.sub(r"\b([Tt]he) the\b", r"\1", t)
if end and not t.endswith(end) and t[-1:] not in ".!?":
t += end
return t
if __name__ == "__main__":
import sys
for line in sys.argv[1:] or sys.stdin:
print(normalize(line.strip()))