Spaces:
Sleeping
Sleeping
Download rules/dates.py from frznkshn/TextNormalizer: direct link, hf CLI and curl.
- Browser
- Download file 4.09 kB
-
https://huggingface.co/spaces/frznkshn/TextNormalizer/resolve/main/rules/dates.py
- Command line
-
hf download hf://spaces/frznkshn/TextNormalizer/rules/dates.py
-
curl -L -o dates.py https://huggingface.co/spaces/frznkshn/TextNormalizer/resolve/main/rules/dates.py
4.09 kB
| import re | |
| from datetime import datetime | |
| from typing import Union | |
| from dateutil import parser as gregorian_parser | |
| from num2fawords import words | |
| from num2words import num2words | |
| from persiantools import digits | |
| from persiantools.jdatetime import JalaliDate | |
| from rules.base import BaseProcessor | |
| import random | |
| Date = Union[datetime, JalaliDate] | |
| GREGORIAN_MONTHS = ( | |
| "January", | |
| "February", | |
| "March", | |
| "April", | |
| "May", | |
| "June", | |
| "July", | |
| "August", | |
| "September", | |
| "October", | |
| "November", | |
| "December", | |
| ) | |
| DATE_PATTERN = re.compile( | |
| r""" | |
| (?: | |
| (?P<year>\d{4})[^\d]+(?P<month>\d{1,2})[^\d]+(?P<day>\d{1,2}) | |
| | | |
| (?P<day_first>\d{1,2})[^\d]+(?P<month_first>\d{1,2})[^\d]+(?P<year_first>\d{4}) | |
| ) | |
| """, | |
| re.VERBOSE, | |
| ) | |
| class DateProcessor(BaseProcessor): | |
| def __init__(self, epsilon:float=1.0): | |
| super(DateProcessor, self).__init__(epsilon=epsilon) | |
| def process(self, date_string: str, language: str, *args, **kwargs) -> str: | |
| if random.random() > self.epsilon: | |
| return date_string | |
| if not self.is_date(date_string): | |
| return date_string | |
| try: | |
| date = self._parse_date(date_string) | |
| except ValueError: | |
| return date_string | |
| if date is None: | |
| raise date_string | |
| return self._date_to_text(date, language) | |
| def is_date(date_string: str) -> bool: | |
| date_string = digits.fa_to_en(date_string) | |
| return DATE_PATTERN.fullmatch(date_string) is not None | |
| def _parse_date(self, date_string: str) -> Date | None: | |
| clean_string = digits.fa_to_en(date_string) | |
| match = DATE_PATTERN.search(clean_string) | |
| if match: | |
| year, month, day = self._extract_date_parts(match) | |
| # Years less than 1600 is considered as Jalali | |
| if year < 1600: | |
| try: | |
| return JalaliDate(year, month, day) | |
| except ValueError: | |
| pass | |
| # Try Gregorian parsing. | |
| try: | |
| date = gregorian_parser.parse(clean_string) | |
| if date.year >= 1600: | |
| return date | |
| except (ValueError, OverflowError): | |
| pass | |
| # If regex matched, give Jalali parsing one final attempt so that | |
| # invalid Jalali dates produce a useful error. | |
| if match: | |
| year, month, day = self._extract_date_parts(match) | |
| try: | |
| return JalaliDate(year, month, day) | |
| except ValueError as exc: | |
| raise ValueError( | |
| f"Invalid Jalali date: {date_string!r}" | |
| ) from exc | |
| return None | |
| def _extract_date_parts( | |
| match: re.Match[str], | |
| ) -> tuple[int, int, int]: | |
| if match.group("year"): | |
| return ( | |
| int(match.group("year")), | |
| int(match.group("month")), | |
| int(match.group("day")), | |
| ) | |
| return ( | |
| int(match.group("year_first")), | |
| int(match.group("month_first")), | |
| int(match.group("day_first")), | |
| ) | |
| def _date_to_text(self, date: Date, language: str) -> str: | |
| if language == "fa": | |
| return self._date_to_persian(date) | |
| if language == "en": | |
| return self._date_to_english(date) | |
| raise ValueError(f"Unsupported language: {language!r}") | |
| def _date_to_persian(date: Date) -> str: | |
| return f"{words(date.day)} {words(date.month)} {words(date.year)}" | |
| def _date_to_english(date: Date) -> str: | |
| year = date.year | |
| if "00" in str(year) or len(str(year)) < 4: | |
| year_text = num2words(year) | |
| else: | |
| year_text = ( | |
| f"{num2words(year // 100)} " | |
| f"{num2words(year % 100)}" | |
| ) | |
| return ( | |
| f"{num2words(date.day, ordinal=True)} " | |
| f"of {GREGORIAN_MONTHS[date.month - 1]}, " | |
| f"{year_text}" | |
| ) |