Spaces:
Running
Running
| # utils.py | |
| import time, urllib.parse, logging, random | |
| from dateutil import parser as dateparser | |
| logger = logging.getLogger(__name__) | |
| logging.basicConfig(level=logging.INFO) | |
| def canonicalize_url(url, base=None): | |
| if base: | |
| return urllib.parse.urljoin(base, url) | |
| return url | |
| def parse_date(s): | |
| try: | |
| return dateparser.parse(s) | |
| except Exception: | |
| return None | |
| def backoff_sleep(attempt): | |
| time.sleep(min(2**attempt + random.random(), 10)) | |