| import hashlib |
| import re |
| import logging |
| from urllib.parse import urlparse |
|
|
| logger = logging.getLogger(__name__) |
|
|
|
|
| def get_namespace_id(url: str) -> str: |
| """ |
| Generates a collision-resistant namespace ID. |
| Format: clean_name_MD5hash |
| """ |
| clean_name = re.sub(r"https?://(www\.)?", "", url) |
| clean_name = re.sub(r"[^a-zA-Z0-9]", "_", clean_name) |
| clean_name = clean_name.strip("_")[:30] |
|
|
| url_hash = hashlib.md5(url.encode("utf-8")).hexdigest()[:6] |
|
|
| return f"{clean_name}_{url_hash}" |
|
|
|
|
| def is_valid_url(url: str, base_domain: str) -> bool: |
| """ |
| Enforces strict crawl scope. |
| """ |
| try: |
| parsed = urlparse(url) |
| base_parsed = urlparse(base_domain) |
|
|
| |
| if parsed.scheme not in ["http", "https"]: |
| return False |
|
|
| |
| |
| if not parsed.netloc.endswith(base_parsed.netloc): |
| return False |
|
|
| |
| ignored_exts = [ |
| ".pdf", |
| ".jpg", |
| ".png", |
| ".gif", |
| ".css", |
| ".js", |
| ".docx", |
| ".xlsx", |
| ".xml", |
| ".zip", |
| ".rar", |
| ".mp4", |
| ] |
| if any(parsed.path.lower().endswith(ext) for ext in ignored_exts): |
| return False |
|
|
| return True |
| except Exception: |
| return False |
|
|
