File size: 6,935 Bytes
dea44a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | """
Normalize raw scraped event/attraction data → events_calendar.json schema.
Merges into existing events_calendar.json (append, deduped by name).
"""
import json
import logging
import re
from typing import Optional
try:
from rapidfuzz import fuzz
HAS_FUZZ = True
except ImportError:
HAS_FUZZ = False
from scripts.scrapers.config import PATHS, SCRAPER_SETTINGS
logger = logging.getLogger(__name__)
# Known recurring annual festivals mapped by keyword
KNOWN_FESTIVALS = {
"new year": {"month": 1, "day": 1, "duration_days": 1, "impact": "high", "recurring": "yearly"},
"lunar new year": {"month": 1, "day": 29, "duration_days": 7, "impact": "high", "recurring": "yearly"},
"christmas": {"month": 12, "day": 25, "duration_days": 3, "impact": "high", "recurring": "yearly"},
"halloween": {"month": 10, "day": 31, "duration_days": 1, "impact": "medium", "recurring": "yearly"},
"carnival": {"month": 2, "day": 1, "duration_days": 5, "impact": "high", "recurring": "yearly"},
"oktoberfest": {"month": 9, "day": 16, "duration_days": 18, "impact": "high", "recurring": "yearly"},
"cherry blossom": {"month": 4, "day": 1, "duration_days": 14, "impact": "high", "recurring": "yearly"},
"songkran": {"month": 4, "day": 13, "duration_days": 3, "impact": "high", "recurring": "yearly"},
"diwali": {"month": 11, "day": 1, "duration_days": 5, "impact": "high", "recurring": "yearly"},
"ramadan": {"month": 3, "day": 10, "duration_days": 30, "impact": "medium", "recurring": "yearly"},
"eid": {"month": 4, "day": 10, "duration_days": 3, "impact": "medium", "recurring": "yearly"},
"holi": {"month": 3, "day": 25, "duration_days": 2, "impact": "high", "recurring": "yearly"},
"lantern festival": {"month": 2, "day": 15, "duration_days": 1, "impact": "medium", "recurring": "yearly"},
"bastille": {"month": 7, "day": 14, "duration_days": 1, "impact": "medium", "recurring": "yearly"},
"marathon": {"month": 11, "day": 1, "duration_days": 1, "impact": "low", "recurring": "yearly"},
"film festival": {"month": 9, "day": 1, "duration_days": 10, "impact": "medium", "recurring": "yearly"},
"fashion week": {"month": 9, "day": 1, "duration_days": 7, "impact": "medium", "recurring": "yearly"},
"fireworks": {"month": 7, "day": 4, "duration_days": 1, "impact": "medium", "recurring": "yearly"},
"music festival": {"month": 7, "day": 1, "duration_days": 3, "impact": "medium", "recurring": "yearly"},
"pride": {"month": 6, "day": 15, "duration_days": 7, "impact": "medium", "recurring": "yearly"},
"tet": {"month": 1, "day": 29, "duration_days": 7, "impact": "high", "recurring": "yearly"},
"full moon": {"month": 0, "day": 15, "duration_days": 1, "impact": "low", "recurring": "monthly"},
}
def _classify_impact(reviews_count: int) -> str:
if reviews_count >= 1000:
return "high"
if reviews_count >= 100:
return "medium"
return "low"
def _match_known_festival(name: str) -> Optional[dict]:
"""Match event name against known festivals."""
name_lower = name.lower()
for keyword, meta in KNOWN_FESTIVALS.items():
if keyword in name_lower:
return meta
return None
def normalize_event(raw: dict) -> Optional[dict]:
"""
Convert raw scraped event dict to events_calendar.json schema.
Returns None if entry lacks minimum required fields.
"""
name = (raw.get("name") or "").strip()
if not name or len(name) < 3:
return None
# Try to match known festival patterns
meta = _match_known_festival(name)
month = raw.get("month") or (meta["month"] if meta else 0)
day = raw.get("day") or (meta["day"] if meta else 1)
duration = raw.get("duration_days") or (meta["duration_days"] if meta else 1)
impact = _classify_impact(raw.get("reviews_count", 0))
if meta:
impact = meta.get("impact", impact)
result = {
"name_vi": "",
"name_en": (raw.get("name_en") or name).strip(),
"month": int(month),
"day": int(day),
"duration_days": int(duration),
"description_vi": "",
"description_en": (raw.get("description") or "")[:400].strip(),
"destinations": [raw["destination_id"]] if raw.get("destination_id") else [],
"impact": impact,
"_source": raw.get("source", "unknown"),
}
# Add recurring flag if detected
if meta and meta.get("recurring"):
result["recurring"] = meta["recurring"]
if meta.get("recurring") == "monthly" and meta.get("day"):
result["recurring_day"] = meta["day"]
return result
def is_duplicate_event(new: dict, existing: list) -> bool:
"""Check if an event is a duplicate by name similarity."""
new_name = new["name_en"].lower().strip()
threshold = SCRAPER_SETTINGS["dedup_name_threshold"]
for ex in existing:
ex_name = (ex.get("name_en") or ex.get("name_vi", "")).lower().strip()
if HAS_FUZZ:
if fuzz.ratio(new_name, ex_name) >= threshold:
return True
else:
if new_name == ex_name:
return True
return False
def merge_into_events_db(new_events: list) -> dict:
"""
Load existing events_calendar.json, append new events (deduped), save.
"""
events_db_path = PATHS["events_db"]
with open(events_db_path, "r", encoding="utf-8") as f:
db = json.load(f)
existing = db.get("events", [])
added = 0
skipped_invalid = 0
skipped_dup = 0
for raw in new_events:
normalized = normalize_event(raw)
if normalized is None:
skipped_invalid += 1
continue
if is_duplicate_event(normalized, existing):
# If same event, enrich destinations list
for ex in existing:
ex_name = (ex.get("name_en") or "").lower()
if ex_name == normalized["name_en"].lower():
dest = normalized["destinations"][0] if normalized["destinations"] else None
if dest and dest not in ex.get("destinations", []):
ex.setdefault("destinations", []).append(dest)
skipped_dup += 1
continue
existing.append(normalized)
added += 1
db["events"] = existing
with open(events_db_path, "w", encoding="utf-8") as f:
json.dump(db, f, ensure_ascii=False, indent=2)
stats = {
"total_in_db": len(existing),
"added": added,
"skipped_invalid": skipped_invalid,
"skipped_duplicate": skipped_dup,
}
logger.info(f"Event merge: +{added} added, {skipped_dup} dupes → {len(existing)} total")
return stats
|