Spaces:
Sleeping
Sleeping
| """Idempotent reference-data bootstrap: categories, states, sources from config.""" | |
| from __future__ import annotations | |
| import json | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from sqlalchemy import insert, select, update | |
| from sqlalchemy.engine import Engine | |
| from .. import constants as C | |
| from ..db import categories, init_db, ref_lgas, ref_states, sources | |
| from ..reference_data import STATES | |
| # From docs/data-source-scraper-ban-check.md (checked 2026-07-19 by S. Akinnusi). | |
| BAN_CHECK_DATE = datetime(2026, 7, 19, tzinfo=timezone.utc) | |
| BAN_CHECK = { | |
| "premium_times": ("no_explicit_ban", "https://www.premiumtimesng.com/terms-and-conditions"), | |
| "prnigeria": ("no_explicit_ban", None), | |
| "humangle": ("no_explicit_ban", None), | |
| "nta": ("no_explicit_ban", None), | |
| "nigerian_tribune": ("explicit_ban", "https://tribuneonlineng.com/terms-and-conditions/"), | |
| } | |
| def bootstrap(engine: Engine) -> dict: | |
| from scrapers.config import OUTLETS # local import keeps `api` importable without scrapers on path | |
| init_db(engine) | |
| added = {"categories": 0, "states": 0, "lgas": 0, "sources": 0} | |
| with engine.begin() as conn: | |
| have = {r[0] for r in conn.execute(select(categories.c.code))} | |
| for code, label, desc in C.CATEGORIES: | |
| if code not in have: | |
| conn.execute(insert(categories).values(code=code, label=label, description=desc)) | |
| added["categories"] += 1 | |
| have = {r[0] for r in conn.execute(select(ref_states.c.state_code))} | |
| for code, name, zone, lat, lng in STATES: | |
| if code not in have: | |
| conn.execute(insert(ref_states).values( | |
| state_code=code, name=name, geopolitical_zone=zone, | |
| centroid_lat=lat, centroid_lng=lng)) | |
| added["states"] += 1 | |
| have_lga = {r[0] for r in conn.execute(select(ref_lgas.c.state_code).distinct())} | |
| rows = [dict(state_code=code, name=name, centroid_lat=lat, centroid_lng=lng) | |
| for code, name, lat, lng in json.loads((Path(__file__).resolve().parent.parent / "data" / "ng_lgas.json").read_text(encoding="utf-8")) | |
| if code not in have_lga] # a state that already has LGAs is left alone (staff may have edited it) | |
| if rows: | |
| conn.execute(insert(ref_lgas), rows) # one batch: a row-by-row insert over a remote database takes minutes | |
| added["lgas"] = len(rows) | |
| have = {r[0] for r in conn.execute(select(sources.c.source_key))} | |
| for key, cfg in OUTLETS.items(): | |
| if key in have: | |
| # An outlet nobody has checked yet picks up the feeds and notes found since it was first listed. | |
| # One that has a recorded check is left exactly as staff or the check left it. | |
| conn.execute(update(sources).where( | |
| sources.c.source_key == key, sources.c.tos_result == "unchecked", sources.c.robots_checked_at.is_(None), | |
| ).values(feeds=list(cfg.feeds), robots_summary=cfg.robots_status, crawl_delay_seconds=cfg.crawl_delay)) | |
| continue | |
| result, tos_url = BAN_CHECK.get(key, ("unchecked", None)) | |
| checked = BAN_CHECK_DATE if key in BAN_CHECK else None | |
| can_enable = bool(cfg.enabled and cfg.feeds and result == "no_explicit_ban") | |
| conn.execute(insert(sources).values( | |
| source_key=key, name=cfg.name, source_type="news_outlet", base_url=cfg.base_url, | |
| feeds=list(cfg.feeds), crawl_delay_seconds=cfg.crawl_delay, | |
| robots_checked_at=checked, robots_summary=cfg.robots_status, | |
| tos_url=tos_url, tos_checked_at=checked, tos_result=result, enabled=can_enable, | |
| notes=("Check recorded in docs/data-source-scraper-ban-check.md. " | |
| + ("No feed configured; HTML fallback not built." if not cfg.feeds else "") | |
| + (" Held back: explicit bot ban, awaiting compliance decision." if result == "explicit_ban" else "")).strip())) | |
| added["sources"] += 1 | |
| return added | |