idiouf's picture
Upload 18 files
a22f12f verified
Raw History Blame Contribute Delete
2.89 kB
#!/usr/bin/env python3
"""Download every dataset once and write a snapshot next to the app.
Why: a free Hugging Face Space restarts cold and loses /tmp, so without a
snapshot every cold start re-downloads Wikidata, the World Bank and 54 boundary
files before the first page can paint. Run this on a machine with normal
internet, commit the `cache_*` files it writes, and the Space starts instantly.
python prefetch.py
The app prefers these files over the network and never expires them, so re-run
and re-commit whenever you want fresher data. The Data & method page shows how
old the snapshot is.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import sources as src
BASE = Path(__file__).resolve().parent
def write(key: str, frame, suffix: str = ".parquet") -> None:
path = BASE / f"cache_{key}{suffix}"
if suffix == ".parquet":
frame.to_parquet(path, index=False)
else:
path.write_text(json.dumps(frame))
size = path.stat().st_size / 1024
print(f" wrote {path.name} ({size:,.0f} KB)")
def main() -> int:
if src.FIXTURE_DIR:
print("CONSERVATION_HD_FIXTURES is set — unset it before prefetching, "
"or you will snapshot the test doubles.")
return 1
failures = []
print("Protected areas…")
pas = src.fetch_protected_areas()
if pas.ok:
write("protected_areas", pas.frame)
else:
failures.append(("Protected areas", pas.status.detail))
print(f" FAILED: {pas.status.detail}")
print("World Bank indicators…")
ind = src.fetch_indicators()
if ind.ok:
write("indicators_1990_2024", ind.frame)
else:
failures.append(("World Bank indicators", ind.status.detail))
print(f" FAILED: {ind.status.detail}")
print("UN World Population Prospects…")
un = src.fetch_un_population()
if un.ok:
write("un_population", un.frame)
else:
print(f" skipped: {un.status.detail}")
print("Subnational HDI…")
shdi = src.fetch_subnational_hdi()
if shdi.ok:
write("subnational_hdi", shdi.frame)
else:
print(f" skipped: {shdi.status.detail}")
print("Country boundaries…")
boundaries, status = src.fetch_boundaries(0)
if status.ok:
write("boundaries_adm0", boundaries, ".json")
else:
failures.append(("Boundaries", status.detail))
print(f" FAILED: {status.detail}")
print()
if failures:
print("Snapshot incomplete:")
for name, detail in failures:
print(f" · {name}: {detail}")
print("\nCommit whatever was written; the app falls back to the network "
"for the rest.")
return 1
print("Snapshot complete. Commit the cache_* files alongside the app.")
return 0
if __name__ == "__main__":
sys.exit(main())