Spaces:
Sleeping
Sleeping
Yousef Awad commited on
Upload src/wqf7009_aqi/dataset.py with huggingface_hub
Browse files- src/wqf7009_aqi/dataset.py +133 -0
src/wqf7009_aqi/dataset.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import random
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import pandas as pd
|
| 7 |
+
from tqdm import tqdm
|
| 8 |
+
|
| 9 |
+
from wqf7009_aqi.features import build_training_frame
|
| 10 |
+
from wqf7009_aqi.openaq_archive import (
|
| 11 |
+
download_object,
|
| 12 |
+
extract_location_catalog,
|
| 13 |
+
list_day_objects,
|
| 14 |
+
list_location_ids,
|
| 15 |
+
read_measurements_csv_gz,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _daily_pm25_from_measurements(df: pd.DataFrame) -> pd.DataFrame:
|
| 20 |
+
df = df.copy()
|
| 21 |
+
if "parameter" not in df.columns:
|
| 22 |
+
return pd.DataFrame()
|
| 23 |
+
df = df[df["parameter"].astype(str).str.lower().eq("pm25")]
|
| 24 |
+
if df.empty:
|
| 25 |
+
return pd.DataFrame()
|
| 26 |
+
|
| 27 |
+
df["datetime"] = pd.to_datetime(df["datetime"], errors="coerce", utc=True)
|
| 28 |
+
df = df.dropna(subset=["datetime", "value", "location_id"])
|
| 29 |
+
df["date"] = df["datetime"].dt.date
|
| 30 |
+
out = (
|
| 31 |
+
df.groupby(["location_id", "location", "lat", "lon", "date"], as_index=False)["value"]
|
| 32 |
+
.mean()
|
| 33 |
+
.rename(columns={"value": "pm25"})
|
| 34 |
+
)
|
| 35 |
+
return out
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def build_dataset(
|
| 39 |
+
*,
|
| 40 |
+
out_path: Path,
|
| 41 |
+
year: int,
|
| 42 |
+
months: list[int],
|
| 43 |
+
max_locations: int,
|
| 44 |
+
max_days_per_location: int,
|
| 45 |
+
seed: int,
|
| 46 |
+
cache_dir: Path,
|
| 47 |
+
scan_location_ids: int,
|
| 48 |
+
) -> None:
|
| 49 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 50 |
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
| 51 |
+
|
| 52 |
+
rng = random.Random(seed)
|
| 53 |
+
cache_ids = Path("data/cache/openaq_location_ids.json")
|
| 54 |
+
location_ids = list_location_ids(max_ids=scan_location_ids, cache_path=cache_ids)
|
| 55 |
+
rng.shuffle(location_ids)
|
| 56 |
+
|
| 57 |
+
# Select locations that actually have data for the requested year/months.
|
| 58 |
+
selected: list[int] = []
|
| 59 |
+
for loc_id in tqdm(location_ids, desc="Selecting locations"):
|
| 60 |
+
has_any = False
|
| 61 |
+
for m in months:
|
| 62 |
+
if list_day_objects(location_id=loc_id, year=year, month=m, max_days=1):
|
| 63 |
+
has_any = True
|
| 64 |
+
break
|
| 65 |
+
if has_any:
|
| 66 |
+
selected.append(loc_id)
|
| 67 |
+
if len(selected) >= max_locations:
|
| 68 |
+
break
|
| 69 |
+
|
| 70 |
+
location_ids = selected
|
| 71 |
+
|
| 72 |
+
pm25_rows: list[pd.DataFrame] = []
|
| 73 |
+
catalogs: list[pd.DataFrame] = []
|
| 74 |
+
|
| 75 |
+
for loc_id in tqdm(location_ids, desc="Downloading locations"):
|
| 76 |
+
objects: list = []
|
| 77 |
+
for m in months:
|
| 78 |
+
objects.extend(list_day_objects(location_id=loc_id, year=year, month=m, max_days=max_days_per_location))
|
| 79 |
+
if not objects:
|
| 80 |
+
continue
|
| 81 |
+
rng.shuffle(objects)
|
| 82 |
+
objects = objects[:max_days_per_location]
|
| 83 |
+
|
| 84 |
+
loc_measurements: list[pd.DataFrame] = []
|
| 85 |
+
for obj in objects:
|
| 86 |
+
try:
|
| 87 |
+
local_path = cache_dir / Path(obj.key)
|
| 88 |
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
| 89 |
+
if local_path.exists():
|
| 90 |
+
blob = local_path.read_bytes()
|
| 91 |
+
else:
|
| 92 |
+
blob = download_object(obj.key)
|
| 93 |
+
local_path.write_bytes(blob)
|
| 94 |
+
except Exception:
|
| 95 |
+
continue
|
| 96 |
+
try:
|
| 97 |
+
meas = read_measurements_csv_gz(blob)
|
| 98 |
+
except Exception:
|
| 99 |
+
continue
|
| 100 |
+
loc_measurements.append(meas)
|
| 101 |
+
|
| 102 |
+
if not loc_measurements:
|
| 103 |
+
continue
|
| 104 |
+
cat = extract_location_catalog(loc_measurements)
|
| 105 |
+
catalogs.append(cat)
|
| 106 |
+
|
| 107 |
+
daily_frames = [_daily_pm25_from_measurements(m) for m in loc_measurements]
|
| 108 |
+
daily_frames = [d for d in daily_frames if not d.empty]
|
| 109 |
+
if not daily_frames:
|
| 110 |
+
continue
|
| 111 |
+
loc_daily = pd.concat(daily_frames, ignore_index=True)
|
| 112 |
+
pm25_rows.append(loc_daily)
|
| 113 |
+
|
| 114 |
+
if not pm25_rows:
|
| 115 |
+
raise RuntimeError("No PM2.5 data collected. Try different year/months or increase max_locations.")
|
| 116 |
+
|
| 117 |
+
pm25_daily = pd.concat(pm25_rows, ignore_index=True).drop_duplicates(subset=["location_id", "date"])
|
| 118 |
+
|
| 119 |
+
location_catalog = (
|
| 120 |
+
pd.concat(catalogs, ignore_index=True).drop_duplicates(subset=["location_id"])
|
| 121 |
+
if catalogs
|
| 122 |
+
else pm25_daily[["location_id", "location", "lat", "lon"]].drop_duplicates(subset=["location_id"])
|
| 123 |
+
)
|
| 124 |
+
if "country" not in location_catalog.columns:
|
| 125 |
+
location_catalog["country"] = "UNKNOWN"
|
| 126 |
+
location_catalog["country"] = location_catalog["country"].fillna("UNKNOWN")
|
| 127 |
+
|
| 128 |
+
df = build_training_frame(pm25_daily=pm25_daily)
|
| 129 |
+
df.to_parquet(out_path, index=False)
|
| 130 |
+
|
| 131 |
+
# Save a location catalog for the app/training artifacts
|
| 132 |
+
catalog_out = out_path.parent / "location_catalog.parquet"
|
| 133 |
+
location_catalog.to_parquet(catalog_out, index=False)
|