Yousef Awad commited on
Commit
a4e5bd8
·
verified ·
1 Parent(s): 69327dc

Upload src/wqf7009_aqi/dataset.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/wqf7009_aqi/dataset.py +133 -0
src/wqf7009_aqi/dataset.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import random
4
+ from pathlib import Path
5
+
6
+ import pandas as pd
7
+ from tqdm import tqdm
8
+
9
+ from wqf7009_aqi.features import build_training_frame
10
+ from wqf7009_aqi.openaq_archive import (
11
+ download_object,
12
+ extract_location_catalog,
13
+ list_day_objects,
14
+ list_location_ids,
15
+ read_measurements_csv_gz,
16
+ )
17
+
18
+
19
+ def _daily_pm25_from_measurements(df: pd.DataFrame) -> pd.DataFrame:
20
+ df = df.copy()
21
+ if "parameter" not in df.columns:
22
+ return pd.DataFrame()
23
+ df = df[df["parameter"].astype(str).str.lower().eq("pm25")]
24
+ if df.empty:
25
+ return pd.DataFrame()
26
+
27
+ df["datetime"] = pd.to_datetime(df["datetime"], errors="coerce", utc=True)
28
+ df = df.dropna(subset=["datetime", "value", "location_id"])
29
+ df["date"] = df["datetime"].dt.date
30
+ out = (
31
+ df.groupby(["location_id", "location", "lat", "lon", "date"], as_index=False)["value"]
32
+ .mean()
33
+ .rename(columns={"value": "pm25"})
34
+ )
35
+ return out
36
+
37
+
38
+ def build_dataset(
39
+ *,
40
+ out_path: Path,
41
+ year: int,
42
+ months: list[int],
43
+ max_locations: int,
44
+ max_days_per_location: int,
45
+ seed: int,
46
+ cache_dir: Path,
47
+ scan_location_ids: int,
48
+ ) -> None:
49
+ out_path.parent.mkdir(parents=True, exist_ok=True)
50
+ cache_dir.mkdir(parents=True, exist_ok=True)
51
+
52
+ rng = random.Random(seed)
53
+ cache_ids = Path("data/cache/openaq_location_ids.json")
54
+ location_ids = list_location_ids(max_ids=scan_location_ids, cache_path=cache_ids)
55
+ rng.shuffle(location_ids)
56
+
57
+ # Select locations that actually have data for the requested year/months.
58
+ selected: list[int] = []
59
+ for loc_id in tqdm(location_ids, desc="Selecting locations"):
60
+ has_any = False
61
+ for m in months:
62
+ if list_day_objects(location_id=loc_id, year=year, month=m, max_days=1):
63
+ has_any = True
64
+ break
65
+ if has_any:
66
+ selected.append(loc_id)
67
+ if len(selected) >= max_locations:
68
+ break
69
+
70
+ location_ids = selected
71
+
72
+ pm25_rows: list[pd.DataFrame] = []
73
+ catalogs: list[pd.DataFrame] = []
74
+
75
+ for loc_id in tqdm(location_ids, desc="Downloading locations"):
76
+ objects: list = []
77
+ for m in months:
78
+ objects.extend(list_day_objects(location_id=loc_id, year=year, month=m, max_days=max_days_per_location))
79
+ if not objects:
80
+ continue
81
+ rng.shuffle(objects)
82
+ objects = objects[:max_days_per_location]
83
+
84
+ loc_measurements: list[pd.DataFrame] = []
85
+ for obj in objects:
86
+ try:
87
+ local_path = cache_dir / Path(obj.key)
88
+ local_path.parent.mkdir(parents=True, exist_ok=True)
89
+ if local_path.exists():
90
+ blob = local_path.read_bytes()
91
+ else:
92
+ blob = download_object(obj.key)
93
+ local_path.write_bytes(blob)
94
+ except Exception:
95
+ continue
96
+ try:
97
+ meas = read_measurements_csv_gz(blob)
98
+ except Exception:
99
+ continue
100
+ loc_measurements.append(meas)
101
+
102
+ if not loc_measurements:
103
+ continue
104
+ cat = extract_location_catalog(loc_measurements)
105
+ catalogs.append(cat)
106
+
107
+ daily_frames = [_daily_pm25_from_measurements(m) for m in loc_measurements]
108
+ daily_frames = [d for d in daily_frames if not d.empty]
109
+ if not daily_frames:
110
+ continue
111
+ loc_daily = pd.concat(daily_frames, ignore_index=True)
112
+ pm25_rows.append(loc_daily)
113
+
114
+ if not pm25_rows:
115
+ raise RuntimeError("No PM2.5 data collected. Try different year/months or increase max_locations.")
116
+
117
+ pm25_daily = pd.concat(pm25_rows, ignore_index=True).drop_duplicates(subset=["location_id", "date"])
118
+
119
+ location_catalog = (
120
+ pd.concat(catalogs, ignore_index=True).drop_duplicates(subset=["location_id"])
121
+ if catalogs
122
+ else pm25_daily[["location_id", "location", "lat", "lon"]].drop_duplicates(subset=["location_id"])
123
+ )
124
+ if "country" not in location_catalog.columns:
125
+ location_catalog["country"] = "UNKNOWN"
126
+ location_catalog["country"] = location_catalog["country"].fillna("UNKNOWN")
127
+
128
+ df = build_training_frame(pm25_daily=pm25_daily)
129
+ df.to_parquet(out_path, index=False)
130
+
131
+ # Save a location catalog for the app/training artifacts
132
+ catalog_out = out_path.parent / "location_catalog.parquet"
133
+ location_catalog.to_parquet(catalog_out, index=False)