Download training_code/scripts/25_qa_cube.py from TrendeX90/expected-greenness: direct link, hf CLI and curl.
- Browser
- Download file 3.65 kB
-
https://huggingface.co/TrendeX90/expected-greenness/resolve/main/training_code/scripts/25_qa_cube.py
- Command line
-
hf download hf://TrendeX90/expected-greenness/training_code/scripts/25_qa_cube.py
-
curl -L -o 25_qa_cube.py https://huggingface.co/TrendeX90/expected-greenness/resolve/main/training_code/scripts/25_qa_cube.py
3.65 kB
| """Sanity checks on the built cube. Prints a report and writes data/cube/qa_report.json. | |
| Checks are descriptive (no thresholds tuned to make things pass); the report is meant to be read. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| CUBE = Path("/root/trendex-ml/data/cube") | |
| PLACES = {"Son-Kul pastures (N shore)": (41.90, 75.15), "Naryn city": (41.43, 75.99), | |
| "At-Bashy valley": (41.17, 75.80), "Suusamyr valley": (42.15, 74.00)} | |
| def nearest_cell(cells: pd.DataFrame, lat: float, lon: float) -> int: | |
| d = (cells["lat"] - lat) ** 2 + ((cells["lon"] - lon) * np.cos(np.deg2rad(lat))) ** 2 | |
| return int(cells.loc[d.idxmin(), "cell_id"]) | |
| def main() -> None: | |
| cells = pd.read_parquet(CUBE / "cells.parquet") | |
| rep: dict = {"cells": len(cells), "in_oblast": int(cells["in_oblast"].sum()), | |
| "boundary_conflict": int(cells["boundary_conflict"].sum()), | |
| "by_district": {str(k): int(v) for k, v in cells["district"].value_counts(dropna=False).items()}, | |
| "mask_reason": {str(k): int(v) for k, v in cells["mask_reason"].value_counts(dropna=False).items()}} | |
| num = ["elev_mean", "elev_std", "slope_mean", "steep_share", "tpi", "grass_frac", "water_frac", "ice_frac"] | |
| rep["static_ranges"] = cells[num].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict() | |
| rep["static_missing_share"] = cells[num + ["lc_type1"]].isna().mean().round(4).to_dict() | |
| monthly_dir = CUBE / "monthly" | |
| if monthly_dir.exists(): | |
| m = pd.read_parquet(monthly_dir) | |
| m["year"] = m["year"].astype(int) | |
| m = m.merge(cells[["cell_id", "in_oblast", "mask_reason"]], on="cell_id") | |
| past = m[m["in_oblast"] & m["mask_reason"].isna()] | |
| rep["monthly_rows"] = len(m) | |
| rep["years"] = [int(m["year"].min()), int(m["year"].max())] | |
| rep["ndvi_ok_share_by_month"] = past.groupby("month")["ndvi_ok"].mean().round(3).to_dict() | |
| cols = ["ndvi", "ndvi_aqua", "lst_day", "lst_night", "snow", "precip", "soil_moist", "swe"] | |
| rep["monthly_ranges"] = past[cols].describe(percentiles=[0.01, 0.5, 0.99]).round(3).to_dict() | |
| rep["monthly_missing_share_MaySep"] = past[past["month"].between(5, 9)][cols].isna().mean().round(3).to_dict() | |
| both = past[past["ndvi_ok"] & past["ndvi_aqua"].notna()] | |
| rep["terra_aqua_ndvi_corr"] = round(float(both["ndvi"].corr(both["ndvi_aqua"])), 3) if len(both) else None | |
| summer = past[past["month"].between(6, 8) & past["ndvi_ok"]] | |
| rep["oblast_summer_ndvi_by_year"] = summer.groupby("year")["ndvi"].mean().round(4).to_dict() | |
| rep["places"] = {} | |
| for name, (lat, lon) in PLACES.items(): | |
| cid = nearest_cell(cells, lat, lon) | |
| s = m[(m["cell_id"] == cid) & m["month"].between(6, 8)] | |
| row = cells[cells["cell_id"] == cid].iloc[0] | |
| rep["places"][name] = { | |
| "cell_id": cid, "district": None if pd.isna(row["district"]) else row["district"], | |
| "mask_reason": None if pd.isna(row["mask_reason"]) else row["mask_reason"], | |
| "elev_m": round(float(row["elev_mean"]), 0), | |
| "summer_ndvi_mean": round(float(s.loc[s["ndvi_ok"], "ndvi"].mean()), 3), | |
| "summer_precip_mm_mean": round(float(s["precip"].mean()), 1), | |
| "summer_lst_day_c_mean": round(float(s["lst_day"].mean()), 1)} | |
| (CUBE / "qa_report.json").write_text(json.dumps(rep, indent=2, default=str, ensure_ascii=False)) | |
| print(json.dumps(rep, indent=1, default=str, ensure_ascii=False)) | |
| if __name__ == "__main__": | |
| main() | |