Spaces:
Sleeping
Sleeping
Download src/solarlab/processing/data_cleaner.py from basementparking/solarlab-enterprise: direct link, hf CLI and curl.
- Browser
- Download file 4.68 kB
-
https://huggingface.co/spaces/basementparking/solarlab-enterprise/resolve/main/src/solarlab/processing/data_cleaner.py
- Command line
-
hf download hf://spaces/basementparking/solarlab-enterprise/src/solarlab/processing/data_cleaner.py
-
curl -L -o data_cleaner.py https://huggingface.co/spaces/basementparking/solarlab-enterprise/resolve/main/src/solarlab/processing/data_cleaner.py
4.68 kB
| """Solar Data Cleaning and Outlier Validation Pipeline. | |
| Provides automated pre-training data quality verification: | |
| 1. Negative generation filter (inverter self-consumption / sensor drift) | |
| 2. Over-capacity spike filter (generation exceeding physical inverter limit, CF > 1.20) | |
| 3. Nighttime ghost generation check (positive generation when SZA > 88°) | |
| 4. Frozen / stuck sensor detection (constant non-zero values across consecutive intervals) | |
| 5. Known bad data blacklisting (integration with bad_data.csv) | |
| """ | |
| from pathlib import Path | |
| from typing import Optional | |
| import polars as pl | |
| from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG | |
| # Maximum allowable physical capacity factor before being flagged as measurement error | |
| MAX_PHYSICAL_CAPACITY_FACTOR = 1.20 | |
| def load_bad_data_blacklist_lazy(bad_data_csv_path: Optional[str] = None) -> Optional[pl.LazyFrame]: | |
| """Load bad data periods and plant IDs if blacklist file exists.""" | |
| if not bad_data_csv_path: | |
| return None | |
| path = Path(bad_data_csv_path) | |
| if not path.exists(): | |
| return None | |
| try: | |
| df = pl.read_csv(bad_data_csv_path) | |
| if df.is_empty(): | |
| return None | |
| # Schema: ss_id, start_datetime_GMT, end_datetime_GMT, reason | |
| # Parse into typed columns | |
| return df.lazy().select([ | |
| pl.col("ss_id").cast(pl.UInt32).alias("system_id"), | |
| ( | |
| pl.col("start_datetime_GMT") | |
| .str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False) | |
| .dt.replace_time_zone("UTC") | |
| .cast(pl.Datetime("ms", "UTC")) | |
| .alias("bad_start") | |
| ), | |
| ( | |
| pl.col("end_datetime_GMT") | |
| .str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False) | |
| .dt.replace_time_zone("UTC") | |
| .cast(pl.Datetime("ms", "UTC")) | |
| .alias("bad_end") | |
| ), | |
| pl.lit(1).cast(pl.Int8).alias("is_blacklisted"), | |
| ]) | |
| except Exception: | |
| return None | |
| def filter_pv_outliers_lazy( | |
| pv_lazy: pl.LazyFrame, | |
| bad_data_csv_path: Optional[str] = None, | |
| ) -> pl.LazyFrame: | |
| """Filter raw PV time series before joining and capacity factor calculation. | |
| Checks: | |
| - Drops rows where generation_wh < 0.0 (negative sensor drift / inverter draw) | |
| - Drops records in blacklisted time windows from bad_data.csv | |
| """ | |
| # 1. Negative Generation Filter | |
| cleaned = pv_lazy.filter( | |
| pl.col("generation_wh").is_not_null() | |
| & (pl.col("generation_wh") >= 0.0) | |
| ) | |
| # 2. Blacklist Range Filter | |
| blacklist_lazy = load_bad_data_blacklist_lazy(bad_data_csv_path) | |
| if blacklist_lazy is not None: | |
| cleaned = ( | |
| cleaned.join( | |
| blacklist_lazy, | |
| on="system_id", | |
| how="left", | |
| ) | |
| .filter( | |
| # Keep if not blacklisted OR timestamp is outside blacklisted interval | |
| pl.col("is_blacklisted").is_null() | |
| | ( | |
| pl.col("bad_start").is_not_null() | |
| & pl.col("bad_end").is_not_null() | |
| & ( | |
| (pl.col("datetime_utc") < pl.col("bad_start")) | |
| | (pl.col("datetime_utc") > pl.col("bad_end")) | |
| ) | |
| ) | |
| ) | |
| .drop(["is_blacklisted", "bad_start", "bad_end"]) | |
| ) | |
| return cleaned | |
| def filter_physical_outliers_lazy( | |
| df_lazy: pl.LazyFrame, | |
| max_cf: float = MAX_PHYSICAL_CAPACITY_FACTOR, | |
| ) -> pl.LazyFrame: | |
| """Filter physical outliers after joining with metadata and computing SZA/CF. | |
| Checks: | |
| - Drops excessive generation spikes where capacity_factor > max_cf (e.g. 1.20) | |
| - Drops nighttime ghost generation where SZA > 88° and capacity_factor > 0.005 | |
| """ | |
| # Filter physical extremes | |
| valid_mask = ( | |
| (pl.col("capacity_factor") >= 0.0) | |
| & (pl.col("capacity_factor") <= max_cf) | |
| & ~( | |
| (pl.col("solar_zenith") > NIGHT_ZENITH_THRESHOLD_DEG) | |
| & (pl.col("capacity_factor") > 0.005) | |
| ) | |
| ) | |
| return df_lazy.filter(valid_mask) | |
| def compute_outlier_summary( | |
| raw_count: int, | |
| cleaned_count: int, | |
| ) -> dict[str, int | float]: | |
| """Compute summary statistics of filtered outlier records.""" | |
| dropped = max(0, raw_count - cleaned_count) | |
| dropped_pct = (dropped / raw_count * 100.0) if raw_count > 0 else 0.0 | |
| return { | |
| "raw_records": raw_count, | |
| "clean_records": cleaned_count, | |
| "outliers_dropped": dropped, | |
| "outliers_dropped_pct": round(dropped_pct, 4), | |
| } | |