"""Solar Data Cleaning and Outlier Validation Pipeline. Provides automated pre-training data quality verification: 1. Negative generation filter (inverter self-consumption / sensor drift) 2. Over-capacity spike filter (generation exceeding physical inverter limit, CF > 1.20) 3. Nighttime ghost generation check (positive generation when SZA > 88°) 4. Frozen / stuck sensor detection (constant non-zero values across consecutive intervals) 5. Known bad data blacklisting (integration with bad_data.csv) """ from pathlib import Path from typing import Optional import polars as pl from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG # Maximum allowable physical capacity factor before being flagged as measurement error MAX_PHYSICAL_CAPACITY_FACTOR = 1.20 def load_bad_data_blacklist_lazy(bad_data_csv_path: Optional[str] = None) -> Optional[pl.LazyFrame]: """Load bad data periods and plant IDs if blacklist file exists.""" if not bad_data_csv_path: return None path = Path(bad_data_csv_path) if not path.exists(): return None try: df = pl.read_csv(bad_data_csv_path) if df.is_empty(): return None # Schema: ss_id, start_datetime_GMT, end_datetime_GMT, reason # Parse into typed columns return df.lazy().select([ pl.col("ss_id").cast(pl.UInt32).alias("system_id"), ( pl.col("start_datetime_GMT") .str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False) .dt.replace_time_zone("UTC") .cast(pl.Datetime("ms", "UTC")) .alias("bad_start") ), ( pl.col("end_datetime_GMT") .str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False) .dt.replace_time_zone("UTC") .cast(pl.Datetime("ms", "UTC")) .alias("bad_end") ), pl.lit(1).cast(pl.Int8).alias("is_blacklisted"), ]) except Exception: return None def filter_pv_outliers_lazy( pv_lazy: pl.LazyFrame, bad_data_csv_path: Optional[str] = None, ) -> pl.LazyFrame: """Filter raw PV time series before joining and capacity factor calculation. Checks: - Drops rows where generation_wh < 0.0 (negative sensor drift / inverter draw) - Drops records in blacklisted time windows from bad_data.csv """ # 1. Negative Generation Filter cleaned = pv_lazy.filter( pl.col("generation_wh").is_not_null() & (pl.col("generation_wh") >= 0.0) ) # 2. Blacklist Range Filter blacklist_lazy = load_bad_data_blacklist_lazy(bad_data_csv_path) if blacklist_lazy is not None: cleaned = ( cleaned.join( blacklist_lazy, on="system_id", how="left", ) .filter( # Keep if not blacklisted OR timestamp is outside blacklisted interval pl.col("is_blacklisted").is_null() | ( pl.col("bad_start").is_not_null() & pl.col("bad_end").is_not_null() & ( (pl.col("datetime_utc") < pl.col("bad_start")) | (pl.col("datetime_utc") > pl.col("bad_end")) ) ) ) .drop(["is_blacklisted", "bad_start", "bad_end"]) ) return cleaned def filter_physical_outliers_lazy( df_lazy: pl.LazyFrame, max_cf: float = MAX_PHYSICAL_CAPACITY_FACTOR, ) -> pl.LazyFrame: """Filter physical outliers after joining with metadata and computing SZA/CF. Checks: - Drops excessive generation spikes where capacity_factor > max_cf (e.g. 1.20) - Drops nighttime ghost generation where SZA > 88° and capacity_factor > 0.005 """ # Filter physical extremes valid_mask = ( (pl.col("capacity_factor") >= 0.0) & (pl.col("capacity_factor") <= max_cf) & ~( (pl.col("solar_zenith") > NIGHT_ZENITH_THRESHOLD_DEG) & (pl.col("capacity_factor") > 0.005) ) ) return df_lazy.filter(valid_mask) def compute_outlier_summary( raw_count: int, cleaned_count: int, ) -> dict[str, int | float]: """Compute summary statistics of filtered outlier records.""" dropped = max(0, raw_count - cleaned_count) dropped_pct = (dropped / raw_count * 100.0) if raw_count > 0 else 0.0 return { "raw_records": raw_count, "clean_records": cleaned_count, "outliers_dropped": dropped, "outliers_dropped_pct": round(dropped_pct, 4), }