Spaces:
Sleeping
Sleeping
File size: 4,680 Bytes
7e7f389 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | """Solar Data Cleaning and Outlier Validation Pipeline.
Provides automated pre-training data quality verification:
1. Negative generation filter (inverter self-consumption / sensor drift)
2. Over-capacity spike filter (generation exceeding physical inverter limit, CF > 1.20)
3. Nighttime ghost generation check (positive generation when SZA > 88°)
4. Frozen / stuck sensor detection (constant non-zero values across consecutive intervals)
5. Known bad data blacklisting (integration with bad_data.csv)
"""
from pathlib import Path
from typing import Optional
import polars as pl
from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG
# Maximum allowable physical capacity factor before being flagged as measurement error
MAX_PHYSICAL_CAPACITY_FACTOR = 1.20
def load_bad_data_blacklist_lazy(bad_data_csv_path: Optional[str] = None) -> Optional[pl.LazyFrame]:
"""Load bad data periods and plant IDs if blacklist file exists."""
if not bad_data_csv_path:
return None
path = Path(bad_data_csv_path)
if not path.exists():
return None
try:
df = pl.read_csv(bad_data_csv_path)
if df.is_empty():
return None
# Schema: ss_id, start_datetime_GMT, end_datetime_GMT, reason
# Parse into typed columns
return df.lazy().select([
pl.col("ss_id").cast(pl.UInt32).alias("system_id"),
(
pl.col("start_datetime_GMT")
.str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False)
.dt.replace_time_zone("UTC")
.cast(pl.Datetime("ms", "UTC"))
.alias("bad_start")
),
(
pl.col("end_datetime_GMT")
.str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False)
.dt.replace_time_zone("UTC")
.cast(pl.Datetime("ms", "UTC"))
.alias("bad_end")
),
pl.lit(1).cast(pl.Int8).alias("is_blacklisted"),
])
except Exception:
return None
def filter_pv_outliers_lazy(
pv_lazy: pl.LazyFrame,
bad_data_csv_path: Optional[str] = None,
) -> pl.LazyFrame:
"""Filter raw PV time series before joining and capacity factor calculation.
Checks:
- Drops rows where generation_wh < 0.0 (negative sensor drift / inverter draw)
- Drops records in blacklisted time windows from bad_data.csv
"""
# 1. Negative Generation Filter
cleaned = pv_lazy.filter(
pl.col("generation_wh").is_not_null()
& (pl.col("generation_wh") >= 0.0)
)
# 2. Blacklist Range Filter
blacklist_lazy = load_bad_data_blacklist_lazy(bad_data_csv_path)
if blacklist_lazy is not None:
cleaned = (
cleaned.join(
blacklist_lazy,
on="system_id",
how="left",
)
.filter(
# Keep if not blacklisted OR timestamp is outside blacklisted interval
pl.col("is_blacklisted").is_null()
| (
pl.col("bad_start").is_not_null()
& pl.col("bad_end").is_not_null()
& (
(pl.col("datetime_utc") < pl.col("bad_start"))
| (pl.col("datetime_utc") > pl.col("bad_end"))
)
)
)
.drop(["is_blacklisted", "bad_start", "bad_end"])
)
return cleaned
def filter_physical_outliers_lazy(
df_lazy: pl.LazyFrame,
max_cf: float = MAX_PHYSICAL_CAPACITY_FACTOR,
) -> pl.LazyFrame:
"""Filter physical outliers after joining with metadata and computing SZA/CF.
Checks:
- Drops excessive generation spikes where capacity_factor > max_cf (e.g. 1.20)
- Drops nighttime ghost generation where SZA > 88° and capacity_factor > 0.005
"""
# Filter physical extremes
valid_mask = (
(pl.col("capacity_factor") >= 0.0)
& (pl.col("capacity_factor") <= max_cf)
& ~(
(pl.col("solar_zenith") > NIGHT_ZENITH_THRESHOLD_DEG)
& (pl.col("capacity_factor") > 0.005)
)
)
return df_lazy.filter(valid_mask)
def compute_outlier_summary(
raw_count: int,
cleaned_count: int,
) -> dict[str, int | float]:
"""Compute summary statistics of filtered outlier records."""
dropped = max(0, raw_count - cleaned_count)
dropped_pct = (dropped / raw_count * 100.0) if raw_count > 0 else 0.0
return {
"raw_records": raw_count,
"clean_records": cleaned_count,
"outliers_dropped": dropped,
"outliers_dropped_pct": round(dropped_pct, 4),
}
|