basementparking's picture
Upload folder using huggingface_hub
7e7f389 verified
Raw History Blame Contribute Delete
4.68 kB
"""Solar Data Cleaning and Outlier Validation Pipeline.
Provides automated pre-training data quality verification:
1. Negative generation filter (inverter self-consumption / sensor drift)
2. Over-capacity spike filter (generation exceeding physical inverter limit, CF > 1.20)
3. Nighttime ghost generation check (positive generation when SZA > 88°)
4. Frozen / stuck sensor detection (constant non-zero values across consecutive intervals)
5. Known bad data blacklisting (integration with bad_data.csv)
"""
from pathlib import Path
from typing import Optional
import polars as pl
from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG
# Maximum allowable physical capacity factor before being flagged as measurement error
MAX_PHYSICAL_CAPACITY_FACTOR = 1.20
def load_bad_data_blacklist_lazy(bad_data_csv_path: Optional[str] = None) -> Optional[pl.LazyFrame]:
"""Load bad data periods and plant IDs if blacklist file exists."""
if not bad_data_csv_path:
return None
path = Path(bad_data_csv_path)
if not path.exists():
return None
try:
df = pl.read_csv(bad_data_csv_path)
if df.is_empty():
return None
# Schema: ss_id, start_datetime_GMT, end_datetime_GMT, reason
# Parse into typed columns
return df.lazy().select([
pl.col("ss_id").cast(pl.UInt32).alias("system_id"),
(
pl.col("start_datetime_GMT")
.str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False)
.dt.replace_time_zone("UTC")
.cast(pl.Datetime("ms", "UTC"))
.alias("bad_start")
),
(
pl.col("end_datetime_GMT")
.str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False)
.dt.replace_time_zone("UTC")
.cast(pl.Datetime("ms", "UTC"))
.alias("bad_end")
),
pl.lit(1).cast(pl.Int8).alias("is_blacklisted"),
])
except Exception:
return None
def filter_pv_outliers_lazy(
pv_lazy: pl.LazyFrame,
bad_data_csv_path: Optional[str] = None,
) -> pl.LazyFrame:
"""Filter raw PV time series before joining and capacity factor calculation.
Checks:
- Drops rows where generation_wh < 0.0 (negative sensor drift / inverter draw)
- Drops records in blacklisted time windows from bad_data.csv
"""
# 1. Negative Generation Filter
cleaned = pv_lazy.filter(
pl.col("generation_wh").is_not_null()
& (pl.col("generation_wh") >= 0.0)
)
# 2. Blacklist Range Filter
blacklist_lazy = load_bad_data_blacklist_lazy(bad_data_csv_path)
if blacklist_lazy is not None:
cleaned = (
cleaned.join(
blacklist_lazy,
on="system_id",
how="left",
)
.filter(
# Keep if not blacklisted OR timestamp is outside blacklisted interval
pl.col("is_blacklisted").is_null()
| (
pl.col("bad_start").is_not_null()
& pl.col("bad_end").is_not_null()
& (
(pl.col("datetime_utc") < pl.col("bad_start"))
| (pl.col("datetime_utc") > pl.col("bad_end"))
)
)
)
.drop(["is_blacklisted", "bad_start", "bad_end"])
)
return cleaned
def filter_physical_outliers_lazy(
df_lazy: pl.LazyFrame,
max_cf: float = MAX_PHYSICAL_CAPACITY_FACTOR,
) -> pl.LazyFrame:
"""Filter physical outliers after joining with metadata and computing SZA/CF.
Checks:
- Drops excessive generation spikes where capacity_factor > max_cf (e.g. 1.20)
- Drops nighttime ghost generation where SZA > 88° and capacity_factor > 0.005
"""
# Filter physical extremes
valid_mask = (
(pl.col("capacity_factor") >= 0.0)
& (pl.col("capacity_factor") <= max_cf)
& ~(
(pl.col("solar_zenith") > NIGHT_ZENITH_THRESHOLD_DEG)
& (pl.col("capacity_factor") > 0.005)
)
)
return df_lazy.filter(valid_mask)
def compute_outlier_summary(
raw_count: int,
cleaned_count: int,
) -> dict[str, int | float]:
"""Compute summary statistics of filtered outlier records."""
dropped = max(0, raw_count - cleaned_count)
dropped_pct = (dropped / raw_count * 100.0) if raw_count > 0 else 0.0
return {
"raw_records": raw_count,
"clean_records": cleaned_count,
"outliers_dropped": dropped,
"outliers_dropped_pct": round(dropped_pct, 4),
}