solarlab-enterprise / src /solarlab /processing /pipeline_processing.py
basementparking's picture
Upload folder using huggingface_hub
7e7f389 verified
Raw History Blame Contribute Delete
8.83 kB
"""Polars-based streaming preprocessing pipeline with downcasting and night filter."""
from pathlib import Path
from typing import Optional
import polars as pl
from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG
def normalize_pv_time_series_lazy(
parquet_path: str,
metadata_lazy: pl.LazyFrame,
filter_night: bool = True,
sample_systems: Optional[list[int]] = None,
compute_lags: bool = True,
weather_df: Optional[pl.DataFrame] = None,
resample_1h: bool = True,
clean_outliers: bool = True,
bad_data_csv_path: Optional[str] = "./data/bad_data.csv",
) -> pl.LazyFrame:
"""Read a PV parquet file lazily, join with metadata, compute capacity factor,
downcast types to Float32/Float16/UInt32, join NWP weather, and apply night filter.
Optionally cleans sensor outliers (bad_data.csv & negative generation) and resamples to 1-hour.
"""
pv_lazy = pl.scan_parquet(parquet_path)
# Cast raw PV columns
# uk_pv parquet contains: ss_id (i32), datetime_GMT (datetime[ns, UTC]), generation_Wh (f32)
pv_prepared = pv_lazy.select([
pl.col("ss_id").cast(pl.UInt32).alias("system_id"),
pl.col("datetime_GMT").cast(pl.Datetime("ms", "UTC")).alias("datetime_utc"),
pl.col("generation_Wh").cast(pl.Float32).alias("generation_wh"),
])
if sample_systems is not None and len(sample_systems) > 0:
pv_prepared = pv_prepared.filter(pl.col("system_id").is_in(sample_systems))
# Pre-training Outlier Validation: Drop negative values & blacklisted bad data intervals
if clean_outliers:
from solarlab.processing.data_cleaner import filter_pv_outliers_lazy
pv_prepared = filter_pv_outliers_lazy(pv_prepared, bad_data_csv_path=bad_data_csv_path)
# 1-Hour Temporal Resampling (Option 2):
# Truncate 30-min timestamps to 1-hour intervals and sum generation_wh
if resample_1h:
pv_prepared = (
pv_prepared.with_columns(
pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc")
)
.group_by(["system_id", "datetime_utc"])
.agg([
pl.col("generation_wh").sum().cast(pl.Float32).alias("generation_wh"),
])
)
# Inner join with metadata (already contains capacity_kw, latitude, longitude, azimuth, tilt)
joined = pv_prepared.join(metadata_lazy, on="system_id", how="inner")
# If NWP weather table is provided, map plants to closest 2D spatial zones (UK & Korea)
if weather_df is not None and not weather_df.is_empty():
from solarlab.ingestion.weather_client import assign_nearest_grid_lazy
joined = assign_nearest_grid_lazy(joined)
w_df = weather_df
if resample_1h:
w_df = (
weather_df.with_columns(
pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc")
)
.group_by(["region", "datetime_utc"])
.agg([
pl.col("weather_ghi").mean().cast(pl.Float32),
pl.col("cloud_cover").mean().cast(pl.Float16),
pl.col("temperature").mean().cast(pl.Float16),
])
)
weather_lazy = w_df.lazy()
joined = (
joined.join(weather_lazy, on=["region", "datetime_utc"], how="left")
.drop("region")
.with_columns([
pl.col("weather_ghi").fill_null(0.0).cast(pl.Float32),
pl.col("cloud_cover").fill_null(50.0).cast(pl.Float16),
pl.col("temperature").fill_null(15.0).cast(pl.Float16),
])
)
# Capacity factor calculation:
# 1h: generation_Wh / (kWp * 1000 W/kW * 1.0 h)
# 30m: generation_Wh / (kWp * 1000 W/kW * 0.5 h)
# Clip between 0.0 and 1.0 to eliminate transient measurement spikes
interval_hours = 1.0 if resample_1h else 0.5
with_cf = joined.with_columns(
(pl.col("generation_wh") / (pl.col("capacity_kw") * 1000.0 * interval_hours))
.fill_nan(0.0)
.clip(0.0, 1.0)
.cast(pl.Float32)
.alias("capacity_factor")
)
# Extract astronomical time variables for high-throughput SZA approximation
with_time = with_cf.with_columns([
pl.col("datetime_utc").dt.ordinal_day().cast(pl.UInt16).alias("doy"),
(
pl.col("datetime_utc").dt.hour().cast(pl.Float32)
+ pl.col("datetime_utc").dt.minute().cast(pl.Float32) / 60.0
).alias("hour_float"),
])
# Fast astronomical solar zenith approximation in Polars expressions:
# Cooper (1969): declination = 23.45 * sin(2pi/365 * (284 + doy))
# hour_angle = (hour_utc + lon/15 - 12) * 15 deg
# cos_sza = sin(lat)*sin(dec) + cos(lat)*cos(dec)*cos(h)
pi = 3.141592653589793
rad_factor = pi / 180.0
deg_factor = 180.0 / pi
dec_angle = (2.0 * pi / 365.0) * (pl.col("doy").cast(pl.Float32) + 284.0)
declination_rad = (23.45 * rad_factor) * dec_angle.sin()
lat_rad = pl.col("latitude") * rad_factor
solar_time_hours = pl.col("hour_float") + (pl.col("longitude") / 15.0)
hour_angle_rad = (solar_time_hours - 12.0) * 15.0 * rad_factor
cos_sza = (
lat_rad.sin() * declination_rad.sin()
+ lat_rad.cos() * declination_rad.cos() * hour_angle_rad.cos()
).clip(-1.0, 1.0)
# Approximate zenith in degrees
sza_deg = cos_sza.arccos() * deg_factor
# Approximate clear sky GHI: GHI = 1361.0 * max(0, cos_sza)^1.15
clearsky_ghi = (
pl.when(sza_deg < NIGHT_ZENITH_THRESHOLD_DEG)
.then(1100.0 * (cos_sza.clip(0.0, 1.0) ** 1.15))
.otherwise(0.0)
.cast(pl.Float32)
)
result = with_time.with_columns([
sza_deg.cast(pl.Float16).alias("solar_zenith"),
clearsky_ghi.alias("clearsky_ghi"),
]).drop(["doy", "hour_float"])
if compute_lags:
from solarlab.features.feature_pipeline import add_lags_and_clearness_lazy
result = add_lags_and_clearness_lazy(result, interval="1h" if resample_1h else "30m")
if clean_outliers:
from solarlab.processing.data_cleaner import filter_physical_outliers_lazy
result = filter_physical_outliers_lazy(result)
if filter_night:
# Rule 3: Night-time Zeroing Filter: Drop rows where SZA > 88° or clearsky_ghi == 0.0
result = result.filter(
(pl.col("solar_zenith") <= NIGHT_ZENITH_THRESHOLD_DEG)
& (pl.col("clearsky_ghi") > 0.0)
)
return result
def collect_processed_dataset(
parquet_path: str,
metadata_lazy: pl.LazyFrame,
filter_night: bool = True,
sample_systems: Optional[list[int]] = None,
output_path: Optional[str] = None,
compute_lags: bool = True,
weather_df: Optional[pl.DataFrame] = None,
resample_1h: bool = True,
clean_outliers: bool = True,
bad_data_csv_path: Optional[str] = "./data/bad_data.csv",
) -> pl.DataFrame:
"""Execute lazy graph using Polars streaming engine (Rule 1).
Persists output to parquet if output_path is provided.
"""
lazy_df = normalize_pv_time_series_lazy(
parquet_path=parquet_path,
metadata_lazy=metadata_lazy,
filter_night=filter_night,
sample_systems=sample_systems,
compute_lags=compute_lags,
weather_df=weather_df,
resample_1h=resample_1h,
clean_outliers=clean_outliers,
bad_data_csv_path=bad_data_csv_path,
)
df = lazy_df.collect(engine="streaming")
if output_path:
out_p = Path(output_path)
out_p.parent.mkdir(parents=True, exist_ok=True)
df.write_parquet(output_path, compression="zstd")
return df
def process_large_dataset_chunked(
parquet_path: str,
metadata_lazy: pl.LazyFrame,
output_dir: str,
chunk_size: int = 500,
filter_night: bool = True,
) -> list[str]:
"""Process massive plant datasets in memory-safe chunks (preventing OOM on 8GB hosts).
Splits plant IDs into chunks of `chunk_size` and persists partitioned parquets.
"""
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
all_systems = (
metadata_lazy.select("system_id")
.collect()["system_id"]
.to_list()
)
saved_files = []
for i in range(0, len(all_systems), chunk_size):
chunk_systems = all_systems[i : i + chunk_size]
chunk_file = str(out_dir / f"chunk_{i // chunk_size:04d}.parquet")
collect_processed_dataset(
parquet_path=parquet_path,
metadata_lazy=metadata_lazy,
filter_night=filter_night,
sample_systems=chunk_systems,
output_path=chunk_file,
compute_lags=True,
)
saved_files.append(chunk_file)
return saved_files