"""Polars-based streaming preprocessing pipeline with downcasting and night filter.""" from pathlib import Path from typing import Optional import polars as pl from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG def normalize_pv_time_series_lazy( parquet_path: str, metadata_lazy: pl.LazyFrame, filter_night: bool = True, sample_systems: Optional[list[int]] = None, compute_lags: bool = True, weather_df: Optional[pl.DataFrame] = None, resample_1h: bool = True, clean_outliers: bool = True, bad_data_csv_path: Optional[str] = "./data/bad_data.csv", ) -> pl.LazyFrame: """Read a PV parquet file lazily, join with metadata, compute capacity factor, downcast types to Float32/Float16/UInt32, join NWP weather, and apply night filter. Optionally cleans sensor outliers (bad_data.csv & negative generation) and resamples to 1-hour. """ pv_lazy = pl.scan_parquet(parquet_path) # Cast raw PV columns # uk_pv parquet contains: ss_id (i32), datetime_GMT (datetime[ns, UTC]), generation_Wh (f32) pv_prepared = pv_lazy.select([ pl.col("ss_id").cast(pl.UInt32).alias("system_id"), pl.col("datetime_GMT").cast(pl.Datetime("ms", "UTC")).alias("datetime_utc"), pl.col("generation_Wh").cast(pl.Float32).alias("generation_wh"), ]) if sample_systems is not None and len(sample_systems) > 0: pv_prepared = pv_prepared.filter(pl.col("system_id").is_in(sample_systems)) # Pre-training Outlier Validation: Drop negative values & blacklisted bad data intervals if clean_outliers: from solarlab.processing.data_cleaner import filter_pv_outliers_lazy pv_prepared = filter_pv_outliers_lazy(pv_prepared, bad_data_csv_path=bad_data_csv_path) # 1-Hour Temporal Resampling (Option 2): # Truncate 30-min timestamps to 1-hour intervals and sum generation_wh if resample_1h: pv_prepared = ( pv_prepared.with_columns( pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc") ) .group_by(["system_id", "datetime_utc"]) .agg([ pl.col("generation_wh").sum().cast(pl.Float32).alias("generation_wh"), ]) ) # Inner join with metadata (already contains capacity_kw, latitude, longitude, azimuth, tilt) joined = pv_prepared.join(metadata_lazy, on="system_id", how="inner") # If NWP weather table is provided, map plants to closest 2D spatial zones (UK & Korea) if weather_df is not None and not weather_df.is_empty(): from solarlab.ingestion.weather_client import assign_nearest_grid_lazy joined = assign_nearest_grid_lazy(joined) w_df = weather_df if resample_1h: w_df = ( weather_df.with_columns( pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc") ) .group_by(["region", "datetime_utc"]) .agg([ pl.col("weather_ghi").mean().cast(pl.Float32), pl.col("cloud_cover").mean().cast(pl.Float16), pl.col("temperature").mean().cast(pl.Float16), ]) ) weather_lazy = w_df.lazy() joined = ( joined.join(weather_lazy, on=["region", "datetime_utc"], how="left") .drop("region") .with_columns([ pl.col("weather_ghi").fill_null(0.0).cast(pl.Float32), pl.col("cloud_cover").fill_null(50.0).cast(pl.Float16), pl.col("temperature").fill_null(15.0).cast(pl.Float16), ]) ) # Capacity factor calculation: # 1h: generation_Wh / (kWp * 1000 W/kW * 1.0 h) # 30m: generation_Wh / (kWp * 1000 W/kW * 0.5 h) # Clip between 0.0 and 1.0 to eliminate transient measurement spikes interval_hours = 1.0 if resample_1h else 0.5 with_cf = joined.with_columns( (pl.col("generation_wh") / (pl.col("capacity_kw") * 1000.0 * interval_hours)) .fill_nan(0.0) .clip(0.0, 1.0) .cast(pl.Float32) .alias("capacity_factor") ) # Extract astronomical time variables for high-throughput SZA approximation with_time = with_cf.with_columns([ pl.col("datetime_utc").dt.ordinal_day().cast(pl.UInt16).alias("doy"), ( pl.col("datetime_utc").dt.hour().cast(pl.Float32) + pl.col("datetime_utc").dt.minute().cast(pl.Float32) / 60.0 ).alias("hour_float"), ]) # Fast astronomical solar zenith approximation in Polars expressions: # Cooper (1969): declination = 23.45 * sin(2pi/365 * (284 + doy)) # hour_angle = (hour_utc + lon/15 - 12) * 15 deg # cos_sza = sin(lat)*sin(dec) + cos(lat)*cos(dec)*cos(h) pi = 3.141592653589793 rad_factor = pi / 180.0 deg_factor = 180.0 / pi dec_angle = (2.0 * pi / 365.0) * (pl.col("doy").cast(pl.Float32) + 284.0) declination_rad = (23.45 * rad_factor) * dec_angle.sin() lat_rad = pl.col("latitude") * rad_factor solar_time_hours = pl.col("hour_float") + (pl.col("longitude") / 15.0) hour_angle_rad = (solar_time_hours - 12.0) * 15.0 * rad_factor cos_sza = ( lat_rad.sin() * declination_rad.sin() + lat_rad.cos() * declination_rad.cos() * hour_angle_rad.cos() ).clip(-1.0, 1.0) # Approximate zenith in degrees sza_deg = cos_sza.arccos() * deg_factor # Approximate clear sky GHI: GHI = 1361.0 * max(0, cos_sza)^1.15 clearsky_ghi = ( pl.when(sza_deg < NIGHT_ZENITH_THRESHOLD_DEG) .then(1100.0 * (cos_sza.clip(0.0, 1.0) ** 1.15)) .otherwise(0.0) .cast(pl.Float32) ) result = with_time.with_columns([ sza_deg.cast(pl.Float16).alias("solar_zenith"), clearsky_ghi.alias("clearsky_ghi"), ]).drop(["doy", "hour_float"]) if compute_lags: from solarlab.features.feature_pipeline import add_lags_and_clearness_lazy result = add_lags_and_clearness_lazy(result, interval="1h" if resample_1h else "30m") if clean_outliers: from solarlab.processing.data_cleaner import filter_physical_outliers_lazy result = filter_physical_outliers_lazy(result) if filter_night: # Rule 3: Night-time Zeroing Filter: Drop rows where SZA > 88° or clearsky_ghi == 0.0 result = result.filter( (pl.col("solar_zenith") <= NIGHT_ZENITH_THRESHOLD_DEG) & (pl.col("clearsky_ghi") > 0.0) ) return result def collect_processed_dataset( parquet_path: str, metadata_lazy: pl.LazyFrame, filter_night: bool = True, sample_systems: Optional[list[int]] = None, output_path: Optional[str] = None, compute_lags: bool = True, weather_df: Optional[pl.DataFrame] = None, resample_1h: bool = True, clean_outliers: bool = True, bad_data_csv_path: Optional[str] = "./data/bad_data.csv", ) -> pl.DataFrame: """Execute lazy graph using Polars streaming engine (Rule 1). Persists output to parquet if output_path is provided. """ lazy_df = normalize_pv_time_series_lazy( parquet_path=parquet_path, metadata_lazy=metadata_lazy, filter_night=filter_night, sample_systems=sample_systems, compute_lags=compute_lags, weather_df=weather_df, resample_1h=resample_1h, clean_outliers=clean_outliers, bad_data_csv_path=bad_data_csv_path, ) df = lazy_df.collect(engine="streaming") if output_path: out_p = Path(output_path) out_p.parent.mkdir(parents=True, exist_ok=True) df.write_parquet(output_path, compression="zstd") return df def process_large_dataset_chunked( parquet_path: str, metadata_lazy: pl.LazyFrame, output_dir: str, chunk_size: int = 500, filter_night: bool = True, ) -> list[str]: """Process massive plant datasets in memory-safe chunks (preventing OOM on 8GB hosts). Splits plant IDs into chunks of `chunk_size` and persists partitioned parquets. """ out_dir = Path(output_dir) out_dir.mkdir(parents=True, exist_ok=True) all_systems = ( metadata_lazy.select("system_id") .collect()["system_id"] .to_list() ) saved_files = [] for i in range(0, len(all_systems), chunk_size): chunk_systems = all_systems[i : i + chunk_size] chunk_file = str(out_dir / f"chunk_{i // chunk_size:04d}.parquet") collect_processed_dataset( parquet_path=parquet_path, metadata_lazy=metadata_lazy, filter_night=filter_night, sample_systems=chunk_systems, output_path=chunk_file, compute_lags=True, ) saved_files.append(chunk_file) return saved_files