Spaces:
Sleeping
Sleeping
Download src/solarlab/processing/pipeline_processing.py from basementparking/solarlab-enterprise: direct link, hf CLI and curl.
- Browser
- Download file 8.83 kB
-
https://huggingface.co/spaces/basementparking/solarlab-enterprise/resolve/main/src/solarlab/processing/pipeline_processing.py
- Command line
-
hf download hf://spaces/basementparking/solarlab-enterprise/src/solarlab/processing/pipeline_processing.py
-
curl -L -o pipeline_processing.py https://huggingface.co/spaces/basementparking/solarlab-enterprise/resolve/main/src/solarlab/processing/pipeline_processing.py
8.83 kB
| """Polars-based streaming preprocessing pipeline with downcasting and night filter.""" | |
| from pathlib import Path | |
| from typing import Optional | |
| import polars as pl | |
| from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG | |
| def normalize_pv_time_series_lazy( | |
| parquet_path: str, | |
| metadata_lazy: pl.LazyFrame, | |
| filter_night: bool = True, | |
| sample_systems: Optional[list[int]] = None, | |
| compute_lags: bool = True, | |
| weather_df: Optional[pl.DataFrame] = None, | |
| resample_1h: bool = True, | |
| clean_outliers: bool = True, | |
| bad_data_csv_path: Optional[str] = "./data/bad_data.csv", | |
| ) -> pl.LazyFrame: | |
| """Read a PV parquet file lazily, join with metadata, compute capacity factor, | |
| downcast types to Float32/Float16/UInt32, join NWP weather, and apply night filter. | |
| Optionally cleans sensor outliers (bad_data.csv & negative generation) and resamples to 1-hour. | |
| """ | |
| pv_lazy = pl.scan_parquet(parquet_path) | |
| # Cast raw PV columns | |
| # uk_pv parquet contains: ss_id (i32), datetime_GMT (datetime[ns, UTC]), generation_Wh (f32) | |
| pv_prepared = pv_lazy.select([ | |
| pl.col("ss_id").cast(pl.UInt32).alias("system_id"), | |
| pl.col("datetime_GMT").cast(pl.Datetime("ms", "UTC")).alias("datetime_utc"), | |
| pl.col("generation_Wh").cast(pl.Float32).alias("generation_wh"), | |
| ]) | |
| if sample_systems is not None and len(sample_systems) > 0: | |
| pv_prepared = pv_prepared.filter(pl.col("system_id").is_in(sample_systems)) | |
| # Pre-training Outlier Validation: Drop negative values & blacklisted bad data intervals | |
| if clean_outliers: | |
| from solarlab.processing.data_cleaner import filter_pv_outliers_lazy | |
| pv_prepared = filter_pv_outliers_lazy(pv_prepared, bad_data_csv_path=bad_data_csv_path) | |
| # 1-Hour Temporal Resampling (Option 2): | |
| # Truncate 30-min timestamps to 1-hour intervals and sum generation_wh | |
| if resample_1h: | |
| pv_prepared = ( | |
| pv_prepared.with_columns( | |
| pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc") | |
| ) | |
| .group_by(["system_id", "datetime_utc"]) | |
| .agg([ | |
| pl.col("generation_wh").sum().cast(pl.Float32).alias("generation_wh"), | |
| ]) | |
| ) | |
| # Inner join with metadata (already contains capacity_kw, latitude, longitude, azimuth, tilt) | |
| joined = pv_prepared.join(metadata_lazy, on="system_id", how="inner") | |
| # If NWP weather table is provided, map plants to closest 2D spatial zones (UK & Korea) | |
| if weather_df is not None and not weather_df.is_empty(): | |
| from solarlab.ingestion.weather_client import assign_nearest_grid_lazy | |
| joined = assign_nearest_grid_lazy(joined) | |
| w_df = weather_df | |
| if resample_1h: | |
| w_df = ( | |
| weather_df.with_columns( | |
| pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc") | |
| ) | |
| .group_by(["region", "datetime_utc"]) | |
| .agg([ | |
| pl.col("weather_ghi").mean().cast(pl.Float32), | |
| pl.col("cloud_cover").mean().cast(pl.Float16), | |
| pl.col("temperature").mean().cast(pl.Float16), | |
| ]) | |
| ) | |
| weather_lazy = w_df.lazy() | |
| joined = ( | |
| joined.join(weather_lazy, on=["region", "datetime_utc"], how="left") | |
| .drop("region") | |
| .with_columns([ | |
| pl.col("weather_ghi").fill_null(0.0).cast(pl.Float32), | |
| pl.col("cloud_cover").fill_null(50.0).cast(pl.Float16), | |
| pl.col("temperature").fill_null(15.0).cast(pl.Float16), | |
| ]) | |
| ) | |
| # Capacity factor calculation: | |
| # 1h: generation_Wh / (kWp * 1000 W/kW * 1.0 h) | |
| # 30m: generation_Wh / (kWp * 1000 W/kW * 0.5 h) | |
| # Clip between 0.0 and 1.0 to eliminate transient measurement spikes | |
| interval_hours = 1.0 if resample_1h else 0.5 | |
| with_cf = joined.with_columns( | |
| (pl.col("generation_wh") / (pl.col("capacity_kw") * 1000.0 * interval_hours)) | |
| .fill_nan(0.0) | |
| .clip(0.0, 1.0) | |
| .cast(pl.Float32) | |
| .alias("capacity_factor") | |
| ) | |
| # Extract astronomical time variables for high-throughput SZA approximation | |
| with_time = with_cf.with_columns([ | |
| pl.col("datetime_utc").dt.ordinal_day().cast(pl.UInt16).alias("doy"), | |
| ( | |
| pl.col("datetime_utc").dt.hour().cast(pl.Float32) | |
| + pl.col("datetime_utc").dt.minute().cast(pl.Float32) / 60.0 | |
| ).alias("hour_float"), | |
| ]) | |
| # Fast astronomical solar zenith approximation in Polars expressions: | |
| # Cooper (1969): declination = 23.45 * sin(2pi/365 * (284 + doy)) | |
| # hour_angle = (hour_utc + lon/15 - 12) * 15 deg | |
| # cos_sza = sin(lat)*sin(dec) + cos(lat)*cos(dec)*cos(h) | |
| pi = 3.141592653589793 | |
| rad_factor = pi / 180.0 | |
| deg_factor = 180.0 / pi | |
| dec_angle = (2.0 * pi / 365.0) * (pl.col("doy").cast(pl.Float32) + 284.0) | |
| declination_rad = (23.45 * rad_factor) * dec_angle.sin() | |
| lat_rad = pl.col("latitude") * rad_factor | |
| solar_time_hours = pl.col("hour_float") + (pl.col("longitude") / 15.0) | |
| hour_angle_rad = (solar_time_hours - 12.0) * 15.0 * rad_factor | |
| cos_sza = ( | |
| lat_rad.sin() * declination_rad.sin() | |
| + lat_rad.cos() * declination_rad.cos() * hour_angle_rad.cos() | |
| ).clip(-1.0, 1.0) | |
| # Approximate zenith in degrees | |
| sza_deg = cos_sza.arccos() * deg_factor | |
| # Approximate clear sky GHI: GHI = 1361.0 * max(0, cos_sza)^1.15 | |
| clearsky_ghi = ( | |
| pl.when(sza_deg < NIGHT_ZENITH_THRESHOLD_DEG) | |
| .then(1100.0 * (cos_sza.clip(0.0, 1.0) ** 1.15)) | |
| .otherwise(0.0) | |
| .cast(pl.Float32) | |
| ) | |
| result = with_time.with_columns([ | |
| sza_deg.cast(pl.Float16).alias("solar_zenith"), | |
| clearsky_ghi.alias("clearsky_ghi"), | |
| ]).drop(["doy", "hour_float"]) | |
| if compute_lags: | |
| from solarlab.features.feature_pipeline import add_lags_and_clearness_lazy | |
| result = add_lags_and_clearness_lazy(result, interval="1h" if resample_1h else "30m") | |
| if clean_outliers: | |
| from solarlab.processing.data_cleaner import filter_physical_outliers_lazy | |
| result = filter_physical_outliers_lazy(result) | |
| if filter_night: | |
| # Rule 3: Night-time Zeroing Filter: Drop rows where SZA > 88° or clearsky_ghi == 0.0 | |
| result = result.filter( | |
| (pl.col("solar_zenith") <= NIGHT_ZENITH_THRESHOLD_DEG) | |
| & (pl.col("clearsky_ghi") > 0.0) | |
| ) | |
| return result | |
| def collect_processed_dataset( | |
| parquet_path: str, | |
| metadata_lazy: pl.LazyFrame, | |
| filter_night: bool = True, | |
| sample_systems: Optional[list[int]] = None, | |
| output_path: Optional[str] = None, | |
| compute_lags: bool = True, | |
| weather_df: Optional[pl.DataFrame] = None, | |
| resample_1h: bool = True, | |
| clean_outliers: bool = True, | |
| bad_data_csv_path: Optional[str] = "./data/bad_data.csv", | |
| ) -> pl.DataFrame: | |
| """Execute lazy graph using Polars streaming engine (Rule 1). | |
| Persists output to parquet if output_path is provided. | |
| """ | |
| lazy_df = normalize_pv_time_series_lazy( | |
| parquet_path=parquet_path, | |
| metadata_lazy=metadata_lazy, | |
| filter_night=filter_night, | |
| sample_systems=sample_systems, | |
| compute_lags=compute_lags, | |
| weather_df=weather_df, | |
| resample_1h=resample_1h, | |
| clean_outliers=clean_outliers, | |
| bad_data_csv_path=bad_data_csv_path, | |
| ) | |
| df = lazy_df.collect(engine="streaming") | |
| if output_path: | |
| out_p = Path(output_path) | |
| out_p.parent.mkdir(parents=True, exist_ok=True) | |
| df.write_parquet(output_path, compression="zstd") | |
| return df | |
| def process_large_dataset_chunked( | |
| parquet_path: str, | |
| metadata_lazy: pl.LazyFrame, | |
| output_dir: str, | |
| chunk_size: int = 500, | |
| filter_night: bool = True, | |
| ) -> list[str]: | |
| """Process massive plant datasets in memory-safe chunks (preventing OOM on 8GB hosts). | |
| Splits plant IDs into chunks of `chunk_size` and persists partitioned parquets. | |
| """ | |
| out_dir = Path(output_dir) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| all_systems = ( | |
| metadata_lazy.select("system_id") | |
| .collect()["system_id"] | |
| .to_list() | |
| ) | |
| saved_files = [] | |
| for i in range(0, len(all_systems), chunk_size): | |
| chunk_systems = all_systems[i : i + chunk_size] | |
| chunk_file = str(out_dir / f"chunk_{i // chunk_size:04d}.parquet") | |
| collect_processed_dataset( | |
| parquet_path=parquet_path, | |
| metadata_lazy=metadata_lazy, | |
| filter_night=filter_night, | |
| sample_systems=chunk_systems, | |
| output_path=chunk_file, | |
| compute_lags=True, | |
| ) | |
| saved_files.append(chunk_file) | |
| return saved_files | |