Spaces:
Sleeping
Sleeping
File size: 8,829 Bytes
64431b0 bd2a7e9 7e7f389 64431b0 7e7f389 64431b0 7e7f389 bd2a7e9 64431b0 f03e4ae 64431b0 f03e4ae 64431b0 f03e4ae bd2a7e9 f03e4ae 64431b0 bd2a7e9 64431b0 bd2a7e9 64431b0 bd2a7e9 64431b0 bd2a7e9 64431b0 7e7f389 64431b0 bd2a7e9 7e7f389 64431b0 bd2a7e9 7e7f389 64431b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 | """Polars-based streaming preprocessing pipeline with downcasting and night filter."""
from pathlib import Path
from typing import Optional
import polars as pl
from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG
def normalize_pv_time_series_lazy(
parquet_path: str,
metadata_lazy: pl.LazyFrame,
filter_night: bool = True,
sample_systems: Optional[list[int]] = None,
compute_lags: bool = True,
weather_df: Optional[pl.DataFrame] = None,
resample_1h: bool = True,
clean_outliers: bool = True,
bad_data_csv_path: Optional[str] = "./data/bad_data.csv",
) -> pl.LazyFrame:
"""Read a PV parquet file lazily, join with metadata, compute capacity factor,
downcast types to Float32/Float16/UInt32, join NWP weather, and apply night filter.
Optionally cleans sensor outliers (bad_data.csv & negative generation) and resamples to 1-hour.
"""
pv_lazy = pl.scan_parquet(parquet_path)
# Cast raw PV columns
# uk_pv parquet contains: ss_id (i32), datetime_GMT (datetime[ns, UTC]), generation_Wh (f32)
pv_prepared = pv_lazy.select([
pl.col("ss_id").cast(pl.UInt32).alias("system_id"),
pl.col("datetime_GMT").cast(pl.Datetime("ms", "UTC")).alias("datetime_utc"),
pl.col("generation_Wh").cast(pl.Float32).alias("generation_wh"),
])
if sample_systems is not None and len(sample_systems) > 0:
pv_prepared = pv_prepared.filter(pl.col("system_id").is_in(sample_systems))
# Pre-training Outlier Validation: Drop negative values & blacklisted bad data intervals
if clean_outliers:
from solarlab.processing.data_cleaner import filter_pv_outliers_lazy
pv_prepared = filter_pv_outliers_lazy(pv_prepared, bad_data_csv_path=bad_data_csv_path)
# 1-Hour Temporal Resampling (Option 2):
# Truncate 30-min timestamps to 1-hour intervals and sum generation_wh
if resample_1h:
pv_prepared = (
pv_prepared.with_columns(
pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc")
)
.group_by(["system_id", "datetime_utc"])
.agg([
pl.col("generation_wh").sum().cast(pl.Float32).alias("generation_wh"),
])
)
# Inner join with metadata (already contains capacity_kw, latitude, longitude, azimuth, tilt)
joined = pv_prepared.join(metadata_lazy, on="system_id", how="inner")
# If NWP weather table is provided, map plants to closest 2D spatial zones (UK & Korea)
if weather_df is not None and not weather_df.is_empty():
from solarlab.ingestion.weather_client import assign_nearest_grid_lazy
joined = assign_nearest_grid_lazy(joined)
w_df = weather_df
if resample_1h:
w_df = (
weather_df.with_columns(
pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc")
)
.group_by(["region", "datetime_utc"])
.agg([
pl.col("weather_ghi").mean().cast(pl.Float32),
pl.col("cloud_cover").mean().cast(pl.Float16),
pl.col("temperature").mean().cast(pl.Float16),
])
)
weather_lazy = w_df.lazy()
joined = (
joined.join(weather_lazy, on=["region", "datetime_utc"], how="left")
.drop("region")
.with_columns([
pl.col("weather_ghi").fill_null(0.0).cast(pl.Float32),
pl.col("cloud_cover").fill_null(50.0).cast(pl.Float16),
pl.col("temperature").fill_null(15.0).cast(pl.Float16),
])
)
# Capacity factor calculation:
# 1h: generation_Wh / (kWp * 1000 W/kW * 1.0 h)
# 30m: generation_Wh / (kWp * 1000 W/kW * 0.5 h)
# Clip between 0.0 and 1.0 to eliminate transient measurement spikes
interval_hours = 1.0 if resample_1h else 0.5
with_cf = joined.with_columns(
(pl.col("generation_wh") / (pl.col("capacity_kw") * 1000.0 * interval_hours))
.fill_nan(0.0)
.clip(0.0, 1.0)
.cast(pl.Float32)
.alias("capacity_factor")
)
# Extract astronomical time variables for high-throughput SZA approximation
with_time = with_cf.with_columns([
pl.col("datetime_utc").dt.ordinal_day().cast(pl.UInt16).alias("doy"),
(
pl.col("datetime_utc").dt.hour().cast(pl.Float32)
+ pl.col("datetime_utc").dt.minute().cast(pl.Float32) / 60.0
).alias("hour_float"),
])
# Fast astronomical solar zenith approximation in Polars expressions:
# Cooper (1969): declination = 23.45 * sin(2pi/365 * (284 + doy))
# hour_angle = (hour_utc + lon/15 - 12) * 15 deg
# cos_sza = sin(lat)*sin(dec) + cos(lat)*cos(dec)*cos(h)
pi = 3.141592653589793
rad_factor = pi / 180.0
deg_factor = 180.0 / pi
dec_angle = (2.0 * pi / 365.0) * (pl.col("doy").cast(pl.Float32) + 284.0)
declination_rad = (23.45 * rad_factor) * dec_angle.sin()
lat_rad = pl.col("latitude") * rad_factor
solar_time_hours = pl.col("hour_float") + (pl.col("longitude") / 15.0)
hour_angle_rad = (solar_time_hours - 12.0) * 15.0 * rad_factor
cos_sza = (
lat_rad.sin() * declination_rad.sin()
+ lat_rad.cos() * declination_rad.cos() * hour_angle_rad.cos()
).clip(-1.0, 1.0)
# Approximate zenith in degrees
sza_deg = cos_sza.arccos() * deg_factor
# Approximate clear sky GHI: GHI = 1361.0 * max(0, cos_sza)^1.15
clearsky_ghi = (
pl.when(sza_deg < NIGHT_ZENITH_THRESHOLD_DEG)
.then(1100.0 * (cos_sza.clip(0.0, 1.0) ** 1.15))
.otherwise(0.0)
.cast(pl.Float32)
)
result = with_time.with_columns([
sza_deg.cast(pl.Float16).alias("solar_zenith"),
clearsky_ghi.alias("clearsky_ghi"),
]).drop(["doy", "hour_float"])
if compute_lags:
from solarlab.features.feature_pipeline import add_lags_and_clearness_lazy
result = add_lags_and_clearness_lazy(result, interval="1h" if resample_1h else "30m")
if clean_outliers:
from solarlab.processing.data_cleaner import filter_physical_outliers_lazy
result = filter_physical_outliers_lazy(result)
if filter_night:
# Rule 3: Night-time Zeroing Filter: Drop rows where SZA > 88° or clearsky_ghi == 0.0
result = result.filter(
(pl.col("solar_zenith") <= NIGHT_ZENITH_THRESHOLD_DEG)
& (pl.col("clearsky_ghi") > 0.0)
)
return result
def collect_processed_dataset(
parquet_path: str,
metadata_lazy: pl.LazyFrame,
filter_night: bool = True,
sample_systems: Optional[list[int]] = None,
output_path: Optional[str] = None,
compute_lags: bool = True,
weather_df: Optional[pl.DataFrame] = None,
resample_1h: bool = True,
clean_outliers: bool = True,
bad_data_csv_path: Optional[str] = "./data/bad_data.csv",
) -> pl.DataFrame:
"""Execute lazy graph using Polars streaming engine (Rule 1).
Persists output to parquet if output_path is provided.
"""
lazy_df = normalize_pv_time_series_lazy(
parquet_path=parquet_path,
metadata_lazy=metadata_lazy,
filter_night=filter_night,
sample_systems=sample_systems,
compute_lags=compute_lags,
weather_df=weather_df,
resample_1h=resample_1h,
clean_outliers=clean_outliers,
bad_data_csv_path=bad_data_csv_path,
)
df = lazy_df.collect(engine="streaming")
if output_path:
out_p = Path(output_path)
out_p.parent.mkdir(parents=True, exist_ok=True)
df.write_parquet(output_path, compression="zstd")
return df
def process_large_dataset_chunked(
parquet_path: str,
metadata_lazy: pl.LazyFrame,
output_dir: str,
chunk_size: int = 500,
filter_night: bool = True,
) -> list[str]:
"""Process massive plant datasets in memory-safe chunks (preventing OOM on 8GB hosts).
Splits plant IDs into chunks of `chunk_size` and persists partitioned parquets.
"""
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
all_systems = (
metadata_lazy.select("system_id")
.collect()["system_id"]
.to_list()
)
saved_files = []
for i in range(0, len(all_systems), chunk_size):
chunk_systems = all_systems[i : i + chunk_size]
chunk_file = str(out_dir / f"chunk_{i // chunk_size:04d}.parquet")
collect_processed_dataset(
parquet_path=parquet_path,
metadata_lazy=metadata_lazy,
filter_night=filter_night,
sample_systems=chunk_systems,
output_path=chunk_file,
compute_lags=True,
)
saved_files.append(chunk_file)
return saved_files
|