File size: 8,829 Bytes
64431b0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bd2a7e9
7e7f389
 
64431b0
 
 
 
7e7f389
64431b0
 
 
 
 
 
 
 
 
 
 
 
 
 
7e7f389
 
 
 
 
 
bd2a7e9
 
 
 
 
 
 
 
 
 
 
 
 
64431b0
 
 
f03e4ae
64431b0
f03e4ae
64431b0
f03e4ae
bd2a7e9
 
 
 
 
 
 
 
 
 
 
 
 
 
f03e4ae
64431b0
 
 
 
 
 
 
 
 
 
 
bd2a7e9
 
64431b0
bd2a7e9
64431b0
bd2a7e9
64431b0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bd2a7e9
 
64431b0
7e7f389
 
 
 
 
64431b0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bd2a7e9
7e7f389
 
64431b0
 
 
 
 
 
 
 
 
 
 
 
bd2a7e9
7e7f389
 
64431b0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
"""Polars-based streaming preprocessing pipeline with downcasting and night filter."""

from pathlib import Path
from typing import Optional

import polars as pl

from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG


def normalize_pv_time_series_lazy(
    parquet_path: str,
    metadata_lazy: pl.LazyFrame,
    filter_night: bool = True,
    sample_systems: Optional[list[int]] = None,
    compute_lags: bool = True,
    weather_df: Optional[pl.DataFrame] = None,
    resample_1h: bool = True,
    clean_outliers: bool = True,
    bad_data_csv_path: Optional[str] = "./data/bad_data.csv",
) -> pl.LazyFrame:
    """Read a PV parquet file lazily, join with metadata, compute capacity factor,

    downcast types to Float32/Float16/UInt32, join NWP weather, and apply night filter.
    Optionally cleans sensor outliers (bad_data.csv & negative generation) and resamples to 1-hour.
    """
    pv_lazy = pl.scan_parquet(parquet_path)

    # Cast raw PV columns
    # uk_pv parquet contains: ss_id (i32), datetime_GMT (datetime[ns, UTC]), generation_Wh (f32)
    pv_prepared = pv_lazy.select([
        pl.col("ss_id").cast(pl.UInt32).alias("system_id"),
        pl.col("datetime_GMT").cast(pl.Datetime("ms", "UTC")).alias("datetime_utc"),
        pl.col("generation_Wh").cast(pl.Float32).alias("generation_wh"),
    ])

    if sample_systems is not None and len(sample_systems) > 0:
        pv_prepared = pv_prepared.filter(pl.col("system_id").is_in(sample_systems))

    # Pre-training Outlier Validation: Drop negative values & blacklisted bad data intervals
    if clean_outliers:
        from solarlab.processing.data_cleaner import filter_pv_outliers_lazy

        pv_prepared = filter_pv_outliers_lazy(pv_prepared, bad_data_csv_path=bad_data_csv_path)

    # 1-Hour Temporal Resampling (Option 2):
    # Truncate 30-min timestamps to 1-hour intervals and sum generation_wh
    if resample_1h:
        pv_prepared = (
            pv_prepared.with_columns(
                pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc")
            )
            .group_by(["system_id", "datetime_utc"])
            .agg([
                pl.col("generation_wh").sum().cast(pl.Float32).alias("generation_wh"),
            ])
        )

    # Inner join with metadata (already contains capacity_kw, latitude, longitude, azimuth, tilt)
    joined = pv_prepared.join(metadata_lazy, on="system_id", how="inner")

    # If NWP weather table is provided, map plants to closest 2D spatial zones (UK & Korea)
    if weather_df is not None and not weather_df.is_empty():
        from solarlab.ingestion.weather_client import assign_nearest_grid_lazy

        joined = assign_nearest_grid_lazy(joined)
        w_df = weather_df
        if resample_1h:
            w_df = (
                weather_df.with_columns(
                    pl.col("datetime_utc").dt.truncate("1h").alias("datetime_utc")
                )
                .group_by(["region", "datetime_utc"])
                .agg([
                    pl.col("weather_ghi").mean().cast(pl.Float32),
                    pl.col("cloud_cover").mean().cast(pl.Float16),
                    pl.col("temperature").mean().cast(pl.Float16),
                ])
            )
        weather_lazy = w_df.lazy()

        joined = (
            joined.join(weather_lazy, on=["region", "datetime_utc"], how="left")
            .drop("region")
            .with_columns([
                pl.col("weather_ghi").fill_null(0.0).cast(pl.Float32),
                pl.col("cloud_cover").fill_null(50.0).cast(pl.Float16),
                pl.col("temperature").fill_null(15.0).cast(pl.Float16),
            ])
        )

    # Capacity factor calculation:
    # 1h: generation_Wh / (kWp * 1000 W/kW * 1.0 h)
    # 30m: generation_Wh / (kWp * 1000 W/kW * 0.5 h)
    # Clip between 0.0 and 1.0 to eliminate transient measurement spikes
    interval_hours = 1.0 if resample_1h else 0.5
    with_cf = joined.with_columns(
        (pl.col("generation_wh") / (pl.col("capacity_kw") * 1000.0 * interval_hours))
        .fill_nan(0.0)
        .clip(0.0, 1.0)
        .cast(pl.Float32)
        .alias("capacity_factor")
    )


    # Extract astronomical time variables for high-throughput SZA approximation
    with_time = with_cf.with_columns([
        pl.col("datetime_utc").dt.ordinal_day().cast(pl.UInt16).alias("doy"),
        (
            pl.col("datetime_utc").dt.hour().cast(pl.Float32)
            + pl.col("datetime_utc").dt.minute().cast(pl.Float32) / 60.0
        ).alias("hour_float"),
    ])

    # Fast astronomical solar zenith approximation in Polars expressions:
    # Cooper (1969): declination = 23.45 * sin(2pi/365 * (284 + doy))
    # hour_angle = (hour_utc + lon/15 - 12) * 15 deg
    # cos_sza = sin(lat)*sin(dec) + cos(lat)*cos(dec)*cos(h)
    pi = 3.141592653589793
    rad_factor = pi / 180.0
    deg_factor = 180.0 / pi

    dec_angle = (2.0 * pi / 365.0) * (pl.col("doy").cast(pl.Float32) + 284.0)
    declination_rad = (23.45 * rad_factor) * dec_angle.sin()
    lat_rad = pl.col("latitude") * rad_factor

    solar_time_hours = pl.col("hour_float") + (pl.col("longitude") / 15.0)
    hour_angle_rad = (solar_time_hours - 12.0) * 15.0 * rad_factor

    cos_sza = (
        lat_rad.sin() * declination_rad.sin()
        + lat_rad.cos() * declination_rad.cos() * hour_angle_rad.cos()
    ).clip(-1.0, 1.0)

    # Approximate zenith in degrees
    sza_deg = cos_sza.arccos() * deg_factor

    # Approximate clear sky GHI: GHI = 1361.0 * max(0, cos_sza)^1.15
    clearsky_ghi = (
        pl.when(sza_deg < NIGHT_ZENITH_THRESHOLD_DEG)
        .then(1100.0 * (cos_sza.clip(0.0, 1.0) ** 1.15))
        .otherwise(0.0)
        .cast(pl.Float32)
    )

    result = with_time.with_columns([
        sza_deg.cast(pl.Float16).alias("solar_zenith"),
        clearsky_ghi.alias("clearsky_ghi"),
    ]).drop(["doy", "hour_float"])

    if compute_lags:
        from solarlab.features.feature_pipeline import add_lags_and_clearness_lazy

        result = add_lags_and_clearness_lazy(result, interval="1h" if resample_1h else "30m")

    if clean_outliers:
        from solarlab.processing.data_cleaner import filter_physical_outliers_lazy

        result = filter_physical_outliers_lazy(result)

    if filter_night:
        # Rule 3: Night-time Zeroing Filter: Drop rows where SZA > 88° or clearsky_ghi == 0.0
        result = result.filter(
            (pl.col("solar_zenith") <= NIGHT_ZENITH_THRESHOLD_DEG)
            & (pl.col("clearsky_ghi") > 0.0)
        )

    return result


def collect_processed_dataset(
    parquet_path: str,
    metadata_lazy: pl.LazyFrame,
    filter_night: bool = True,
    sample_systems: Optional[list[int]] = None,
    output_path: Optional[str] = None,
    compute_lags: bool = True,
    weather_df: Optional[pl.DataFrame] = None,
    resample_1h: bool = True,
    clean_outliers: bool = True,
    bad_data_csv_path: Optional[str] = "./data/bad_data.csv",
) -> pl.DataFrame:
    """Execute lazy graph using Polars streaming engine (Rule 1).

    Persists output to parquet if output_path is provided.
    """
    lazy_df = normalize_pv_time_series_lazy(
        parquet_path=parquet_path,
        metadata_lazy=metadata_lazy,
        filter_night=filter_night,
        sample_systems=sample_systems,
        compute_lags=compute_lags,
        weather_df=weather_df,
        resample_1h=resample_1h,
        clean_outliers=clean_outliers,
        bad_data_csv_path=bad_data_csv_path,
    )

    df = lazy_df.collect(engine="streaming")

    if output_path:
        out_p = Path(output_path)
        out_p.parent.mkdir(parents=True, exist_ok=True)
        df.write_parquet(output_path, compression="zstd")

    return df


def process_large_dataset_chunked(
    parquet_path: str,
    metadata_lazy: pl.LazyFrame,
    output_dir: str,
    chunk_size: int = 500,
    filter_night: bool = True,
) -> list[str]:
    """Process massive plant datasets in memory-safe chunks (preventing OOM on 8GB hosts).

    Splits plant IDs into chunks of `chunk_size` and persists partitioned parquets.
    """
    out_dir = Path(output_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    all_systems = (
        metadata_lazy.select("system_id")
        .collect()["system_id"]
        .to_list()
    )

    saved_files = []
    for i in range(0, len(all_systems), chunk_size):
        chunk_systems = all_systems[i : i + chunk_size]
        chunk_file = str(out_dir / f"chunk_{i // chunk_size:04d}.parquet")

        collect_processed_dataset(
            parquet_path=parquet_path,
            metadata_lazy=metadata_lazy,
            filter_night=filter_night,
            sample_systems=chunk_systems,
            output_path=chunk_file,
            compute_lags=True,
        )
        saved_files.append(chunk_file)

    return saved_files