File size: 4,680 Bytes
7e7f389
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
"""Solar Data Cleaning and Outlier Validation Pipeline.

Provides automated pre-training data quality verification:
1. Negative generation filter (inverter self-consumption / sensor drift)
2. Over-capacity spike filter (generation exceeding physical inverter limit, CF > 1.20)
3. Nighttime ghost generation check (positive generation when SZA > 88°)
4. Frozen / stuck sensor detection (constant non-zero values across consecutive intervals)
5. Known bad data blacklisting (integration with bad_data.csv)
"""

from pathlib import Path
from typing import Optional

import polars as pl

from solarlab.geometry.solar_calculator import NIGHT_ZENITH_THRESHOLD_DEG

# Maximum allowable physical capacity factor before being flagged as measurement error
MAX_PHYSICAL_CAPACITY_FACTOR = 1.20


def load_bad_data_blacklist_lazy(bad_data_csv_path: Optional[str] = None) -> Optional[pl.LazyFrame]:
    """Load bad data periods and plant IDs if blacklist file exists."""
    if not bad_data_csv_path:
        return None

    path = Path(bad_data_csv_path)
    if not path.exists():
        return None

    try:
        df = pl.read_csv(bad_data_csv_path)
        if df.is_empty():
            return None

        # Schema: ss_id, start_datetime_GMT, end_datetime_GMT, reason
        # Parse into typed columns
        return df.lazy().select([
            pl.col("ss_id").cast(pl.UInt32).alias("system_id"),
            (
                pl.col("start_datetime_GMT")
                .str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False)
                .dt.replace_time_zone("UTC")
                .cast(pl.Datetime("ms", "UTC"))
                .alias("bad_start")
            ),
            (
                pl.col("end_datetime_GMT")
                .str.to_datetime("%Y-%m-%dT%H:%M:%SZ", strict=False)
                .dt.replace_time_zone("UTC")
                .cast(pl.Datetime("ms", "UTC"))
                .alias("bad_end")
            ),
            pl.lit(1).cast(pl.Int8).alias("is_blacklisted"),
        ])
    except Exception:
        return None


def filter_pv_outliers_lazy(
    pv_lazy: pl.LazyFrame,
    bad_data_csv_path: Optional[str] = None,
) -> pl.LazyFrame:
    """Filter raw PV time series before joining and capacity factor calculation.

    Checks:
    - Drops rows where generation_wh < 0.0 (negative sensor drift / inverter draw)
    - Drops records in blacklisted time windows from bad_data.csv
    """
    # 1. Negative Generation Filter
    cleaned = pv_lazy.filter(
        pl.col("generation_wh").is_not_null()
        & (pl.col("generation_wh") >= 0.0)
    )

    # 2. Blacklist Range Filter
    blacklist_lazy = load_bad_data_blacklist_lazy(bad_data_csv_path)
    if blacklist_lazy is not None:
        cleaned = (
            cleaned.join(
                blacklist_lazy,
                on="system_id",
                how="left",
            )
            .filter(
                # Keep if not blacklisted OR timestamp is outside blacklisted interval
                pl.col("is_blacklisted").is_null()
                | (
                    pl.col("bad_start").is_not_null()
                    & pl.col("bad_end").is_not_null()
                    & (
                        (pl.col("datetime_utc") < pl.col("bad_start"))
                        | (pl.col("datetime_utc") > pl.col("bad_end"))
                    )
                )
            )
            .drop(["is_blacklisted", "bad_start", "bad_end"])
        )

    return cleaned


def filter_physical_outliers_lazy(
    df_lazy: pl.LazyFrame,
    max_cf: float = MAX_PHYSICAL_CAPACITY_FACTOR,
) -> pl.LazyFrame:
    """Filter physical outliers after joining with metadata and computing SZA/CF.

    Checks:
    - Drops excessive generation spikes where capacity_factor > max_cf (e.g. 1.20)
    - Drops nighttime ghost generation where SZA > 88° and capacity_factor > 0.005
    """
    # Filter physical extremes
    valid_mask = (
        (pl.col("capacity_factor") >= 0.0)
        & (pl.col("capacity_factor") <= max_cf)
        & ~(
            (pl.col("solar_zenith") > NIGHT_ZENITH_THRESHOLD_DEG)
            & (pl.col("capacity_factor") > 0.005)
        )
    )

    return df_lazy.filter(valid_mask)


def compute_outlier_summary(
    raw_count: int,
    cleaned_count: int,
) -> dict[str, int | float]:
    """Compute summary statistics of filtered outlier records."""
    dropped = max(0, raw_count - cleaned_count)
    dropped_pct = (dropped / raw_count * 100.0) if raw_count > 0 else 0.0

    return {
        "raw_records": raw_count,
        "clean_records": cleaned_count,
        "outliers_dropped": dropped,
        "outliers_dropped_pct": round(dropped_pct, 4),
    }