| """ |
| One-off script: extract Stage 1 columns for growing season (May-Sep) |
| from sensors_wide.csv → sensors_wide_sample.csv (~2-3 MB). |
| """ |
|
|
| import sys |
| from pathlib import Path |
|
|
| PROJECT_ROOT = Path(__file__).resolve().parent.parent |
| if str(PROJECT_ROOT) not in sys.path: |
| sys.path.insert(0, str(PROJECT_ROOT)) |
|
|
| import pandas as pd |
|
|
| from config.settings import ( |
| GROWING_SEASON_MONTHS, |
| SENSORS_WIDE_PATH, |
| ) |
| from src.sensor_data_loader import DEFAULT_TIMESTAMP_COL, STAGE1_COLUMNS |
|
|
| SAMPLE_PATH = SENSORS_WIDE_PATH.parent / "sensors_wide_sample.csv" |
|
|
| |
| STAGE1_TEST_COLUMNS = [ |
| "Air2_PAR_test", |
| "Air2_leafTemperature_test", |
| "Air2_airTemperature_test", |
| "Air2_CO2_test", |
| "Air2_VPD_test", |
| ] |
|
|
|
|
| def main(): |
| if not SENSORS_WIDE_PATH.exists(): |
| print(f"Full CSV not found: {SENSORS_WIDE_PATH}") |
| sys.exit(1) |
|
|
| cols = [DEFAULT_TIMESTAMP_COL] + list(STAGE1_COLUMNS) + STAGE1_TEST_COLUMNS |
| print(f"Reading {SENSORS_WIDE_PATH} ...") |
| df = pd.read_csv(SENSORS_WIDE_PATH, usecols=lambda c: c in cols) |
| print(f" Total rows: {len(df)}") |
|
|
| df[DEFAULT_TIMESTAMP_COL] = pd.to_datetime(df[DEFAULT_TIMESTAMP_COL], utc=True) |
| df = df[df[DEFAULT_TIMESTAMP_COL].dt.month.isin(GROWING_SEASON_MONTHS)] |
| df = df.sort_values(DEFAULT_TIMESTAMP_COL).reset_index(drop=True) |
| print(f" Growing-season rows (months {GROWING_SEASON_MONTHS}): {len(df)}") |
|
|
| df.to_csv(SAMPLE_PATH, index=False) |
| size_mb = SAMPLE_PATH.stat().st_size / 1_000_000 |
| print(f"Saved {SAMPLE_PATH} ({size_mb:.1f} MB)") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
