import fnmatch import json import re import unicodedata import zipfile from pathlib import Path import pandas as pd from data.schema import Col ROOT_DIR = Path(__file__).resolve().parent.parent DEFAULT_CONFIG = ROOT_DIR / "config.json" # ============================================================ # CONFIG # ============================================================ def load_config( path: str | Path | None = None, ) -> dict: with open(path or DEFAULT_CONFIG, encoding="utf-8") as f: return json.load(f) def resolve_source_path( config: dict, path: str | Path | None = None, ) -> Path: """ Explicit path if given, otherwise the configured one (relative paths resolve against the project root). """ path = Path(path or config["source"]["path"]) if not path.is_absolute(): path = ROOT_DIR / path if not path.exists(): raise FileNotFoundError( f"Source file not found: {path}" ) return path # ============================================================ # ZIP ACCESS # ============================================================ def slugify(text: str) -> str: """ 'Parque Eólico de Coll del Moro' -> 'parque_eolico_de_coll_del_moro' """ ascii_text = ( unicodedata.normalize("NFKD", str(text)) .encode("ascii", "ignore") .decode("ascii") ) return re.sub( r"[^a-z0-9]+", "_", ascii_text.lower() ).strip("_") def read_parquets( zip_path: str | Path, pattern: str, ) -> dict[str, pd.DataFrame]: """ Read every parquet in the zip whose file name matches `pattern` (shell-style). Returns {file name: DataFrame}. """ frames = {} with zipfile.ZipFile(zip_path) as z: for member in z.namelist(): name = Path(member).name if fnmatch.fnmatch(name, pattern): with z.open(member) as f: frames[name] = pd.read_parquet(f) if not frames: raise ValueError( f"No file matching '{pattern}' in {Path(zip_path).name}." ) return frames def _require_columns(df, columns, source): missing = set(columns.values()) - set(df.columns) if missing: raise ValueError( f"{source} is missing columns {sorted(missing)}. " "Check the column mapping in config.json." ) def _to_local_date(series, timezone): """ Timestamps -> tz-naive calendar dates in the market timezone. """ series = pd.to_datetime(series) if series.dt.tz is None: series = series.dt.tz_localize("UTC") return ( series.dt.tz_convert(timezone) .dt.tz_localize(None) .dt.normalize() ) # ============================================================ # PRICES # ============================================================ def load_prices( zip_path: str | Path, config: dict, ) -> tuple[pd.DataFrame, pd.DataFrame]: """ Load forward-curve prices from the zip. Returns ------- prices : pd.DataFrame Index = pricing date, columns = factor_id. delivery : pd.DataFrame Index = factor_id. Delivery period of every factor (tz-aware), taken from its most recent quote. """ source = config["source"] cfg = source["prices"] cols = cfg["columns"] frames = read_parquets(zip_path, cfg["file_pattern"]) raw = pd.concat(frames.values(), ignore_index=True) _require_columns(raw, cols, "Price file") raw = raw.copy() raw[Col.DATE] = _to_local_date( raw[cols["date"]], source["timezone"] ) raw[Col.FACTOR] = raw[cols["tenor"]].map( lambda tenor: cfg["factor_id_template"].format(tenor=tenor) ) if raw.duplicated([Col.DATE, Col.FACTOR]).any(): raise ValueError( "Price file contains duplicated (date, tenor) quotes." ) prices = ( raw.pivot( index=Col.DATE, columns=Col.FACTOR, values=cols["price"], ) .sort_index() .rename_axis(columns=None) ) latest = ( raw.sort_values(Col.DATE) .groupby(Col.FACTOR) .last() ) delivery = pd.DataFrame( { Col.DELIVERY_START: pd.to_datetime( latest[cols["delivery_start"]], utc=True ), Col.DELIVERY_END: pd.to_datetime( latest[cols["delivery_end"]], utc=True ), } ).loc[prices.columns] return prices, delivery # ============================================================ # GENERATION # ============================================================ def load_generation( zip_path: str | Path, config: dict, ) -> dict[str, pd.DataFrame]: """ Load metered generation per plant. Returns {plant slug: DataFrame} with columns delivery_start, delivery_end (tz-aware UTC) and the average power in MW over each interval. """ cfg = config["source"]["generation"] cols = cfg["columns"] unit_regex = re.compile(cfg["unit_from_filename_regex"]) generation = {} for name, raw in read_parquets( zip_path, cfg["file_pattern"] ).items(): _require_columns(raw, cols, name) match = unit_regex.search(name) if match is None: raise ValueError( f"Cannot extract the plant name from '{name}'. " "Check 'unit_from_filename_regex' in config.json." ) slug = slugify(match.group("unit")) if slug in generation: raise ValueError( f"More than one generation file for plant '{slug}'." ) if cfg.get("status_filter"): raw = raw[raw[cols["status"]].isin(cfg["status_filter"])] df = pd.DataFrame( { Col.DELIVERY_START: pd.to_datetime( raw[cols["start"]], utc=True ), Col.DELIVERY_END: pd.to_datetime( raw[cols["end"]], utc=True ), Col.POWER_MW: ( raw[cols["quantity"]].astype(float) * cfg["quantity_to_mw"] ), } ).sort_values(Col.DELIVERY_START) if df[Col.DELIVERY_START].duplicated().any(): raise ValueError( f"{name} contains duplicated delivery intervals." ) if (df[Col.POWER_MW] < 0).any(): raise ValueError( f"{name} contains negative generation." ) generation[slug] = df.reset_index(drop=True) return generation def load_units( zip_path: str | Path, config: dict, ) -> pd.DataFrame: """ Load plant master data. Index = plant slug. """ cfg = config["source"]["units"] cols = cfg["columns"] raw = pd.concat( read_parquets(zip_path, cfg["file_pattern"]).values(), ignore_index=True, ) _require_columns(raw, cols, "Generation units file") unknown_units = ( set(raw[cols["capacity_unit"]].dropna()) - set(cfg["capacity_to_mw"]) ) if unknown_units: raise ValueError( f"Unknown capacity units {sorted(unknown_units)}. " "Add them to 'capacity_to_mw' in config.json." ) units = pd.DataFrame( { Col.PLANT: raw[cols["name"]], Col.TECHNOLOGY: raw[cols["technology"]], Col.COUNTRY: raw[cols["country"]], Col.CAPACITY_MW: ( raw[cols["capacity"]].astype(float) * raw[cols["capacity_unit"]].map(cfg["capacity_to_mw"]) ), } ) units.index = units[Col.PLANT].map(slugify) if units.index.duplicated().any(): raise ValueError( "Generation units file contains duplicated plants." ) return units