Spaces:
Sleeping
Sleeping
Download data/loader.py from MinsaitEU/VaR_model: direct link, hf CLI and curl.
- Browser
- Download file 7.95 kB
-
https://huggingface.co/spaces/MinsaitEU/VaR_model/resolve/main/data/loader.py
- Command line
-
hf download hf://spaces/MinsaitEU/VaR_model/data/loader.py
-
curl -L -o loader.py https://huggingface.co/spaces/MinsaitEU/VaR_model/resolve/main/data/loader.py
7.95 kB
| import fnmatch | |
| import json | |
| import re | |
| import unicodedata | |
| import zipfile | |
| from pathlib import Path | |
| import pandas as pd | |
| from data.schema import Col | |
| ROOT_DIR = Path(__file__).resolve().parent.parent | |
| DEFAULT_CONFIG = ROOT_DIR / "config.json" | |
| # ============================================================ | |
| # CONFIG | |
| # ============================================================ | |
| def load_config( | |
| path: str | Path | None = None, | |
| ) -> dict: | |
| with open(path or DEFAULT_CONFIG, encoding="utf-8") as f: | |
| return json.load(f) | |
| def resolve_source_path( | |
| config: dict, | |
| path: str | Path | None = None, | |
| ) -> Path: | |
| """ | |
| Explicit path if given, otherwise the configured one | |
| (relative paths resolve against the project root). | |
| """ | |
| path = Path(path or config["source"]["path"]) | |
| if not path.is_absolute(): | |
| path = ROOT_DIR / path | |
| if not path.exists(): | |
| raise FileNotFoundError( | |
| f"Source file not found: {path}" | |
| ) | |
| return path | |
| # ============================================================ | |
| # ZIP ACCESS | |
| # ============================================================ | |
| def slugify(text: str) -> str: | |
| """ | |
| 'Parque Eólico de Coll del Moro' -> 'parque_eolico_de_coll_del_moro' | |
| """ | |
| ascii_text = ( | |
| unicodedata.normalize("NFKD", str(text)) | |
| .encode("ascii", "ignore") | |
| .decode("ascii") | |
| ) | |
| return re.sub( | |
| r"[^a-z0-9]+", "_", ascii_text.lower() | |
| ).strip("_") | |
| def read_parquets( | |
| zip_path: str | Path, | |
| pattern: str, | |
| ) -> dict[str, pd.DataFrame]: | |
| """ | |
| Read every parquet in the zip whose file name matches | |
| `pattern` (shell-style). Returns {file name: DataFrame}. | |
| """ | |
| frames = {} | |
| with zipfile.ZipFile(zip_path) as z: | |
| for member in z.namelist(): | |
| name = Path(member).name | |
| if fnmatch.fnmatch(name, pattern): | |
| with z.open(member) as f: | |
| frames[name] = pd.read_parquet(f) | |
| if not frames: | |
| raise ValueError( | |
| f"No file matching '{pattern}' in {Path(zip_path).name}." | |
| ) | |
| return frames | |
| def _require_columns(df, columns, source): | |
| missing = set(columns.values()) - set(df.columns) | |
| if missing: | |
| raise ValueError( | |
| f"{source} is missing columns {sorted(missing)}. " | |
| "Check the column mapping in config.json." | |
| ) | |
| def _to_local_date(series, timezone): | |
| """ | |
| Timestamps -> tz-naive calendar dates in the market timezone. | |
| """ | |
| series = pd.to_datetime(series) | |
| if series.dt.tz is None: | |
| series = series.dt.tz_localize("UTC") | |
| return ( | |
| series.dt.tz_convert(timezone) | |
| .dt.tz_localize(None) | |
| .dt.normalize() | |
| ) | |
| # ============================================================ | |
| # PRICES | |
| # ============================================================ | |
| def load_prices( | |
| zip_path: str | Path, | |
| config: dict, | |
| ) -> tuple[pd.DataFrame, pd.DataFrame]: | |
| """ | |
| Load forward-curve prices from the zip. | |
| Returns | |
| ------- | |
| prices : pd.DataFrame | |
| Index = pricing date, columns = factor_id. | |
| delivery : pd.DataFrame | |
| Index = factor_id. Delivery period of every factor | |
| (tz-aware), taken from its most recent quote. | |
| """ | |
| source = config["source"] | |
| cfg = source["prices"] | |
| cols = cfg["columns"] | |
| frames = read_parquets(zip_path, cfg["file_pattern"]) | |
| raw = pd.concat(frames.values(), ignore_index=True) | |
| _require_columns(raw, cols, "Price file") | |
| raw = raw.copy() | |
| raw[Col.DATE] = _to_local_date( | |
| raw[cols["date"]], source["timezone"] | |
| ) | |
| raw[Col.FACTOR] = raw[cols["tenor"]].map( | |
| lambda tenor: cfg["factor_id_template"].format(tenor=tenor) | |
| ) | |
| if raw.duplicated([Col.DATE, Col.FACTOR]).any(): | |
| raise ValueError( | |
| "Price file contains duplicated (date, tenor) quotes." | |
| ) | |
| prices = ( | |
| raw.pivot( | |
| index=Col.DATE, | |
| columns=Col.FACTOR, | |
| values=cols["price"], | |
| ) | |
| .sort_index() | |
| .rename_axis(columns=None) | |
| ) | |
| latest = ( | |
| raw.sort_values(Col.DATE) | |
| .groupby(Col.FACTOR) | |
| .last() | |
| ) | |
| delivery = pd.DataFrame( | |
| { | |
| Col.DELIVERY_START: pd.to_datetime( | |
| latest[cols["delivery_start"]], utc=True | |
| ), | |
| Col.DELIVERY_END: pd.to_datetime( | |
| latest[cols["delivery_end"]], utc=True | |
| ), | |
| } | |
| ).loc[prices.columns] | |
| return prices, delivery | |
| # ============================================================ | |
| # GENERATION | |
| # ============================================================ | |
| def load_generation( | |
| zip_path: str | Path, | |
| config: dict, | |
| ) -> dict[str, pd.DataFrame]: | |
| """ | |
| Load metered generation per plant. | |
| Returns {plant slug: DataFrame} with columns | |
| delivery_start, delivery_end (tz-aware UTC) and the | |
| average power in MW over each interval. | |
| """ | |
| cfg = config["source"]["generation"] | |
| cols = cfg["columns"] | |
| unit_regex = re.compile(cfg["unit_from_filename_regex"]) | |
| generation = {} | |
| for name, raw in read_parquets( | |
| zip_path, cfg["file_pattern"] | |
| ).items(): | |
| _require_columns(raw, cols, name) | |
| match = unit_regex.search(name) | |
| if match is None: | |
| raise ValueError( | |
| f"Cannot extract the plant name from '{name}'. " | |
| "Check 'unit_from_filename_regex' in config.json." | |
| ) | |
| slug = slugify(match.group("unit")) | |
| if slug in generation: | |
| raise ValueError( | |
| f"More than one generation file for plant '{slug}'." | |
| ) | |
| if cfg.get("status_filter"): | |
| raw = raw[raw[cols["status"]].isin(cfg["status_filter"])] | |
| df = pd.DataFrame( | |
| { | |
| Col.DELIVERY_START: pd.to_datetime( | |
| raw[cols["start"]], utc=True | |
| ), | |
| Col.DELIVERY_END: pd.to_datetime( | |
| raw[cols["end"]], utc=True | |
| ), | |
| Col.POWER_MW: ( | |
| raw[cols["quantity"]].astype(float) | |
| * cfg["quantity_to_mw"] | |
| ), | |
| } | |
| ).sort_values(Col.DELIVERY_START) | |
| if df[Col.DELIVERY_START].duplicated().any(): | |
| raise ValueError( | |
| f"{name} contains duplicated delivery intervals." | |
| ) | |
| if (df[Col.POWER_MW] < 0).any(): | |
| raise ValueError( | |
| f"{name} contains negative generation." | |
| ) | |
| generation[slug] = df.reset_index(drop=True) | |
| return generation | |
| def load_units( | |
| zip_path: str | Path, | |
| config: dict, | |
| ) -> pd.DataFrame: | |
| """ | |
| Load plant master data. Index = plant slug. | |
| """ | |
| cfg = config["source"]["units"] | |
| cols = cfg["columns"] | |
| raw = pd.concat( | |
| read_parquets(zip_path, cfg["file_pattern"]).values(), | |
| ignore_index=True, | |
| ) | |
| _require_columns(raw, cols, "Generation units file") | |
| unknown_units = ( | |
| set(raw[cols["capacity_unit"]].dropna()) | |
| - set(cfg["capacity_to_mw"]) | |
| ) | |
| if unknown_units: | |
| raise ValueError( | |
| f"Unknown capacity units {sorted(unknown_units)}. " | |
| "Add them to 'capacity_to_mw' in config.json." | |
| ) | |
| units = pd.DataFrame( | |
| { | |
| Col.PLANT: raw[cols["name"]], | |
| Col.TECHNOLOGY: raw[cols["technology"]], | |
| Col.COUNTRY: raw[cols["country"]], | |
| Col.CAPACITY_MW: ( | |
| raw[cols["capacity"]].astype(float) | |
| * raw[cols["capacity_unit"]].map(cfg["capacity_to_mw"]) | |
| ), | |
| } | |
| ) | |
| units.index = units[Col.PLANT].map(slugify) | |
| if units.index.duplicated().any(): | |
| raise ValueError( | |
| "Generation units file contains duplicated plants." | |
| ) | |
| return units | |