Download data_handler.py from entropy25/IndustrialProductionIntelligence: direct link, hf CLI and curl.
- Browser
- Download file 5.87 kB
-
https://huggingface.co/spaces/entropy25/IndustrialProductionIntelligence/resolve/main/data_handler.py
- Command line
-
hf download hf://spaces/entropy25/IndustrialProductionIntelligence/data_handler.py
-
curl -L -o data_handler.py https://huggingface.co/spaces/entropy25/IndustrialProductionIntelligence/resolve/main/data_handler.py
5.87 kB
| """Read and validate production records before analysis.""" | |
| import hashlib | |
| import io | |
| import numpy as np | |
| import pandas as pd | |
| import requests | |
| import streamlit as st | |
| from config import DATA_URLS, SAMPLE_BASE_WEIGHTS, SAMPLE_MATERIALS, SAMPLE_SHIFTS, get_translation | |
| REQUIRED_COLUMNS = {"date", "weight_kg", "material_type"} | |
| def _read_table(content: bytes) -> pd.DataFrame: | |
| for encoding in ("utf-8-sig", "cp1252"): | |
| try: | |
| text = content.decode(encoding) | |
| break | |
| except UnicodeDecodeError: | |
| continue | |
| else: | |
| raise ValueError("The file could not be decoded as UTF-8 or cp1252.") | |
| try: | |
| return pd.read_csv( | |
| io.StringIO(text), sep=None, engine="python", dtype=str, | |
| keep_default_na=False, skipinitialspace=True, | |
| ) | |
| except pd.errors.ParserError as error: | |
| raise ValueError("Could not read the table. Use tab, comma, or semicolon columns.") from error | |
| def _parse_dates(values: pd.Series) -> pd.Series: | |
| text = values.astype("string").str.strip() | |
| parsed = pd.Series(pd.NaT, index=values.index, dtype="datetime64[ns]") | |
| for date_format in ("%d.%m.%Y", "%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y"): | |
| parsed = parsed.fillna(pd.to_datetime(text, format=date_format, errors="coerce")) | |
| return parsed | |
| def _normalize_weight(value) -> str: | |
| text = str(value).strip().replace(" ", "").replace("\u00a0", "").replace("\u202f", "") | |
| if "," in text and "." in text: | |
| if text.rfind(",") > text.rfind("."): | |
| return text.replace(".", "").replace(",", ".") | |
| return text.replace(",", "") | |
| return text.replace(",", ".") | |
| def validate_dataframe(source: pd.DataFrame) -> pd.DataFrame: | |
| missing = REQUIRED_COLUMNS - set(source.columns) | |
| if missing: | |
| raise ValueError( | |
| f"Missing columns: {', '.join(sorted(missing))}. " | |
| "Expected tab, comma, or semicolon-separated date, weight_kg, material_type." | |
| ) | |
| if source.empty: | |
| raise ValueError("No production records were provided.") | |
| df = source.copy() | |
| df["date"] = _parse_dates(df["date"]) | |
| df["weight_kg"] = pd.to_numeric(df["weight_kg"].map(_normalize_weight), errors="coerce") | |
| material = (df["material_type"].astype("string").str.strip() | |
| .str.replace(r"\s+", " ", regex=True).str.casefold()) | |
| valid_date = df["date"].notna() | |
| valid_weight = df["weight_kg"].notna() & np.isfinite(df["weight_kg"]) & df["weight_kg"].ge(0) | |
| valid_material = material.notna() & material.ne("") | |
| valid = valid_date & valid_weight & valid_material | |
| errors = [] | |
| for index in df.index[~valid][:5]: | |
| bad_fields = [] | |
| if not valid_date.loc[index]: | |
| bad_fields.append("date") | |
| if not valid_weight.loc[index]: | |
| bad_fields.append("weight_kg") | |
| if not valid_material.loc[index]: | |
| bad_fields.append("material_type") | |
| errors.append(f"Data row {index + 1}: {', '.join(bad_fields)}") | |
| df = df.loc[valid].copy() | |
| if df.empty: | |
| raise ValueError("No valid production records remain after validation.") | |
| df["material_type"] = material.loc[valid].astype(str) | |
| if "shift" in df.columns: | |
| df["shift"] = df["shift"].astype("string").str.strip().replace("", pd.NA) | |
| df["day_name"] = df["date"].dt.day_name() | |
| df.attrs["excluded_rows"] = len(source) - len(df) | |
| df.attrs["validation_errors"] = errors | |
| return df | |
| def _download_preset_data(year: str) -> pd.DataFrame: | |
| response = requests.get(DATA_URLS[year], timeout=10) | |
| response.raise_for_status() | |
| raw = _read_table(response.content) | |
| df = validate_dataframe(raw) | |
| df.attrs.update( | |
| source_kind="preset", | |
| source_year=year, | |
| source_sha256=hashlib.sha256(response.content).hexdigest(), | |
| ) | |
| return df | |
| def load_preset_data(year: str) -> pd.DataFrame: | |
| if year not in DATA_URLS: | |
| raise ValueError(f"Unknown sample year: {year}") | |
| try: | |
| return _download_preset_data(year) | |
| except requests.RequestException: | |
| return generate_sample_data(year) | |
| def generate_sample_data(year: str) -> pd.DataFrame: | |
| rng = np.random.default_rng(42 if year == "2024" else 84) | |
| dates = pd.date_range(f"{year}-01-01", f"{year}-12-31", freq="B") | |
| records = [] | |
| for date in dates: | |
| for material in SAMPLE_MATERIALS: | |
| for shift in SAMPLE_SHIFTS: | |
| base = SAMPLE_BASE_WEIGHTS[material] | |
| weight = max(rng.normal(base, base * 0.2), base * 0.3) | |
| records.append((date, round(weight, 1), material, shift)) | |
| df = pd.DataFrame(records, columns=["date", "weight_kg", "material_type", "shift"]) | |
| df["day_name"] = df["date"].dt.day_name() | |
| df.attrs.update(source_kind="generated", source_year=year, excluded_rows=0) | |
| return df | |
| def load_uploaded_data(file) -> pd.DataFrame: | |
| content = file.getvalue() | |
| raw = _read_table(content) | |
| df = validate_dataframe(raw) | |
| df.attrs.update( | |
| source_kind="upload", | |
| source_label=file.name, | |
| source_sha256=hashlib.sha256(content).hexdigest(), | |
| ) | |
| return df | |
| def create_validated_tsv(df: pd.DataFrame) -> str: | |
| """Export the accepted rows in a format the upload control can read again.""" | |
| export = df.drop(columns="day_name", errors="ignore").copy() | |
| export["date"] = export["date"].dt.strftime("%m/%d/%Y") | |
| return export.to_csv(index=False, sep="\t") | |
| def data_health_check(df: pd.DataFrame, lang: str = "English") -> dict: | |
| t = get_translation(lang) | |
| return { | |
| t["health_valid_records"]: f"{len(df):,}", | |
| t["health_recorded_days"]: f"{df['date'].nunique():,}", | |
| t["health_excluded_rows"]: str(df.attrs.get("excluded_rows", "—")), | |
| t["health_last_date"]: df["date"].max().strftime("%Y-%m-%d"), | |
| } | |