Entropyobserver
Support Norwegian imports and reliable AI review
d4b4379
Raw History Blame Contribute Delete
5.87 kB
"""Read and validate production records before analysis."""
import hashlib
import io
import numpy as np
import pandas as pd
import requests
import streamlit as st
from config import DATA_URLS, SAMPLE_BASE_WEIGHTS, SAMPLE_MATERIALS, SAMPLE_SHIFTS, get_translation
REQUIRED_COLUMNS = {"date", "weight_kg", "material_type"}
def _read_table(content: bytes) -> pd.DataFrame:
for encoding in ("utf-8-sig", "cp1252"):
try:
text = content.decode(encoding)
break
except UnicodeDecodeError:
continue
else:
raise ValueError("The file could not be decoded as UTF-8 or cp1252.")
try:
return pd.read_csv(
io.StringIO(text), sep=None, engine="python", dtype=str,
keep_default_na=False, skipinitialspace=True,
)
except pd.errors.ParserError as error:
raise ValueError("Could not read the table. Use tab, comma, or semicolon columns.") from error
def _parse_dates(values: pd.Series) -> pd.Series:
text = values.astype("string").str.strip()
parsed = pd.Series(pd.NaT, index=values.index, dtype="datetime64[ns]")
for date_format in ("%d.%m.%Y", "%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y"):
parsed = parsed.fillna(pd.to_datetime(text, format=date_format, errors="coerce"))
return parsed
def _normalize_weight(value) -> str:
text = str(value).strip().replace(" ", "").replace("\u00a0", "").replace("\u202f", "")
if "," in text and "." in text:
if text.rfind(",") > text.rfind("."):
return text.replace(".", "").replace(",", ".")
return text.replace(",", "")
return text.replace(",", ".")
def validate_dataframe(source: pd.DataFrame) -> pd.DataFrame:
missing = REQUIRED_COLUMNS - set(source.columns)
if missing:
raise ValueError(
f"Missing columns: {', '.join(sorted(missing))}. "
"Expected tab, comma, or semicolon-separated date, weight_kg, material_type."
)
if source.empty:
raise ValueError("No production records were provided.")
df = source.copy()
df["date"] = _parse_dates(df["date"])
df["weight_kg"] = pd.to_numeric(df["weight_kg"].map(_normalize_weight), errors="coerce")
material = (df["material_type"].astype("string").str.strip()
.str.replace(r"\s+", " ", regex=True).str.casefold())
valid_date = df["date"].notna()
valid_weight = df["weight_kg"].notna() & np.isfinite(df["weight_kg"]) & df["weight_kg"].ge(0)
valid_material = material.notna() & material.ne("")
valid = valid_date & valid_weight & valid_material
errors = []
for index in df.index[~valid][:5]:
bad_fields = []
if not valid_date.loc[index]:
bad_fields.append("date")
if not valid_weight.loc[index]:
bad_fields.append("weight_kg")
if not valid_material.loc[index]:
bad_fields.append("material_type")
errors.append(f"Data row {index + 1}: {', '.join(bad_fields)}")
df = df.loc[valid].copy()
if df.empty:
raise ValueError("No valid production records remain after validation.")
df["material_type"] = material.loc[valid].astype(str)
if "shift" in df.columns:
df["shift"] = df["shift"].astype("string").str.strip().replace("", pd.NA)
df["day_name"] = df["date"].dt.day_name()
df.attrs["excluded_rows"] = len(source) - len(df)
df.attrs["validation_errors"] = errors
return df
@st.cache_data(ttl=3600)
def _download_preset_data(year: str) -> pd.DataFrame:
response = requests.get(DATA_URLS[year], timeout=10)
response.raise_for_status()
raw = _read_table(response.content)
df = validate_dataframe(raw)
df.attrs.update(
source_kind="preset",
source_year=year,
source_sha256=hashlib.sha256(response.content).hexdigest(),
)
return df
def load_preset_data(year: str) -> pd.DataFrame:
if year not in DATA_URLS:
raise ValueError(f"Unknown sample year: {year}")
try:
return _download_preset_data(year)
except requests.RequestException:
return generate_sample_data(year)
def generate_sample_data(year: str) -> pd.DataFrame:
rng = np.random.default_rng(42 if year == "2024" else 84)
dates = pd.date_range(f"{year}-01-01", f"{year}-12-31", freq="B")
records = []
for date in dates:
for material in SAMPLE_MATERIALS:
for shift in SAMPLE_SHIFTS:
base = SAMPLE_BASE_WEIGHTS[material]
weight = max(rng.normal(base, base * 0.2), base * 0.3)
records.append((date, round(weight, 1), material, shift))
df = pd.DataFrame(records, columns=["date", "weight_kg", "material_type", "shift"])
df["day_name"] = df["date"].dt.day_name()
df.attrs.update(source_kind="generated", source_year=year, excluded_rows=0)
return df
def load_uploaded_data(file) -> pd.DataFrame:
content = file.getvalue()
raw = _read_table(content)
df = validate_dataframe(raw)
df.attrs.update(
source_kind="upload",
source_label=file.name,
source_sha256=hashlib.sha256(content).hexdigest(),
)
return df
def create_validated_tsv(df: pd.DataFrame) -> str:
"""Export the accepted rows in a format the upload control can read again."""
export = df.drop(columns="day_name", errors="ignore").copy()
export["date"] = export["date"].dt.strftime("%m/%d/%Y")
return export.to_csv(index=False, sep="\t")
def data_health_check(df: pd.DataFrame, lang: str = "English") -> dict:
t = get_translation(lang)
return {
t["health_valid_records"]: f"{len(df):,}",
t["health_recorded_days"]: f"{df['date'].nunique():,}",
t["health_excluded_rows"]: str(df.attrs.get("excluded_rows", "—")),
t["health_last_date"]: df["date"].max().strftime("%Y-%m-%d"),
}