Spaces:
Running on Zero
Running on Zero
File size: 4,147 Bytes
fad317b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | """Deterministic validation of an extracted snapshot (spec 6.2, state 3)."""
from dataclasses import dataclass
from typing import Literal
import polars as pl
from pydantic import BaseModel, ConfigDict
from weekly_analyst.data.metrics_file import MetricDef, MetricsFile
FindingKind = Literal[
"schema_change", "nulls", "duplicates", "row_jump", "implausible_combo", "constant_column"
]
ROW_JUMP_LIMIT = 0.5
@dataclass(frozen=True)
class Finding:
kind: FindingKind
metric: str | None # None for dataset-level findings
detail: str
@dataclass(frozen=True)
class ValidationResult:
findings: list[Finding]
halted_metrics: set[str]
class PlausibilityRule(BaseModel):
"""Rows where column_a == value_a and column_b == value_b are implausible."""
model_config = ConfigDict(extra="forbid")
name: str
column_a: str
value_a: int | float | str
column_b: str
value_b: int | float | str
def _metric_columns(metric: MetricDef, date_column: str) -> set[str]:
return {date_column, metric.column, *metric.segment_columns}
def _missing_count(series: pl.Series) -> int:
count = series.null_count()
if series.dtype.is_float():
count += int(series.is_nan().sum())
return count
def _present_values(series: pl.Series) -> pl.Series:
values = series.drop_nulls()
if values.dtype.is_float():
values = values.filter(values.is_not_nan())
return values
def _is_row_jump(rows: int, prior: int) -> bool:
if prior == 0:
return rows != 0
return abs(rows - prior) / prior > ROW_JUMP_LIMIT
def validate(
df: pl.DataFrame,
metrics: MetricsFile,
prior_schema: dict[str, str] | None,
prior_rows: int | None,
) -> ValidationResult:
findings: list[Finding] = []
halted: set[str] = set()
broken: set[str] = set()
users: dict[str, list[str]] = {}
for metric in metrics.metrics:
for col in _metric_columns(metric, metrics.date_column):
users.setdefault(col, []).append(metric.name)
for col in sorted(users):
if col not in df.columns:
reason = "is missing"
elif (
prior_schema is not None
and col in prior_schema
and str(df.schema[col]) != prior_schema[col]
):
reason = f"changed type from {prior_schema[col]} to {df.schema[col]}"
else:
continue
broken.add(col)
for name in users[col]:
halted.add(name)
findings.append(Finding("schema_change", name, f"column {col!r} {reason}"))
for col in sorted(users):
if col not in df.columns:
continue
series = df.get_column(col)
missing = _missing_count(series)
present = _present_values(series)
constant = present.len() > 0 and present.n_unique() == 1
for name in users[col]:
if missing:
detail = f"column {col!r}: {missing} of {df.height} value(s) null or NaN"
findings.append(Finding("nulls", name, detail))
if constant:
detail = f"column {col!r} has one value in every row"
findings.append(Finding("constant_column", name, detail))
duplicates = df.height - df.unique().height
if duplicates > 0:
findings.append(Finding("duplicates", None, f"{duplicates} duplicate row(s)"))
if prior_rows is not None and _is_row_jump(df.height, prior_rows):
detail = f"row count {df.height} vs {prior_rows} in prior snapshot"
findings.append(Finding("row_jump", None, detail))
for rule in getattr(metrics, "plausibility", []):
if rule.column_a in broken or rule.column_b in broken:
continue
if rule.column_a not in df.columns or rule.column_b not in df.columns:
continue
hits = df.filter(
(pl.col(rule.column_a) == rule.value_a) & (pl.col(rule.column_b) == rule.value_b)
).height
if hits:
findings.append(Finding("implausible_combo", None, f"{rule.name}: {hits} row(s)"))
return ValidationResult(findings=findings, halted_metrics=halted)
|