Spaces:
Sleeping
Sleeping
File size: 4,243 Bytes
1e91358 fa69f0d e2855b7 1e91358 e2855b7 1e91358 e2855b7 1e91358 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | import logging
from typing import Optional
import pandas as pd
from schemas import DatasetSchema
from exceptions import (
MissingColumnsError,
ExtraColumnsError,
ColumnOrderMismatchError,
ColumnTypeMismatchError,
)
logger = logging.getLogger(__name__)
def _is_dtype_compatible(expected: str, actual: str) -> bool:
expected = expected.lower()
actual = actual.lower()
if expected in {"string", "str"}:
return "object" in actual or "string" in actual
if expected in {"int", "int64", "int32"}:
return actual.startswith("int")
if expected in {"float", "float64", "float32"}:
return actual.startswith("float")
if expected in {"datetime", "datetime64[ns]", "datetime64"}:
return "datetime64" in actual
return expected == actual
class ColumnValidator:
def __init__(self, schema: DatasetSchema) -> None:
self.schema = schema
def validate_columns(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
dataset_name = dataset_name or self.schema.name
expected = self.schema.column_names
actual = list(df.columns)
logger.debug(
"Validating columns for dataset '%s'. Expected=%s, Actual=%s",
dataset_name, expected, actual
)
missing = [col for col in expected if col not in actual]
extra = [col for col in actual if col not in expected]
if missing:
logger.error("Missing columns in dataset '%s': %s", dataset_name, missing)
raise MissingColumnsError(missing, dataset_name=dataset_name)
if extra:
logger.error("Extra columns in dataset '%s': %s", dataset_name, extra)
raise ExtraColumnsError(extra, dataset_name=dataset_name)
if self.schema.strict_order and actual != expected:
logger.error(
"Column order mismatch in dataset '%s'. Expected=%s, Actual=%s",
dataset_name, expected, actual
)
raise ColumnOrderMismatchError(
expected=expected,
actual=actual,
dataset_name=dataset_name,
)
logger.info("Column presence/order validation passed for '%s'", dataset_name)
def validate_types(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
if not self.schema.strict_types:
logger.debug(
"strict_types is False for schema '%s'; skipping type validation.",
self.schema.name,
)
return
dataset_name = dataset_name or self.schema.name
mismatches = {}
logger.debug("Validating dtypes for dataset '%s'", dataset_name)
for col_schema in self.schema.columns:
col_name = col_schema.name
expected_dtype = col_schema.dtype
if expected_dtype is None:
continue
if col_name not in df.columns:
continue
actual_dtype = str(df[col_name].dtype)
if not _is_dtype_compatible(expected_dtype, actual_dtype):
logger.warning(
"Type mismatch for column '%s' in dataset '%s': expected=%s, actual=%s",
col_name, dataset_name, expected_dtype, actual_dtype
)
mismatches[col_name] = {"expected": expected_dtype, "actual": actual_dtype}
if mismatches:
logger.error(
"Type validation failed for dataset '%s' with mismatches: %s",
dataset_name, mismatches
)
raise ColumnTypeMismatchError(mismatches, dataset_name=dataset_name)
logger.info("Type validation passed for '%s'", dataset_name)
def validate(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
self.validate_columns(df, dataset_name=dataset_name)
self.validate_types(df, dataset_name=dataset_name)
def align_columns(self, df: pd.DataFrame) -> pd.DataFrame:
ordered_cols = self.schema.column_names
logger.debug(
"Aligning columns for dataset '%s' to order: %s",
self.schema.name, ordered_cols
)
return df[ordered_cols] |