Agile_Lab / validator.py
jonels1988's picture
Update validator.py
1e91358 verified
Raw History Blame Contribute Delete
4.24 kB
import logging
from typing import Optional
import pandas as pd
from schemas import DatasetSchema
from exceptions import (
MissingColumnsError,
ExtraColumnsError,
ColumnOrderMismatchError,
ColumnTypeMismatchError,
)
logger = logging.getLogger(__name__)
def _is_dtype_compatible(expected: str, actual: str) -> bool:
expected = expected.lower()
actual = actual.lower()
if expected in {"string", "str"}:
return "object" in actual or "string" in actual
if expected in {"int", "int64", "int32"}:
return actual.startswith("int")
if expected in {"float", "float64", "float32"}:
return actual.startswith("float")
if expected in {"datetime", "datetime64[ns]", "datetime64"}:
return "datetime64" in actual
return expected == actual
class ColumnValidator:
def __init__(self, schema: DatasetSchema) -> None:
self.schema = schema
def validate_columns(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
dataset_name = dataset_name or self.schema.name
expected = self.schema.column_names
actual = list(df.columns)
logger.debug(
"Validating columns for dataset '%s'. Expected=%s, Actual=%s",
dataset_name, expected, actual
)
missing = [col for col in expected if col not in actual]
extra = [col for col in actual if col not in expected]
if missing:
logger.error("Missing columns in dataset '%s': %s", dataset_name, missing)
raise MissingColumnsError(missing, dataset_name=dataset_name)
if extra:
logger.error("Extra columns in dataset '%s': %s", dataset_name, extra)
raise ExtraColumnsError(extra, dataset_name=dataset_name)
if self.schema.strict_order and actual != expected:
logger.error(
"Column order mismatch in dataset '%s'. Expected=%s, Actual=%s",
dataset_name, expected, actual
)
raise ColumnOrderMismatchError(
expected=expected,
actual=actual,
dataset_name=dataset_name,
)
logger.info("Column presence/order validation passed for '%s'", dataset_name)
def validate_types(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
if not self.schema.strict_types:
logger.debug(
"strict_types is False for schema '%s'; skipping type validation.",
self.schema.name,
)
return
dataset_name = dataset_name or self.schema.name
mismatches = {}
logger.debug("Validating dtypes for dataset '%s'", dataset_name)
for col_schema in self.schema.columns:
col_name = col_schema.name
expected_dtype = col_schema.dtype
if expected_dtype is None:
continue
if col_name not in df.columns:
continue
actual_dtype = str(df[col_name].dtype)
if not _is_dtype_compatible(expected_dtype, actual_dtype):
logger.warning(
"Type mismatch for column '%s' in dataset '%s': expected=%s, actual=%s",
col_name, dataset_name, expected_dtype, actual_dtype
)
mismatches[col_name] = {"expected": expected_dtype, "actual": actual_dtype}
if mismatches:
logger.error(
"Type validation failed for dataset '%s' with mismatches: %s",
dataset_name, mismatches
)
raise ColumnTypeMismatchError(mismatches, dataset_name=dataset_name)
logger.info("Type validation passed for '%s'", dataset_name)
def validate(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
self.validate_columns(df, dataset_name=dataset_name)
self.validate_types(df, dataset_name=dataset_name)
def align_columns(self, df: pd.DataFrame) -> pd.DataFrame:
ordered_cols = self.schema.column_names
logger.debug(
"Aligning columns for dataset '%s' to order: %s",
self.schema.name, ordered_cols
)
return df[ordered_cols]