import logging from typing import Optional import pandas as pd from schemas import DatasetSchema from exceptions import ( MissingColumnsError, ExtraColumnsError, ColumnOrderMismatchError, ColumnTypeMismatchError, ) logger = logging.getLogger(__name__) def _is_dtype_compatible(expected: str, actual: str) -> bool: expected = expected.lower() actual = actual.lower() if expected in {"string", "str"}: return "object" in actual or "string" in actual if expected in {"int", "int64", "int32"}: return actual.startswith("int") if expected in {"float", "float64", "float32"}: return actual.startswith("float") if expected in {"datetime", "datetime64[ns]", "datetime64"}: return "datetime64" in actual return expected == actual class ColumnValidator: def __init__(self, schema: DatasetSchema) -> None: self.schema = schema def validate_columns(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None: dataset_name = dataset_name or self.schema.name expected = self.schema.column_names actual = list(df.columns) logger.debug( "Validating columns for dataset '%s'. Expected=%s, Actual=%s", dataset_name, expected, actual ) missing = [col for col in expected if col not in actual] extra = [col for col in actual if col not in expected] if missing: logger.error("Missing columns in dataset '%s': %s", dataset_name, missing) raise MissingColumnsError(missing, dataset_name=dataset_name) if extra: logger.error("Extra columns in dataset '%s': %s", dataset_name, extra) raise ExtraColumnsError(extra, dataset_name=dataset_name) if self.schema.strict_order and actual != expected: logger.error( "Column order mismatch in dataset '%s'. Expected=%s, Actual=%s", dataset_name, expected, actual ) raise ColumnOrderMismatchError( expected=expected, actual=actual, dataset_name=dataset_name, ) logger.info("Column presence/order validation passed for '%s'", dataset_name) def validate_types(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None: if not self.schema.strict_types: logger.debug( "strict_types is False for schema '%s'; skipping type validation.", self.schema.name, ) return dataset_name = dataset_name or self.schema.name mismatches = {} logger.debug("Validating dtypes for dataset '%s'", dataset_name) for col_schema in self.schema.columns: col_name = col_schema.name expected_dtype = col_schema.dtype if expected_dtype is None: continue if col_name not in df.columns: continue actual_dtype = str(df[col_name].dtype) if not _is_dtype_compatible(expected_dtype, actual_dtype): logger.warning( "Type mismatch for column '%s' in dataset '%s': expected=%s, actual=%s", col_name, dataset_name, expected_dtype, actual_dtype ) mismatches[col_name] = {"expected": expected_dtype, "actual": actual_dtype} if mismatches: logger.error( "Type validation failed for dataset '%s' with mismatches: %s", dataset_name, mismatches ) raise ColumnTypeMismatchError(mismatches, dataset_name=dataset_name) logger.info("Type validation passed for '%s'", dataset_name) def validate(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None: self.validate_columns(df, dataset_name=dataset_name) self.validate_types(df, dataset_name=dataset_name) def align_columns(self, df: pd.DataFrame) -> pd.DataFrame: ordered_cols = self.schema.column_names logger.debug( "Aligning columns for dataset '%s' to order: %s", self.schema.name, ordered_cols ) return df[ordered_cols]