File size: 4,567 Bytes
28bf28d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
import logging
from typing import Optional

import pandas as pd

from .schemas import DatasetSchema
from .exceptions import (
    MissingColumnsError,
    ExtraColumnsError,
    ColumnOrderMismatchError,
    ColumnTypeMismatchError,
)

logger = logging.getLogger(__name__)


def _is_dtype_compatible(expected: str, actual: str) -> bool:
    """Small compatibility layer between schema dtype strings and pandas dtypes."""
    expected = expected.lower()
    actual = actual.lower()

    if expected in {"string", "str"}:
        return "object" in actual or "string" in actual

    if expected in {"int", "int64", "int32"}:
        return actual.startswith("int")

    if expected in {"float", "float64", "float32"}:
        return actual.startswith("float")

    if expected in {"datetime", "datetime64[ns]", "datetime64"}:
        return "datetime64" in actual

    # fallback: require exact match
    return expected == actual


class ColumnValidator:
    """Validates a pandas DataFrame against a DatasetSchema."""

    def __init__(self, schema: DatasetSchema) -> None:
        self.schema = schema

    def validate_columns(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
        dataset_name = dataset_name or self.schema.name

        expected = self.schema.column_names
        actual = list(df.columns)

        logger.debug(
            "Validating columns for dataset '%s'. Expected=%s, Actual=%s",
            dataset_name, expected, actual
        )

        missing = [col for col in expected if col not in actual]
        extra = [col for col in actual if col not in expected]

        if missing:
            logger.error("Missing columns in dataset '%s': %s", dataset_name, missing)
            raise MissingColumnsError(missing, dataset_name=dataset_name)

        if extra:
            logger.error("Extra columns in dataset '%s': %s", dataset_name, extra)
            raise ExtraColumnsError(extra, dataset_name=dataset_name)

        if self.schema.strict_order and actual != expected:
            logger.error(
                "Column order mismatch in dataset '%s'. Expected=%s, Actual=%s",
                dataset_name, expected, actual
            )
            raise ColumnOrderMismatchError(expected=expected, actual=actual, dataset_name=dataset_name)

        logger.info("Column presence/order validation passed for '%s'", dataset_name)

    def validate_types(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
        """Validate data types of columns if strict_types is enabled."""
        if not self.schema.strict_types:
            logger.debug(
                "strict_types is False for schema '%s'; skipping type validation.",
                self.schema.name,
            )
            return

        dataset_name = dataset_name or self.schema.name
        mismatches = {}

        logger.debug("Validating dtypes for dataset '%s'", dataset_name)

        for col_schema in self.schema.columns:
            col_name = col_schema.name
            expected_dtype = col_schema.dtype
            if expected_dtype is None:
                continue

            if col_name not in df.columns:
                continue  # handled in validate_columns

            actual_dtype = str(df[col_name].dtype)

            if not _is_dtype_compatible(expected_dtype, actual_dtype):
                logger.warning(
                    "Type mismatch for column '%s' in dataset '%s': expected=%s, actual=%s",
                    col_name, dataset_name, expected_dtype, actual_dtype
                )
                mismatches[col_name] = {"expected": expected_dtype, "actual": actual_dtype}

        if mismatches:
            logger.error(
                "Type validation failed for dataset '%s' with mismatches: %s",
                dataset_name, mismatches
            )
            raise ColumnTypeMismatchError(mismatches, dataset_name=dataset_name)

        logger.info("Type validation passed for dataset '%s'", dataset_name)

    def validate(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
        self.validate_columns(df, dataset_name=dataset_name)
        self.validate_types(df, dataset_name=dataset_name)

    def align_columns(self, df: pd.DataFrame) -> pd.DataFrame:
        """Return new DataFrame with columns ordered according to the schema."""
        ordered_cols = self.schema.column_names
        logger.debug(
            "Aligning columns for dataset '%s' to order: %s",
            self.schema.name, ordered_cols
        )
        return df[ordered_cols]