File size: 4,243 Bytes
1e91358
 
 
 
 
fa69f0d
 
e2855b7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1e91358
 
 
 
 
e2855b7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1e91358
e2855b7
 
 
 
 
 
 
 
 
 
 
1e91358
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import logging
from typing import Optional

import pandas as pd

from schemas import DatasetSchema
from exceptions import (
    MissingColumnsError,
    ExtraColumnsError,
    ColumnOrderMismatchError,
    ColumnTypeMismatchError,
)

logger = logging.getLogger(__name__)


def _is_dtype_compatible(expected: str, actual: str) -> bool:
    expected = expected.lower()
    actual = actual.lower()

    if expected in {"string", "str"}:
        return "object" in actual or "string" in actual
    if expected in {"int", "int64", "int32"}:
        return actual.startswith("int")
    if expected in {"float", "float64", "float32"}:
        return actual.startswith("float")
    if expected in {"datetime", "datetime64[ns]", "datetime64"}:
        return "datetime64" in actual
    return expected == actual


class ColumnValidator:
    def __init__(self, schema: DatasetSchema) -> None:
        self.schema = schema

    def validate_columns(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
        dataset_name = dataset_name or self.schema.name

        expected = self.schema.column_names
        actual = list(df.columns)

        logger.debug(
            "Validating columns for dataset '%s'. Expected=%s, Actual=%s",
            dataset_name, expected, actual
        )

        missing = [col for col in expected if col not in actual]
        extra = [col for col in actual if col not in expected]

        if missing:
            logger.error("Missing columns in dataset '%s': %s", dataset_name, missing)
            raise MissingColumnsError(missing, dataset_name=dataset_name)

        if extra:
            logger.error("Extra columns in dataset '%s': %s", dataset_name, extra)
            raise ExtraColumnsError(extra, dataset_name=dataset_name)

        if self.schema.strict_order and actual != expected:
            logger.error(
                "Column order mismatch in dataset '%s'. Expected=%s, Actual=%s",
                dataset_name, expected, actual
            )
            raise ColumnOrderMismatchError(
                expected=expected,
                actual=actual,
                dataset_name=dataset_name,
            )

        logger.info("Column presence/order validation passed for '%s'", dataset_name)

    def validate_types(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
        if not self.schema.strict_types:
            logger.debug(
                "strict_types is False for schema '%s'; skipping type validation.",
                self.schema.name,
            )
            return

        dataset_name = dataset_name or self.schema.name
        mismatches = {}

        logger.debug("Validating dtypes for dataset '%s'", dataset_name)

        for col_schema in self.schema.columns:
            col_name = col_schema.name
            expected_dtype = col_schema.dtype
            if expected_dtype is None:
                continue
            if col_name not in df.columns:
                continue

            actual_dtype = str(df[col_name].dtype)
            if not _is_dtype_compatible(expected_dtype, actual_dtype):
                logger.warning(
                    "Type mismatch for column '%s' in dataset '%s': expected=%s, actual=%s",
                    col_name, dataset_name, expected_dtype, actual_dtype
                )
                mismatches[col_name] = {"expected": expected_dtype, "actual": actual_dtype}

        if mismatches:
            logger.error(
                "Type validation failed for dataset '%s' with mismatches: %s",
                dataset_name, mismatches
            )
            raise ColumnTypeMismatchError(mismatches, dataset_name=dataset_name)

        logger.info("Type validation passed for '%s'", dataset_name)

    def validate(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None:
        self.validate_columns(df, dataset_name=dataset_name)
        self.validate_types(df, dataset_name=dataset_name)

    def align_columns(self, df: pd.DataFrame) -> pd.DataFrame:
        ordered_cols = self.schema.column_names
        logger.debug(
            "Aligning columns for dataset '%s' to order: %s",
            self.schema.name, ordered_cols
        )
        return df[ordered_cols]