Spaces:
Sleeping
Sleeping
Download validator.py from jonels1988/Agile_Lab: direct link, hf CLI and curl.
- Browser
- Download file 4.24 kB
-
https://huggingface.co/spaces/jonels1988/Agile_Lab/resolve/main/validator.py
- Command line
-
hf download hf://spaces/jonels1988/Agile_Lab/validator.py
-
curl -L -o validator.py https://huggingface.co/spaces/jonels1988/Agile_Lab/resolve/main/validator.py
4.24 kB
| import logging | |
| from typing import Optional | |
| import pandas as pd | |
| from schemas import DatasetSchema | |
| from exceptions import ( | |
| MissingColumnsError, | |
| ExtraColumnsError, | |
| ColumnOrderMismatchError, | |
| ColumnTypeMismatchError, | |
| ) | |
| logger = logging.getLogger(__name__) | |
| def _is_dtype_compatible(expected: str, actual: str) -> bool: | |
| expected = expected.lower() | |
| actual = actual.lower() | |
| if expected in {"string", "str"}: | |
| return "object" in actual or "string" in actual | |
| if expected in {"int", "int64", "int32"}: | |
| return actual.startswith("int") | |
| if expected in {"float", "float64", "float32"}: | |
| return actual.startswith("float") | |
| if expected in {"datetime", "datetime64[ns]", "datetime64"}: | |
| return "datetime64" in actual | |
| return expected == actual | |
| class ColumnValidator: | |
| def __init__(self, schema: DatasetSchema) -> None: | |
| self.schema = schema | |
| def validate_columns(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None: | |
| dataset_name = dataset_name or self.schema.name | |
| expected = self.schema.column_names | |
| actual = list(df.columns) | |
| logger.debug( | |
| "Validating columns for dataset '%s'. Expected=%s, Actual=%s", | |
| dataset_name, expected, actual | |
| ) | |
| missing = [col for col in expected if col not in actual] | |
| extra = [col for col in actual if col not in expected] | |
| if missing: | |
| logger.error("Missing columns in dataset '%s': %s", dataset_name, missing) | |
| raise MissingColumnsError(missing, dataset_name=dataset_name) | |
| if extra: | |
| logger.error("Extra columns in dataset '%s': %s", dataset_name, extra) | |
| raise ExtraColumnsError(extra, dataset_name=dataset_name) | |
| if self.schema.strict_order and actual != expected: | |
| logger.error( | |
| "Column order mismatch in dataset '%s'. Expected=%s, Actual=%s", | |
| dataset_name, expected, actual | |
| ) | |
| raise ColumnOrderMismatchError( | |
| expected=expected, | |
| actual=actual, | |
| dataset_name=dataset_name, | |
| ) | |
| logger.info("Column presence/order validation passed for '%s'", dataset_name) | |
| def validate_types(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None: | |
| if not self.schema.strict_types: | |
| logger.debug( | |
| "strict_types is False for schema '%s'; skipping type validation.", | |
| self.schema.name, | |
| ) | |
| return | |
| dataset_name = dataset_name or self.schema.name | |
| mismatches = {} | |
| logger.debug("Validating dtypes for dataset '%s'", dataset_name) | |
| for col_schema in self.schema.columns: | |
| col_name = col_schema.name | |
| expected_dtype = col_schema.dtype | |
| if expected_dtype is None: | |
| continue | |
| if col_name not in df.columns: | |
| continue | |
| actual_dtype = str(df[col_name].dtype) | |
| if not _is_dtype_compatible(expected_dtype, actual_dtype): | |
| logger.warning( | |
| "Type mismatch for column '%s' in dataset '%s': expected=%s, actual=%s", | |
| col_name, dataset_name, expected_dtype, actual_dtype | |
| ) | |
| mismatches[col_name] = {"expected": expected_dtype, "actual": actual_dtype} | |
| if mismatches: | |
| logger.error( | |
| "Type validation failed for dataset '%s' with mismatches: %s", | |
| dataset_name, mismatches | |
| ) | |
| raise ColumnTypeMismatchError(mismatches, dataset_name=dataset_name) | |
| logger.info("Type validation passed for '%s'", dataset_name) | |
| def validate(self, df: pd.DataFrame, dataset_name: Optional[str] = None) -> None: | |
| self.validate_columns(df, dataset_name=dataset_name) | |
| self.validate_types(df, dataset_name=dataset_name) | |
| def align_columns(self, df: pd.DataFrame) -> pd.DataFrame: | |
| ordered_cols = self.schema.column_names | |
| logger.debug( | |
| "Aligning columns for dataset '%s' to order: %s", | |
| self.schema.name, ordered_cols | |
| ) | |
| return df[ordered_cols] |