Spaces:
Running
Running
Download server/uploads/validation.py from lilpookie404/tabulaclean: direct link, hf CLI and curl.
- Browser
- Download file 6.12 kB
-
https://huggingface.co/spaces/lilpookie404/tabulaclean/resolve/main/server/uploads/validation.py
- Command line
-
hf download hf://spaces/lilpookie404/tabulaclean/server/uploads/validation.py
-
curl -L -o validation.py https://huggingface.co/spaces/lilpookie404/tabulaclean/resolve/main/server/uploads/validation.py
6.12 kB
| """Revisioned uploaded-file validation for temporary sessions.""" | |
| from __future__ import annotations | |
| from datetime import datetime | |
| from typing import Any, Iterable | |
| import pandas as pd | |
| from .schemas import DetectedIssue, DownloadWarning, ValidationCheck, ValidationResult | |
| def is_missing(value: Any) -> bool: | |
| if value is None: | |
| return True | |
| if isinstance(value, str): | |
| return value.strip() == "" | |
| try: | |
| return bool(pd.isna(value)) | |
| except (TypeError, ValueError): | |
| return False | |
| def row_numbers(mask: pd.Series) -> list[int]: | |
| return [int(index) + 2 for index in mask[mask].index[:5]] | |
| def formula_download_warnings(dataframe: pd.DataFrame) -> list[DownloadWarning]: | |
| formula_like_count = sum( | |
| 1 | |
| for row in dataframe.itertuples(index=False, name=None) | |
| for value in row | |
| if isinstance(value, str) and value.startswith(("=", "+", "-", "@")) | |
| ) | |
| if not formula_like_count: | |
| return [] | |
| return [ | |
| DownloadWarning( | |
| code="formula_like_values", | |
| title="Some text may open as a spreadsheet formula", | |
| message=( | |
| f"{formula_like_count} values begin with characters that spreadsheet " | |
| "software may interpret as formulas. TabulaClean has not changed them." | |
| ), | |
| affected_count=formula_like_count, | |
| ) | |
| ] | |
| def _pending_review_check(has_pending_change: bool) -> ValidationCheck: | |
| if has_pending_change: | |
| return ValidationCheck( | |
| check_id="pending_review", | |
| title="Pending change review", | |
| status="failed", | |
| severity="error", | |
| message="Approve or reject the pending risky change before trusting this file.", | |
| affected_count=1, | |
| ) | |
| return ValidationCheck( | |
| check_id="pending_review", | |
| title="Pending change review", | |
| status="passed", | |
| severity="info", | |
| message="No risky change is waiting for review.", | |
| ) | |
| def _required_columns_check( | |
| dataframe: pd.DataFrame, | |
| required_column_ids: list[str], | |
| ) -> ValidationCheck: | |
| affected_columns: list[str] = [] | |
| example_rows: list[int] = [] | |
| affected_count = 0 | |
| for column_id in required_column_ids: | |
| mask = dataframe[column_id].map(is_missing) | |
| column_count = int(mask.sum()) | |
| if column_count: | |
| affected_columns.append(column_id) | |
| affected_count += column_count | |
| example_rows.extend(row_numbers(mask)) | |
| if affected_count: | |
| return ValidationCheck( | |
| check_id="required_columns", | |
| title="Required columns", | |
| status="failed", | |
| severity="error", | |
| message=f"{affected_count} required cells are still blank.", | |
| affected_count=affected_count, | |
| affected_columns=affected_columns, | |
| example_rows=list(dict.fromkeys(example_rows))[:5], | |
| ) | |
| return ValidationCheck( | |
| check_id="required_columns", | |
| title="Required columns", | |
| status="passed", | |
| severity="info", | |
| message="Required columns are filled.", | |
| affected_columns=list(required_column_ids), | |
| ) | |
| def _remaining_issues_check(issues: Iterable[DetectedIssue]) -> ValidationCheck: | |
| issue_list = list(issues) | |
| if issue_list: | |
| return ValidationCheck( | |
| check_id="remaining_issues", | |
| title="Remaining quality warnings", | |
| status="warning", | |
| severity="warning", | |
| message=f"{len(issue_list)} quality issue groups still deserve review.", | |
| affected_count=len(issue_list), | |
| affected_columns=list( | |
| dict.fromkeys( | |
| column_id | |
| for issue in issue_list | |
| for column_id in issue.affected_columns | |
| ) | |
| ), | |
| example_rows=list( | |
| dict.fromkeys( | |
| row for issue in issue_list for row in issue.example_rows | |
| ) | |
| )[:5], | |
| ) | |
| return ValidationCheck( | |
| check_id="remaining_issues", | |
| title="Remaining quality warnings", | |
| status="passed", | |
| severity="info", | |
| message="No remaining quality issue groups were detected.", | |
| ) | |
| def _download_warning_check(warnings: list[DownloadWarning]) -> ValidationCheck: | |
| affected_count = sum(warning.affected_count for warning in warnings) | |
| if affected_count: | |
| return ValidationCheck( | |
| check_id="download_warnings", | |
| title="Download warnings", | |
| status="warning", | |
| severity="warning", | |
| message=f"{affected_count} cells may need attention when opened in spreadsheet software.", | |
| affected_count=affected_count, | |
| ) | |
| return ValidationCheck( | |
| check_id="download_warnings", | |
| title="Download warnings", | |
| status="passed", | |
| severity="info", | |
| message="No spreadsheet download warnings were detected.", | |
| ) | |
| def validate_upload_table( | |
| *, | |
| dataframe: pd.DataFrame, | |
| required_column_ids: list[str], | |
| has_pending_change: bool, | |
| issues: list[DetectedIssue], | |
| download_warnings: list[DownloadWarning], | |
| revision: int, | |
| ran_at: datetime, | |
| ) -> ValidationResult: | |
| checks = [ | |
| _pending_review_check(has_pending_change), | |
| _required_columns_check(dataframe, required_column_ids), | |
| _remaining_issues_check(issues), | |
| _download_warning_check(download_warnings), | |
| ] | |
| errors = sum( | |
| 1 for check in checks if check.status == "failed" and check.severity == "error" | |
| ) | |
| warnings = sum(1 for check in checks if check.status == "warning") | |
| return ValidationResult( | |
| status="failed" if errors else "passed", | |
| revision=revision, | |
| required_column_ids=list(required_column_ids), | |
| ran_at=ran_at, | |
| checks=checks, | |
| summary={ | |
| "errors": errors, | |
| "warnings": warnings, | |
| "passed": sum(1 for check in checks if check.status == "passed"), | |
| }, | |
| ) | |