Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /utils /validators.py
| """ | |
| Validation utilities for effGen framework. | |
| This module provides comprehensive validation capabilities including input validation, | |
| schema validation, type checking, and data sanitization for the effGen framework. | |
| Features: | |
| - Input validation helpers for common data types | |
| - JSON Schema validation | |
| - Pydantic model validation | |
| - Type checking utilities | |
| - Data sanitization and normalization | |
| - Custom validator creation | |
| - Validation error handling and reporting | |
| Example: | |
| Basic validation: | |
| >>> from effgen.utils.validators import validate_json_schema, validate_model_name | |
| >>> validate_model_name("gpt-4") | |
| True | |
| >>> validate_model_name("") | |
| Traceback (most recent call last): | |
| ... | |
| effgen.utils.validators.ValidationError: Model name must be a non-empty string... | |
| Schema validation: | |
| >>> schema = {"type": "object", "properties": {"name": {"type": "string"}}} | |
| >>> validate_json_schema({"name": "test"}, schema) | |
| True | |
| Custom validators: | |
| >>> @require_type(str, int) | |
| ... def process_data(text, count): | |
| ... pass | |
| """ | |
| from __future__ import annotations | |
| import importlib.util as _importlib_util | |
| import json | |
| import os | |
| import re | |
| from collections.abc import Callable | |
| from functools import wraps | |
| from pathlib import Path | |
| from typing import Any | |
| from urllib.parse import urlparse | |
| from effgen.errors import quote_for_message, with_next_step | |
| # jsonschema is surprisingly heavy to import (it pulls rfc3987 / referencing), | |
| # so we only probe availability here via find_spec (which does not execute the | |
| # module) and import it lazily inside validate_json_schema(). This keeps | |
| # importing effgen.utils — and therefore `from effgen import Agent` — cheap. | |
| JSONSCHEMA_AVAILABLE = _importlib_util.find_spec("jsonschema") is not None | |
| try: | |
| from pydantic import BaseModel | |
| from pydantic import ValidationError as PydanticValidationError | |
| PYDANTIC_AVAILABLE = True | |
| except ImportError: | |
| PYDANTIC_AVAILABLE = False | |
| class ValidationError(Exception): | |
| """ | |
| Custom exception for validation errors. | |
| Attributes: | |
| message: Error message | |
| field: Field that failed validation | |
| value: Value that failed validation | |
| errors: List of detailed error messages | |
| """ | |
| def __init__( | |
| self, | |
| message: str, | |
| field: str | None = None, | |
| value: Any = None, | |
| errors: list[str] | None = None | |
| ) -> None: | |
| super().__init__(message) | |
| self.message = message | |
| self.field = field | |
| self.value = value | |
| self.errors = errors or [] | |
| def __str__(self) -> str: | |
| """Return the message, the field and value that failed, and the fix.""" | |
| parts = [quote_for_message(self.message)] | |
| if self.field: | |
| parts.append(f"Field: {self.field}") | |
| if self.value is not None: | |
| parts.append(f"Value: {quote_for_message(self.value)}") | |
| if self.errors: | |
| parts.append(f"Errors: {quote_for_message(', '.join(self.errors))}") | |
| return with_next_step( | |
| " | ".join(parts), | |
| "Correct the value named above to match what the field accepts, " | |
| "then re-run.", | |
| ) | |
| # ============================================================================ | |
| # Type Validators | |
| # ============================================================================ | |
| def validate_type(value: Any, expected_type: type, allow_none: bool = False) -> bool: | |
| """ | |
| Validate that a value is of the expected type. | |
| Args: | |
| value: Value to validate | |
| expected_type: Expected type | |
| allow_none: Whether to allow None values | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if value is None and allow_none: | |
| return True | |
| if not isinstance(value, expected_type): | |
| raise ValidationError( | |
| f"Expected type {expected_type.__name__}, got {type(value).__name__}", | |
| value=value | |
| ) | |
| return True | |
| def validate_string( | |
| value: Any, | |
| min_length: int | None = None, | |
| max_length: int | None = None, | |
| pattern: str | None = None, | |
| allow_empty: bool = True, | |
| ) -> bool: | |
| """ | |
| Validate a string value. | |
| Args: | |
| value: Value to validate | |
| min_length: Minimum length | |
| max_length: Maximum length | |
| pattern: Regex pattern to match | |
| allow_empty: Whether to allow empty strings | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(value, str): | |
| raise ValidationError(f"Expected string, got {type(value).__name__}", value=value) | |
| if not allow_empty and len(value) == 0: | |
| raise ValidationError("String cannot be empty", value=value) | |
| if min_length is not None and len(value) < min_length: | |
| raise ValidationError( | |
| f"String length {len(value)} is less than minimum {min_length}", | |
| value=value | |
| ) | |
| if max_length is not None and len(value) > max_length: | |
| raise ValidationError( | |
| f"String length {len(value)} exceeds maximum {max_length}", | |
| value=value | |
| ) | |
| if pattern is not None and not re.match(pattern, value): | |
| raise ValidationError( | |
| f"String does not match pattern {pattern}", | |
| value=value | |
| ) | |
| return True | |
| def validate_number( | |
| value: Any, | |
| min_value: int | float | None = None, | |
| max_value: int | float | None = None, | |
| allow_negative: bool = True, | |
| ) -> bool: | |
| """ | |
| Validate a numeric value. | |
| Args: | |
| value: Value to validate | |
| min_value: Minimum value | |
| max_value: Maximum value | |
| allow_negative: Whether to allow negative numbers | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(value, int | float) or isinstance(value, bool): | |
| raise ValidationError(f"Expected number, got {type(value).__name__}", value=value) | |
| if not allow_negative and value < 0: | |
| raise ValidationError("Negative numbers not allowed", value=value) | |
| if min_value is not None and value < min_value: | |
| raise ValidationError( | |
| f"Value {value} is less than minimum {min_value}", | |
| value=value | |
| ) | |
| if max_value is not None and value > max_value: | |
| raise ValidationError( | |
| f"Value {value} exceeds maximum {max_value}", | |
| value=value | |
| ) | |
| return True | |
| def validate_list( | |
| value: Any, | |
| item_type: type | None = None, | |
| min_length: int | None = None, | |
| max_length: int | None = None, | |
| allow_empty: bool = True, | |
| ) -> bool: | |
| """ | |
| Validate a list value. | |
| Args: | |
| value: Value to validate | |
| item_type: Expected type of list items | |
| min_length: Minimum length | |
| max_length: Maximum length | |
| allow_empty: Whether to allow empty lists | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(value, list): | |
| raise ValidationError(f"Expected list, got {type(value).__name__}", value=value) | |
| if not allow_empty and len(value) == 0: | |
| raise ValidationError("List cannot be empty", value=value) | |
| if min_length is not None and len(value) < min_length: | |
| raise ValidationError( | |
| f"List length {len(value)} is less than minimum {min_length}", | |
| value=value | |
| ) | |
| if max_length is not None and len(value) > max_length: | |
| raise ValidationError( | |
| f"List length {len(value)} exceeds maximum {max_length}", | |
| value=value | |
| ) | |
| if item_type is not None: | |
| for i, item in enumerate(value): | |
| if not isinstance(item, item_type): | |
| raise ValidationError( | |
| f"List item at index {i} is not of type {item_type.__name__}", | |
| value=item | |
| ) | |
| return True | |
| def validate_dict( | |
| value: Any, | |
| required_keys: list[str] | None = None, | |
| allowed_keys: list[str] | None = None, | |
| key_type: type | None = None, | |
| value_type: type | None = None, | |
| ) -> bool: | |
| """ | |
| Validate a dictionary value. | |
| Args: | |
| value: Value to validate | |
| required_keys: Keys that must be present | |
| allowed_keys: Only these keys are allowed | |
| key_type: Expected type of keys | |
| value_type: Expected type of values | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(value, dict): | |
| raise ValidationError(f"Expected dict, got {type(value).__name__}", value=value) | |
| if required_keys: | |
| missing_keys = set(required_keys) - set(value.keys()) | |
| if missing_keys: | |
| raise ValidationError( | |
| f"Missing required keys: {', '.join(missing_keys)}", | |
| value=value | |
| ) | |
| if allowed_keys: | |
| extra_keys = set(value.keys()) - set(allowed_keys) | |
| if extra_keys: | |
| raise ValidationError( | |
| f"Extra keys not allowed: {', '.join(extra_keys)}", | |
| value=value | |
| ) | |
| if key_type is not None: | |
| for key in value.keys(): | |
| if not isinstance(key, key_type): | |
| raise ValidationError( | |
| f"Key '{key}' is not of type {key_type.__name__}", | |
| value=key | |
| ) | |
| if value_type is not None: | |
| for key, val in value.items(): | |
| if not isinstance(val, value_type): | |
| raise ValidationError( | |
| f"Value for key '{key}' is not of type {value_type.__name__}", | |
| value=val | |
| ) | |
| return True | |
| # ============================================================================ | |
| # Path and File Validators | |
| # ============================================================================ | |
| def validate_path( | |
| path: str | Path, | |
| must_exist: bool = False, | |
| must_be_file: bool = False, | |
| must_be_dir: bool = False, | |
| create_if_missing: bool = False, | |
| ) -> bool: | |
| """ | |
| Validate a file system path. | |
| Args: | |
| path: Path to validate | |
| must_exist: Whether the path must exist | |
| must_be_file: Whether the path must be a file | |
| must_be_dir: Whether the path must be a directory | |
| create_if_missing: Create directory if it doesn't exist | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(path, str | Path): | |
| raise ValidationError(f"Expected path, got {type(path).__name__}", value=path) | |
| path = Path(path) | |
| if must_exist and not path.exists(): | |
| if create_if_missing and must_be_dir: | |
| path.mkdir(parents=True, exist_ok=True) | |
| else: | |
| raise ValidationError(f"Path does not exist: {path}", value=str(path)) | |
| if path.exists(): | |
| if must_be_file and not path.is_file(): | |
| raise ValidationError(f"Path is not a file: {path}", value=str(path)) | |
| if must_be_dir and not path.is_dir(): | |
| raise ValidationError(f"Path is not a directory: {path}", value=str(path)) | |
| return True | |
| def validate_file_extension( | |
| path: str | Path, | |
| allowed_extensions: list[str], | |
| ) -> bool: | |
| """ | |
| Validate file extension. | |
| Args: | |
| path: File path | |
| allowed_extensions: List of allowed extensions (e.g., ['.txt', '.json']) | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| path = Path(path) | |
| ext = path.suffix.lower() | |
| allowed_extensions = [e.lower() if e.startswith('.') else f'.{e.lower()}' | |
| for e in allowed_extensions] | |
| if ext not in allowed_extensions: | |
| raise ValidationError( | |
| f"File extension '{ext}' not in allowed extensions: {allowed_extensions}", | |
| value=str(path) | |
| ) | |
| return True | |
| # ============================================================================ | |
| # URL and Network Validators | |
| # ============================================================================ | |
| def validate_url( | |
| url: str, | |
| require_scheme: bool = True, | |
| allowed_schemes: list[str] | None = None, | |
| ) -> bool: | |
| """ | |
| Validate a URL. | |
| Args: | |
| url: URL to validate | |
| require_scheme: Whether to require a scheme (http://, https://, etc.) | |
| allowed_schemes: List of allowed schemes | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(url, str): | |
| raise ValidationError(f"Expected string, got {type(url).__name__}", value=url) | |
| try: | |
| result = urlparse(url) | |
| except Exception as e: | |
| raise ValidationError(f"Invalid URL: {e}", value=url) | |
| if require_scheme and not result.scheme: | |
| raise ValidationError("URL missing scheme (http://, https://, etc.)", value=url) | |
| if allowed_schemes and result.scheme not in allowed_schemes: | |
| raise ValidationError( | |
| f"URL scheme '{result.scheme}' not in allowed schemes: {allowed_schemes}", | |
| value=url | |
| ) | |
| return True | |
| def validate_email(email: str) -> bool: | |
| """ | |
| Validate an email address. | |
| Args: | |
| email: Email address to validate | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' | |
| if not re.match(pattern, email): | |
| raise ValidationError("Invalid email address", value=email) | |
| return True | |
| # ============================================================================ | |
| # Model and Configuration Validators | |
| # ============================================================================ | |
| def validate_model_name(model_name: str) -> bool: | |
| """ | |
| Validate a model name. | |
| Args: | |
| model_name: Model name to validate | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(model_name, str) or not model_name.strip(): | |
| raise ValidationError("Model name must be a non-empty string", value=model_name) | |
| # Allow alphanumeric, hyphens, underscores, dots, and slashes (for HF models) | |
| pattern = r'^[a-zA-Z0-9._/-]+$' | |
| if not re.match(pattern, model_name): | |
| raise ValidationError( | |
| "Model name contains invalid characters. " | |
| "Only alphanumeric, hyphens, underscores, dots, and slashes are allowed.", | |
| value=model_name | |
| ) | |
| return True | |
| def validate_config_dict( | |
| config: dict[str, Any], | |
| required_fields: list[str] | None = None, | |
| field_types: dict[str, type] | None = None, | |
| ) -> bool: | |
| """ | |
| Validate a configuration dictionary. | |
| Args: | |
| config: Configuration dictionary to validate | |
| required_fields: List of required field names | |
| field_types: Dictionary mapping field names to expected types | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| if not isinstance(config, dict): | |
| raise ValidationError(f"Expected dict, got {type(config).__name__}", value=config) | |
| errors = [] | |
| # Check required fields | |
| if required_fields: | |
| missing = [field for field in required_fields if field not in config] | |
| if missing: | |
| errors.append(f"Missing required fields: {', '.join(missing)}") | |
| # Check field types | |
| if field_types: | |
| for field, expected_type in field_types.items(): | |
| if field in config: | |
| value = config[field] | |
| if not isinstance(value, expected_type): | |
| errors.append( | |
| f"Field '{field}': expected {expected_type.__name__}, " | |
| f"got {type(value).__name__}" | |
| ) | |
| if errors: | |
| raise ValidationError("Configuration validation failed", errors=errors) | |
| return True | |
| # ============================================================================ | |
| # JSON Schema Validators | |
| # ============================================================================ | |
| def validate_json_schema(data: Any, schema: dict[str, Any]) -> bool: | |
| """ | |
| Validate data against a JSON schema. | |
| Args: | |
| data: Data to validate | |
| schema: JSON schema | |
| Returns: | |
| True if valid | |
| Raises: | |
| ValidationError: If validation fails or jsonschema not available | |
| """ | |
| if not JSONSCHEMA_AVAILABLE: | |
| raise ValidationError( | |
| "jsonschema package not available. Install with: pip install jsonschema" | |
| ) | |
| from jsonschema import ValidationError as JSONSchemaValidationError | |
| from jsonschema import validate as json_validate | |
| try: | |
| json_validate(instance=data, schema=schema) | |
| return True | |
| except JSONSchemaValidationError as e: | |
| raise ValidationError( | |
| f"JSON schema validation failed: {e.message}", | |
| field=".".join(str(p) for p in e.path), | |
| value=e.instance, | |
| errors=[e.message] | |
| ) | |
| def load_and_validate_json( | |
| filepath: str | Path, | |
| schema: dict[str, Any] | None = None, | |
| ) -> dict[str, Any]: | |
| """ | |
| Load and validate a JSON file. | |
| Args: | |
| filepath: Path to JSON file | |
| schema: Optional JSON schema to validate against | |
| Returns: | |
| Loaded and validated JSON data | |
| Raises: | |
| ValidationError: If validation fails | |
| """ | |
| filepath = Path(filepath) | |
| if not filepath.exists(): | |
| raise ValidationError(f"File does not exist: {filepath}", value=str(filepath)) | |
| try: | |
| with open(filepath) as f: | |
| data = json.load(f) | |
| except json.JSONDecodeError as e: | |
| raise ValidationError(f"Invalid JSON: {e}", value=str(filepath)) | |
| if schema: | |
| validate_json_schema(data, schema) | |
| return data | |
| # ============================================================================ | |
| # Pydantic Validators | |
| # ============================================================================ | |
| def validate_pydantic_model(data: Any, model_class: type[BaseModel]) -> BaseModel: | |
| """ | |
| Validate data against a Pydantic model. | |
| Args: | |
| data: Data to validate | |
| model_class: Pydantic model class | |
| Returns: | |
| Validated Pydantic model instance | |
| Raises: | |
| ValidationError: If validation fails or Pydantic not available | |
| """ | |
| if not PYDANTIC_AVAILABLE: | |
| raise ValidationError( | |
| "Pydantic package not available. Install with: pip install pydantic" | |
| ) | |
| try: | |
| return model_class(**data) if isinstance(data, dict) else model_class(data) | |
| except PydanticValidationError as e: | |
| errors = [f"{err['loc']}: {err['msg']}" for err in e.errors()] | |
| raise ValidationError( | |
| "Pydantic validation failed", | |
| errors=errors | |
| ) | |
| # ============================================================================ | |
| # Data Sanitization | |
| # ============================================================================ | |
| def sanitize_string( | |
| value: str, | |
| remove_whitespace: bool = False, | |
| remove_special_chars: bool = False, | |
| lowercase: bool = False, | |
| max_length: int | None = None, | |
| ) -> str: | |
| """ | |
| Sanitize a string value. | |
| Args: | |
| value: String to sanitize | |
| remove_whitespace: Remove all whitespace | |
| remove_special_chars: Remove special characters | |
| lowercase: Convert to lowercase | |
| max_length: Maximum length (truncate if longer) | |
| Returns: | |
| Sanitized string | |
| """ | |
| result = str(value) | |
| if remove_whitespace: | |
| result = re.sub(r'\s+', '', result) | |
| else: | |
| result = result.strip() | |
| result = re.sub(r'\s+', ' ', result) | |
| if remove_special_chars: | |
| result = re.sub(r'[^a-zA-Z0-9\s]', '', result) | |
| if lowercase: | |
| result = result.lower() | |
| if max_length and len(result) > max_length: | |
| result = result[:max_length] | |
| return result | |
| def sanitize_filename(filename: str, max_length: int = 255) -> str: | |
| """ | |
| Sanitize a filename to be safe for file systems. | |
| Args: | |
| filename: Filename to sanitize | |
| max_length: Maximum filename length | |
| Returns: | |
| Sanitized filename | |
| """ | |
| # Remove path separators and other unsafe characters | |
| filename = re.sub(r'[<>:"/\\|?*]', '_', filename) | |
| # Remove leading/trailing dots and spaces | |
| filename = filename.strip('. ') | |
| # Ensure not empty | |
| if not filename: | |
| filename = 'unnamed' | |
| # Truncate if too long | |
| if len(filename) > max_length: | |
| name, ext = os.path.splitext(filename) | |
| filename = name[:max_length - len(ext)] + ext | |
| return filename | |
| # ============================================================================ | |
| # Decorator Validators | |
| # ============================================================================ | |
| def require_type(*expected_types: type) -> Callable[[Callable], Callable]: | |
| """ | |
| Decorator to validate function argument types. | |
| Args: | |
| *expected_types: Expected types for each argument | |
| Example: | |
| >>> @require_type(str, int) | |
| ... def process(name, count): | |
| ... pass | |
| """ | |
| def decorator(func: Callable) -> Callable: | |
| def wrapper(*args, **kwargs): | |
| # Validate positional arguments | |
| for i, (arg, expected_type) in enumerate(zip(args, expected_types)): | |
| if not isinstance(arg, expected_type): | |
| raise ValidationError( | |
| f"Argument {i} must be of type {expected_type.__name__}, " | |
| f"got {type(arg).__name__}", | |
| value=arg | |
| ) | |
| return func(*args, **kwargs) | |
| return wrapper | |
| return decorator | |
| def validate_input(**validators: Callable) -> Callable[[Callable], Callable]: | |
| """ | |
| Decorator to validate function arguments using custom validators. | |
| Args: | |
| **validators: Mapping of argument names to validator functions | |
| Example: | |
| >>> @validate_input(name=lambda x: len(x) > 0, count=lambda x: x > 0) | |
| ... def process(name, count): | |
| ... pass | |
| """ | |
| def decorator(func: Callable) -> Callable: | |
| def wrapper(*args, **kwargs): | |
| # Get function signature | |
| sig = func.__code__.co_varnames[:func.__code__.co_argcount] | |
| # Combine positional and keyword arguments | |
| all_args = dict(zip(sig, args)) | |
| all_args.update(kwargs) | |
| # Validate each argument | |
| for arg_name, validator in validators.items(): | |
| if arg_name in all_args: | |
| value = all_args[arg_name] | |
| try: | |
| if not validator(value): | |
| raise ValidationError( | |
| f"Validation failed for argument '{arg_name}'", | |
| field=arg_name, | |
| value=value | |
| ) | |
| except Exception as e: | |
| if isinstance(e, ValidationError): | |
| raise | |
| raise ValidationError( | |
| f"Validation error for argument '{arg_name}': {e}", | |
| field=arg_name, | |
| value=value | |
| ) | |
| return func(*args, **kwargs) | |
| return wrapper | |
| return decorator | |
| def validate_output(validator: Callable) -> Callable[[Callable], Callable]: | |
| """ | |
| Decorator to validate function return value. | |
| Args: | |
| validator: Function to validate the return value | |
| Example: | |
| >>> @validate_output(lambda x: isinstance(x, int) and x > 0) | |
| ... def get_count(): | |
| ... return 42 | |
| """ | |
| def decorator(func: Callable) -> Callable: | |
| def wrapper(*args, **kwargs): | |
| result = func(*args, **kwargs) | |
| try: | |
| if not validator(result): | |
| raise ValidationError( | |
| f"Return value validation failed for {func.__name__}", | |
| value=result | |
| ) | |
| except Exception as e: | |
| if isinstance(e, ValidationError): | |
| raise | |
| raise ValidationError( | |
| f"Return value validation error: {e}", | |
| value=result | |
| ) | |
| return result | |
| return wrapper | |
| return decorator | |
| __all__ = [ | |
| 'ValidationError', | |
| 'validate_type', | |
| 'validate_string', | |
| 'validate_number', | |
| 'validate_list', | |
| 'validate_dict', | |
| 'validate_path', | |
| 'validate_file_extension', | |
| 'validate_url', | |
| 'validate_email', | |
| 'validate_model_name', | |
| 'validate_config_dict', | |
| 'validate_json_schema', | |
| 'load_and_validate_json', | |
| 'validate_pydantic_model', | |
| 'sanitize_string', | |
| 'sanitize_filename', | |
| 'require_type', | |
| 'validate_input', | |
| 'validate_output', | |
| ] | |
Xet Storage Details
- Size:
- 25.3 kB
- Xet hash:
- 4c75ac70a958b659aaf822ab4dfcf64c78ce6632195f5e9450778afe8b0b539a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.