Spaces:
Configuration error
Configuration error
Niranjan-Murugarasu024
feat: TradeSense AI Chairman Edition production release ready for Hugging Face Spaces
0125724 Download backend/data/validation_profiler.py from BlueMen/tradesenseAi: direct link, hf CLI and curl.
- Browser
- Download file 7.39 kB
-
https://huggingface.co/spaces/BlueMen/tradesenseAi/resolve/main/backend/data/validation_profiler.py
- Command line
-
hf download hf://spaces/BlueMen/tradesenseAi/backend/data/validation_profiler.py
-
curl -L -o validation_profiler.py https://huggingface.co/spaces/BlueMen/tradesenseAi/resolve/main/backend/data/validation_profiler.py
7.39 kB
| """ | |
| TradeSense AI β Dataset Validation Profiler | |
| Profiles a canonical OHLCV DataFrame for data quality characteristics: | |
| missing sessions, duplicate sessions, zero-volume sessions, flatline sequences, | |
| price range anomalies, and volume statistics. | |
| IMPORTANT: This profiler NEVER interpolates or synthesizes missing data. | |
| A missing session is reported as missing β not filled. | |
| """ | |
| import logging | |
| from datetime import date, datetime, timezone | |
| from typing import List, Optional, Dict, Any | |
| import pandas as pd | |
| import numpy as np | |
| from pydantic import BaseModel, Field | |
| logger = logging.getLogger("tradesense.data.validation_profiler") | |
| class DatasetProfile(BaseModel): | |
| """Safe, credential-free profile of a validated canonical dataset.""" | |
| ticker: str | |
| provider: str | |
| requested_start: date | |
| requested_end: date | |
| actual_start: Optional[date] = None | |
| actual_end: Optional[date] = None | |
| row_count: int = 0 | |
| # Session integrity | |
| missing_sessions: int = 0 | |
| missing_session_dates: List[str] = Field(default_factory=list) | |
| duplicate_sessions: int = 0 | |
| weekend_sessions_detected: int = 0 | |
| # Price characteristics | |
| min_price: Optional[float] = None | |
| max_price: Optional[float] = None | |
| price_range_pct: Optional[float] = None | |
| zero_or_negative_closes: int = 0 | |
| # Volume characteristics | |
| zero_volume_sessions: int = 0 | |
| average_volume: Optional[float] = None | |
| min_volume: Optional[float] = None | |
| max_volume: Optional[float] = None | |
| # Staleness / flatline | |
| flatline_sequences: int = 0 | |
| max_flatline_length: int = 0 | |
| # OHLC invariant violations (before quality gate filtering) | |
| ohlc_violations_detected: int = 0 | |
| # Retrieval metadata | |
| retrieved_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) | |
| latency_ms: float = 0.0 | |
| # Adjustment awareness | |
| price_series_type: str = "ADJUSTED" | |
| adjustment_methodology: str = "PROVIDER_NATIVE" | |
| def safe_dict(self) -> Dict[str, Any]: | |
| data = self.model_dump() | |
| # Convert dates to strings for JSON safety | |
| for k in ("requested_start", "requested_end", "actual_start", "actual_end", "retrieved_at"): | |
| if data.get(k) is not None: | |
| data[k] = str(data[k]) | |
| return data | |
| class DatasetValidationProfiler: | |
| """ | |
| Profiles a canonical OHLCV DataFrame against NYSE trading sessions. | |
| Reports deficiencies WITHOUT synthesizing missing data. | |
| """ | |
| def profile( | |
| df: pd.DataFrame, | |
| ticker: str, | |
| provider: str, | |
| requested_start: date, | |
| requested_end: date, | |
| latency_ms: float = 0.0, | |
| price_series_type: str = "ADJUSTED", | |
| adjustment_methodology: str = "PROVIDER_NATIVE", | |
| ) -> DatasetProfile: | |
| """ | |
| Build a full DatasetProfile for a canonical DataFrame. | |
| Missing sessions are REPORTED, never filled. | |
| """ | |
| profile = DatasetProfile( | |
| ticker=ticker, | |
| provider=provider, | |
| requested_start=requested_start, | |
| requested_end=requested_end, | |
| latency_ms=latency_ms, | |
| price_series_type=price_series_type, | |
| adjustment_methodology=adjustment_methodology, | |
| ) | |
| if df.empty: | |
| return profile | |
| profile.row_count = len(df) | |
| profile.actual_start = df["session"].min() | |
| profile.actual_end = df["session"].max() | |
| # ββ Weekend session detection ββββββββββββββββββββββββββββββββββββββ | |
| sessions_as_dates = pd.to_datetime(df["session"]) | |
| weekend_mask = sessions_as_dates.dt.dayofweek >= 5 | |
| profile.weekend_sessions_detected = int(weekend_mask.sum()) | |
| # ββ Duplicate sessions βββββββββββββββββββββββββββββββββββββββββββββ | |
| profile.duplicate_sessions = int(df["session"].duplicated().sum()) | |
| # ββ Missing NYSE sessions (approximate β business days only) βββββββ | |
| # Use business day frequency as proxy; exchange_calendar would be more precise | |
| try: | |
| bdays = pd.bdate_range( | |
| start=requested_start, | |
| end=min(requested_end, df["session"].max()), | |
| ) | |
| expected_sessions = set(b.date() for b in bdays) | |
| actual_sessions = set(df["session"].tolist()) | |
| missing = sorted(expected_sessions - actual_sessions) | |
| profile.missing_sessions = len(missing) | |
| # Only record up to 20 sample missing dates to keep payload manageable | |
| profile.missing_session_dates = [str(d) for d in missing[:20]] | |
| except Exception: | |
| profile.missing_sessions = 0 | |
| # ββ Price characteristics ββββββββββββββββββββββββββββββββββββββββββ | |
| closes = df["Close"].values | |
| profile.min_price = float(np.nanmin(closes)) | |
| profile.max_price = float(np.nanmax(closes)) | |
| if profile.min_price and profile.min_price > 0: | |
| profile.price_range_pct = round( | |
| (profile.max_price - profile.min_price) / profile.min_price * 100, 2 | |
| ) | |
| profile.zero_or_negative_closes = int((closes <= 0).sum()) | |
| # ββ OHLC invariant violations ββββββββββββββββββββββββββββββββββββββ | |
| violations = ( | |
| (df["High"] < df["Low"]) | | |
| (df["High"] < df["Close"]) | | |
| (df["High"] < df["Open"]) | | |
| (df["Low"] > df["Close"]) | | |
| (df["Low"] > df["Open"]) | |
| ) | |
| profile.ohlc_violations_detected = int(violations.sum()) | |
| # ββ Volume characteristics βββββββββββββββββββββββββββββββββββββββββ | |
| volumes = df["Volume"].values | |
| profile.zero_volume_sessions = int((volumes == 0).sum()) | |
| profile.average_volume = round(float(np.nanmean(volumes)), 0) | |
| profile.min_volume = float(np.nanmin(volumes)) | |
| profile.max_volume = float(np.nanmax(volumes)) | |
| # ββ Flatline detection (10+ consecutive identical closes) ββββββββββ | |
| if len(closes) >= 2: | |
| # Find runs of identical consecutive close prices | |
| diffs = np.diff(closes) | |
| same = (diffs == 0).astype(int) | |
| max_run = 0 | |
| flatline_count = 0 | |
| current_run = 0 | |
| for s in same: | |
| if s == 1: | |
| current_run += 1 | |
| if current_run >= 9: # 10th identical = flatline | |
| if current_run == 9: | |
| flatline_count += 1 | |
| max_run = max(max_run, current_run + 1) | |
| else: | |
| current_run = 0 | |
| profile.flatline_sequences = flatline_count | |
| profile.max_flatline_length = max_run | |
| logger.debug( | |
| f"Profile({ticker}/{provider}): {profile.row_count} bars, " | |
| f"{profile.missing_sessions} missing, {profile.ohlc_violations_detected} OHLC violations" | |
| ) | |
| return profile | |