""" TradeSense AI — Dataset Validation Profiler Profiles a canonical OHLCV DataFrame for data quality characteristics: missing sessions, duplicate sessions, zero-volume sessions, flatline sequences, price range anomalies, and volume statistics. IMPORTANT: This profiler NEVER interpolates or synthesizes missing data. A missing session is reported as missing — not filled. """ import logging from datetime import date, datetime, timezone from typing import List, Optional, Dict, Any import pandas as pd import numpy as np from pydantic import BaseModel, Field logger = logging.getLogger("tradesense.data.validation_profiler") class DatasetProfile(BaseModel): """Safe, credential-free profile of a validated canonical dataset.""" ticker: str provider: str requested_start: date requested_end: date actual_start: Optional[date] = None actual_end: Optional[date] = None row_count: int = 0 # Session integrity missing_sessions: int = 0 missing_session_dates: List[str] = Field(default_factory=list) duplicate_sessions: int = 0 weekend_sessions_detected: int = 0 # Price characteristics min_price: Optional[float] = None max_price: Optional[float] = None price_range_pct: Optional[float] = None zero_or_negative_closes: int = 0 # Volume characteristics zero_volume_sessions: int = 0 average_volume: Optional[float] = None min_volume: Optional[float] = None max_volume: Optional[float] = None # Staleness / flatline flatline_sequences: int = 0 max_flatline_length: int = 0 # OHLC invariant violations (before quality gate filtering) ohlc_violations_detected: int = 0 # Retrieval metadata retrieved_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) latency_ms: float = 0.0 # Adjustment awareness price_series_type: str = "ADJUSTED" adjustment_methodology: str = "PROVIDER_NATIVE" def safe_dict(self) -> Dict[str, Any]: data = self.model_dump() # Convert dates to strings for JSON safety for k in ("requested_start", "requested_end", "actual_start", "actual_end", "retrieved_at"): if data.get(k) is not None: data[k] = str(data[k]) return data class DatasetValidationProfiler: """ Profiles a canonical OHLCV DataFrame against NYSE trading sessions. Reports deficiencies WITHOUT synthesizing missing data. """ @staticmethod def profile( df: pd.DataFrame, ticker: str, provider: str, requested_start: date, requested_end: date, latency_ms: float = 0.0, price_series_type: str = "ADJUSTED", adjustment_methodology: str = "PROVIDER_NATIVE", ) -> DatasetProfile: """ Build a full DatasetProfile for a canonical DataFrame. Missing sessions are REPORTED, never filled. """ profile = DatasetProfile( ticker=ticker, provider=provider, requested_start=requested_start, requested_end=requested_end, latency_ms=latency_ms, price_series_type=price_series_type, adjustment_methodology=adjustment_methodology, ) if df.empty: return profile profile.row_count = len(df) profile.actual_start = df["session"].min() profile.actual_end = df["session"].max() # ── Weekend session detection ────────────────────────────────────── sessions_as_dates = pd.to_datetime(df["session"]) weekend_mask = sessions_as_dates.dt.dayofweek >= 5 profile.weekend_sessions_detected = int(weekend_mask.sum()) # ── Duplicate sessions ───────────────────────────────────────────── profile.duplicate_sessions = int(df["session"].duplicated().sum()) # ── Missing NYSE sessions (approximate — business days only) ─────── # Use business day frequency as proxy; exchange_calendar would be more precise try: bdays = pd.bdate_range( start=requested_start, end=min(requested_end, df["session"].max()), ) expected_sessions = set(b.date() for b in bdays) actual_sessions = set(df["session"].tolist()) missing = sorted(expected_sessions - actual_sessions) profile.missing_sessions = len(missing) # Only record up to 20 sample missing dates to keep payload manageable profile.missing_session_dates = [str(d) for d in missing[:20]] except Exception: profile.missing_sessions = 0 # ── Price characteristics ────────────────────────────────────────── closes = df["Close"].values profile.min_price = float(np.nanmin(closes)) profile.max_price = float(np.nanmax(closes)) if profile.min_price and profile.min_price > 0: profile.price_range_pct = round( (profile.max_price - profile.min_price) / profile.min_price * 100, 2 ) profile.zero_or_negative_closes = int((closes <= 0).sum()) # ── OHLC invariant violations ────────────────────────────────────── violations = ( (df["High"] < df["Low"]) | (df["High"] < df["Close"]) | (df["High"] < df["Open"]) | (df["Low"] > df["Close"]) | (df["Low"] > df["Open"]) ) profile.ohlc_violations_detected = int(violations.sum()) # ── Volume characteristics ───────────────────────────────────────── volumes = df["Volume"].values profile.zero_volume_sessions = int((volumes == 0).sum()) profile.average_volume = round(float(np.nanmean(volumes)), 0) profile.min_volume = float(np.nanmin(volumes)) profile.max_volume = float(np.nanmax(volumes)) # ── Flatline detection (10+ consecutive identical closes) ────────── if len(closes) >= 2: # Find runs of identical consecutive close prices diffs = np.diff(closes) same = (diffs == 0).astype(int) max_run = 0 flatline_count = 0 current_run = 0 for s in same: if s == 1: current_run += 1 if current_run >= 9: # 10th identical = flatline if current_run == 9: flatline_count += 1 max_run = max(max_run, current_run + 1) else: current_run = 0 profile.flatline_sequences = flatline_count profile.max_flatline_length = max_run logger.debug( f"Profile({ticker}/{provider}): {profile.row_count} bars, " f"{profile.missing_sessions} missing, {profile.ohlc_violations_detected} OHLC violations" ) return profile