tradesenseAi / backend /data /validation_profiler.py
Niranjan-Murugarasu024
feat: TradeSense AI Chairman Edition production release ready for Hugging Face Spaces
0125724
Raw History Blame Contribute Delete
7.39 kB
"""
TradeSense AI β€” Dataset Validation Profiler
Profiles a canonical OHLCV DataFrame for data quality characteristics:
missing sessions, duplicate sessions, zero-volume sessions, flatline sequences,
price range anomalies, and volume statistics.
IMPORTANT: This profiler NEVER interpolates or synthesizes missing data.
A missing session is reported as missing β€” not filled.
"""
import logging
from datetime import date, datetime, timezone
from typing import List, Optional, Dict, Any
import pandas as pd
import numpy as np
from pydantic import BaseModel, Field
logger = logging.getLogger("tradesense.data.validation_profiler")
class DatasetProfile(BaseModel):
"""Safe, credential-free profile of a validated canonical dataset."""
ticker: str
provider: str
requested_start: date
requested_end: date
actual_start: Optional[date] = None
actual_end: Optional[date] = None
row_count: int = 0
# Session integrity
missing_sessions: int = 0
missing_session_dates: List[str] = Field(default_factory=list)
duplicate_sessions: int = 0
weekend_sessions_detected: int = 0
# Price characteristics
min_price: Optional[float] = None
max_price: Optional[float] = None
price_range_pct: Optional[float] = None
zero_or_negative_closes: int = 0
# Volume characteristics
zero_volume_sessions: int = 0
average_volume: Optional[float] = None
min_volume: Optional[float] = None
max_volume: Optional[float] = None
# Staleness / flatline
flatline_sequences: int = 0
max_flatline_length: int = 0
# OHLC invariant violations (before quality gate filtering)
ohlc_violations_detected: int = 0
# Retrieval metadata
retrieved_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
latency_ms: float = 0.0
# Adjustment awareness
price_series_type: str = "ADJUSTED"
adjustment_methodology: str = "PROVIDER_NATIVE"
def safe_dict(self) -> Dict[str, Any]:
data = self.model_dump()
# Convert dates to strings for JSON safety
for k in ("requested_start", "requested_end", "actual_start", "actual_end", "retrieved_at"):
if data.get(k) is not None:
data[k] = str(data[k])
return data
class DatasetValidationProfiler:
"""
Profiles a canonical OHLCV DataFrame against NYSE trading sessions.
Reports deficiencies WITHOUT synthesizing missing data.
"""
@staticmethod
def profile(
df: pd.DataFrame,
ticker: str,
provider: str,
requested_start: date,
requested_end: date,
latency_ms: float = 0.0,
price_series_type: str = "ADJUSTED",
adjustment_methodology: str = "PROVIDER_NATIVE",
) -> DatasetProfile:
"""
Build a full DatasetProfile for a canonical DataFrame.
Missing sessions are REPORTED, never filled.
"""
profile = DatasetProfile(
ticker=ticker,
provider=provider,
requested_start=requested_start,
requested_end=requested_end,
latency_ms=latency_ms,
price_series_type=price_series_type,
adjustment_methodology=adjustment_methodology,
)
if df.empty:
return profile
profile.row_count = len(df)
profile.actual_start = df["session"].min()
profile.actual_end = df["session"].max()
# ── Weekend session detection ──────────────────────────────────────
sessions_as_dates = pd.to_datetime(df["session"])
weekend_mask = sessions_as_dates.dt.dayofweek >= 5
profile.weekend_sessions_detected = int(weekend_mask.sum())
# ── Duplicate sessions ─────────────────────────────────────────────
profile.duplicate_sessions = int(df["session"].duplicated().sum())
# ── Missing NYSE sessions (approximate β€” business days only) ───────
# Use business day frequency as proxy; exchange_calendar would be more precise
try:
bdays = pd.bdate_range(
start=requested_start,
end=min(requested_end, df["session"].max()),
)
expected_sessions = set(b.date() for b in bdays)
actual_sessions = set(df["session"].tolist())
missing = sorted(expected_sessions - actual_sessions)
profile.missing_sessions = len(missing)
# Only record up to 20 sample missing dates to keep payload manageable
profile.missing_session_dates = [str(d) for d in missing[:20]]
except Exception:
profile.missing_sessions = 0
# ── Price characteristics ──────────────────────────────────────────
closes = df["Close"].values
profile.min_price = float(np.nanmin(closes))
profile.max_price = float(np.nanmax(closes))
if profile.min_price and profile.min_price > 0:
profile.price_range_pct = round(
(profile.max_price - profile.min_price) / profile.min_price * 100, 2
)
profile.zero_or_negative_closes = int((closes <= 0).sum())
# ── OHLC invariant violations ──────────────────────────────────────
violations = (
(df["High"] < df["Low"]) |
(df["High"] < df["Close"]) |
(df["High"] < df["Open"]) |
(df["Low"] > df["Close"]) |
(df["Low"] > df["Open"])
)
profile.ohlc_violations_detected = int(violations.sum())
# ── Volume characteristics ─────────────────────────────────────────
volumes = df["Volume"].values
profile.zero_volume_sessions = int((volumes == 0).sum())
profile.average_volume = round(float(np.nanmean(volumes)), 0)
profile.min_volume = float(np.nanmin(volumes))
profile.max_volume = float(np.nanmax(volumes))
# ── Flatline detection (10+ consecutive identical closes) ──────────
if len(closes) >= 2:
# Find runs of identical consecutive close prices
diffs = np.diff(closes)
same = (diffs == 0).astype(int)
max_run = 0
flatline_count = 0
current_run = 0
for s in same:
if s == 1:
current_run += 1
if current_run >= 9: # 10th identical = flatline
if current_run == 9:
flatline_count += 1
max_run = max(max_run, current_run + 1)
else:
current_run = 0
profile.flatline_sequences = flatline_count
profile.max_flatline_length = max_run
logger.debug(
f"Profile({ticker}/{provider}): {profile.row_count} bars, "
f"{profile.missing_sessions} missing, {profile.ohlc_violations_detected} OHLC violations"
)
return profile