ifieryarrows commited on
Commit
bbf55fa
Β·
verified Β·
1 Parent(s): d88b104

Sync from GitHub (tests passed)

Browse files
app/features.py CHANGED
@@ -378,7 +378,7 @@ def build_feature_matrix(
378
 
379
 
380
  def get_feature_descriptions() -> dict[str, str]:
381
- """Get human-readable descriptions for feature names."""
382
  return {
383
  "sentiment__index": "Market Sentiment Index",
384
  "sentiment__news_count": "Daily News Volume",
@@ -398,3 +398,170 @@ def get_feature_descriptions() -> dict[str, str]:
398
  "price_sma_ratio": "Price/SMA Ratio",
399
  }
400
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
378
 
379
 
380
  def get_feature_descriptions() -> dict[str, str]:
381
+ """Get human-readable descriptions for feature names (legacy dict)."""
382
  return {
383
  "sentiment__index": "Market Sentiment Index",
384
  "sentiment__news_count": "Daily News Volume",
 
398
  "price_sma_ratio": "Price/SMA Ratio",
399
  }
400
 
401
+
402
+ # ---------------------------------------------------------------------------
403
+ # Symbol and feature-pattern dictionaries used for user-facing labels.
404
+ # Extend these whenever new proxies enter the feature universe.
405
+ # ---------------------------------------------------------------------------
406
+ _SYMBOL_LABEL_MAP: dict[str, str] = {
407
+ "HG=F": "Copper Futures",
408
+ "CL=F": "WTI Crude Oil",
409
+ "BZ=F": "Brent Crude",
410
+ "GC=F": "Gold Futures",
411
+ "SI=F": "Silver Futures",
412
+ "PL=F": "Platinum Futures",
413
+ "PA=F": "Palladium Futures",
414
+ "NG=F": "Natural Gas",
415
+ "DX-Y.NYB": "US Dollar Index",
416
+ "^TNX": "US 10Y Treasury Yield",
417
+ "^VIX": "VIX Volatility",
418
+ "^GSPC": "S&P 500 Index",
419
+ "^DJI": "Dow Jones Index",
420
+ "^IXIC": "Nasdaq Composite",
421
+ "COPX": "Global Copper Miners ETF",
422
+ "PICK": "Metals & Mining ETF",
423
+ "XME": "Metals & Mining ETF",
424
+ "SCCO": "Southern Copper",
425
+ "FCX": "Freeport-McMoRan",
426
+ "BHP": "BHP Group",
427
+ "RIO": "Rio Tinto",
428
+ "VALE": "Vale",
429
+ "GDX": "Gold Miners ETF",
430
+ "USO": "US Oil Fund",
431
+ "UUP": "US Dollar Bull ETF",
432
+ "TIP": "TIPS Bond ETF",
433
+ "EEM": "Emerging Markets ETF",
434
+ "FXI": "China Large-Cap ETF",
435
+ "DBA": "Agricultural Commodity ETF",
436
+ "DBB": "Base Metals ETF",
437
+ "DBC": "Broad Commodity ETF",
438
+ }
439
+
440
+
441
+ def _humanize_symbol(sym: str) -> str:
442
+ """Convert ticker to a user-facing label."""
443
+ if not sym:
444
+ return ""
445
+ if sym in _SYMBOL_LABEL_MAP:
446
+ return _SYMBOL_LABEL_MAP[sym]
447
+ # Strip yfinance suffixes for a cleaner display
448
+ return sym.replace("=F", "").replace("^", "").replace("-Y.NYB", " Index")
449
+
450
+
451
+ _INDICATOR_LABEL_MAP: dict[str, str] = {
452
+ "ret1": "1-day return",
453
+ "ret2": "2-day return",
454
+ "ret5": "5-day return",
455
+ "ret10": "10-day return",
456
+ "RSI_14": "14-day RSI (momentum)",
457
+ "SMA_5": "5-day moving average",
458
+ "SMA_10": "10-day moving average",
459
+ "SMA_20": "20-day moving average",
460
+ "EMA_5": "5-day exponential MA",
461
+ "EMA_10": "10-day exponential MA",
462
+ "EMA_20": "20-day exponential MA",
463
+ "vol_5": "5-day volatility",
464
+ "vol_10": "10-day volatility",
465
+ "vol_20": "20-day volatility",
466
+ "price_sma_ratio": "Price vs. 20-day MA",
467
+ "target": "Model target (next-day return)",
468
+ }
469
+
470
+
471
+ def _label_indicator(token: str) -> str:
472
+ """Return a friendly label for a bare indicator token."""
473
+ if token in _INDICATOR_LABEL_MAP:
474
+ return _INDICATOR_LABEL_MAP[token]
475
+ # Lagged return β€” e.g. lag_ret1_2 β†’ "return lag 2 days"
476
+ if token.startswith("lag_ret1_"):
477
+ n = token.replace("lag_ret1_", "")
478
+ return f"return lag {n} days"
479
+ if token.startswith("lag_ret2_"):
480
+ n = token.replace("lag_ret2_", "")
481
+ return f"2-day return lag {n}"
482
+ # Embedding features
483
+ if token.startswith("emb_") or token.startswith("news_emb_"):
484
+ return f"news embedding component {token.split('_')[-1]}"
485
+ # Sentiment features
486
+ if token.startswith("sentiment"):
487
+ if "news_count" in token:
488
+ return "daily news volume"
489
+ return "sentiment index"
490
+ # Fallback: prettify by replacing underscores and lower-casing
491
+ return token.replace("_", " ").lower()
492
+
493
+
494
+ def describe_feature(feature: str) -> dict[str, str]:
495
+ """
496
+ Return a structured, user-facing description of a feature.
497
+
498
+ Output keys:
499
+ - label: short, human-readable label (β‰ˆ 30 chars)
500
+ - description: longer description with subject and metric
501
+ - category: high-level bucket (Price, Momentum, Volatility,
502
+ Sentiment, Macro, Sector, Embedding, Other)
503
+ - time_horizon optional β€” lookback window in days when applicable
504
+ """
505
+ if not feature:
506
+ return {"label": "β€”", "description": "", "category": "Other"}
507
+
508
+ # Pattern: <SYMBOL>_<token> where token can itself contain underscores.
509
+ # We try the longest matching symbol prefix.
510
+ symbol = ""
511
+ token = feature
512
+ for known in sorted(_SYMBOL_LABEL_MAP.keys(), key=len, reverse=True):
513
+ prefix = f"{known}_"
514
+ if feature.startswith(prefix):
515
+ symbol = known
516
+ token = feature[len(prefix) :]
517
+ break
518
+
519
+ if not symbol:
520
+ # Fallback: any prefix before the first underscore *might* be a symbol
521
+ # but we cannot rely on that β€” treat as bare indicator.
522
+ indicator_label = _label_indicator(token)
523
+ else:
524
+ indicator_label = _label_indicator(token)
525
+
526
+ # Category classification
527
+ category = "Other"
528
+ t = token.lower()
529
+ if t.startswith("sentiment"):
530
+ category = "Sentiment"
531
+ elif t.startswith("emb_") or t.startswith("news_emb_"):
532
+ category = "Embedding"
533
+ elif t.startswith("rsi") or t.startswith("ret") or t.startswith("lag_ret"):
534
+ category = "Momentum"
535
+ elif t.startswith("vol"):
536
+ category = "Volatility"
537
+ elif t.startswith("sma") or t.startswith("ema") or "price_sma" in t:
538
+ category = "Trend"
539
+ elif symbol and symbol != "HG=F":
540
+ category = "Macro" if symbol in {"DX-Y.NYB", "^TNX", "^VIX", "^GSPC", "^DJI", "^IXIC"} else "Sector"
541
+
542
+ # Time horizon extraction (e.g. RSI_14, vol_10)
543
+ time_horizon = ""
544
+ import re
545
+ m = re.search(r"_(\d+)$", token)
546
+ if m:
547
+ time_horizon = f"{m.group(1)}d"
548
+
549
+ if symbol:
550
+ subject = _humanize_symbol(symbol)
551
+ label = f"{subject} Β· {indicator_label}"
552
+ description = f"{subject} β€” {indicator_label}"
553
+ else:
554
+ label = indicator_label[:1].upper() + indicator_label[1:]
555
+ description = label
556
+
557
+ # Keep label concise
558
+ if len(label) > 48:
559
+ label = label[:45] + "…"
560
+
561
+ return {
562
+ "label": label,
563
+ "description": description,
564
+ "category": category,
565
+ "time_horizon": time_horizon,
566
+ }
567
+
app/inference.py CHANGED
@@ -42,7 +42,7 @@ from app.features import (
42
  load_sentiment_data,
43
  generate_symbol_features,
44
  align_to_target_calendar,
45
- get_feature_descriptions,
46
  )
47
  from app.ai_engine import load_model, load_model_metadata
48
 
@@ -590,27 +590,21 @@ def generate_analysis_report(
590
  session, target_symbol, predicted_price
591
  )
592
 
593
- # Build influencer descriptions
594
- descriptions = get_feature_descriptions()
 
595
  top_influencers = []
596
-
597
  for item in importance[:10]:
598
  feat = item["feature"]
599
- # Try to find description
600
- desc = None
601
- for key, value in descriptions.items():
602
- if key in feat:
603
- desc = value
604
- break
605
-
606
- if desc is None:
607
- # Build from feature name
608
- desc = feat.replace("_", " ").replace(" ", " ").title()
609
-
610
  top_influencers.append({
611
  "feature": feat,
612
  "importance": item["importance"],
613
- "description": desc,
 
 
 
614
  })
615
 
616
  import math
 
42
  load_sentiment_data,
43
  generate_symbol_features,
44
  align_to_target_calendar,
45
+ get_feature_descriptions, # legacy, still re-exported
46
  )
47
  from app.ai_engine import load_model, load_model_metadata
48
 
 
590
  session, target_symbol, predicted_price
591
  )
592
 
593
+ # Build influencer descriptions using the structured describe_feature helper
594
+ # so the frontend receives label / description / category for each driver.
595
+ from .features import describe_feature
596
  top_influencers = []
597
+
598
  for item in importance[:10]:
599
  feat = item["feature"]
600
+ meta_desc = describe_feature(feat)
 
 
 
 
 
 
 
 
 
 
601
  top_influencers.append({
602
  "feature": feat,
603
  "importance": item["importance"],
604
+ "label": meta_desc["label"],
605
+ "description": meta_desc["description"],
606
+ "category": meta_desc["category"],
607
+ "time_horizon": meta_desc.get("time_horizon", ""),
608
  })
609
 
610
  import math
app/main.py CHANGED
@@ -22,7 +22,7 @@ from fastapi.middleware.cors import CORSMiddleware
22
  from sqlalchemy import func
23
 
24
  from app.db import init_db, SessionLocal, get_db_type
25
- from app.models import NewsArticle, PriceBar, DailySentiment, DailySentimentV2, AnalysisSnapshot
26
  from app.settings import get_settings
27
  from app.lock import is_pipeline_locked
28
  # NOTE: Faz 1 - API is snapshot-only, no report generation
@@ -455,12 +455,36 @@ async def get_market_heatmap(background_tasks: BackgroundTasks):
455
  from app.models import HeatmapCache
456
  from app.heatmap import refresh_market_heatmap
457
 
 
 
 
 
 
 
458
  with SessionLocal() as session:
459
  cache = session.query(HeatmapCache).first()
460
  now = datetime.now(timezone.utc)
461
 
 
 
 
 
 
 
 
 
 
462
  # If no cache or completely empty payload β€” trigger background refresh
463
- if not cache or not cache.payload_json:
 
 
 
 
 
 
 
 
 
464
  background_tasks.add_task(refresh_market_heatmap)
465
  return {
466
  "name": "CopperMind Universe",
@@ -470,32 +494,60 @@ async def get_market_heatmap(background_tasks: BackgroundTasks):
470
  "refresh_in_progress": True,
471
  "last_updated_at": None,
472
  "next_refresh_at": None,
473
- "source_delay_minutes": 15
474
- }
 
 
 
475
  }
476
-
477
  # Check if stale
478
- is_stale = now > cache.expires_at
 
 
 
 
479
  refresh_in_progress = cache.refresh_started_at is not None
480
-
 
 
 
 
 
 
 
 
 
 
 
 
 
481
  if is_stale and not refresh_in_progress:
482
- # Trigger background refresh
483
  background_tasks.add_task(refresh_market_heatmap)
484
- # Mark it as refreshing immediately so we don't trigger multiple times
485
  cache.refresh_started_at = now
486
  session.commit()
487
  refresh_in_progress = True
488
-
489
  payload = cache.payload_json
 
 
 
 
 
 
 
490
  if isinstance(payload, dict):
491
  payload["_meta"] = {
492
  "is_stale": is_stale,
493
  "refresh_in_progress": refresh_in_progress,
494
  "last_updated_at": cache.cached_at.isoformat() if cache.cached_at else None,
495
  "next_refresh_at": cache.expires_at.isoformat() if cache.expires_at else None,
496
- "source_delay_minutes": 15
 
 
 
497
  }
498
-
499
  return payload
500
 
501
 
@@ -938,14 +990,22 @@ async def get_tft_summary(
938
  variable_importance = []
939
  try:
940
  import pathlib
941
- # Use the artifact dir from config if present, or guess
 
942
  artifact_dir = pathlib.Path(config.get("feature_store", {}).get("artifact_dir", "artifacts/feature_store"))
943
  mrmr_path = artifact_dir / "latest" / "mrmr_results.json"
944
  if mrmr_path.exists():
945
  mrmr_data = json.loads(mrmr_path.read_text(encoding="utf-8"))
946
  for feat, imp in mrmr_data.get("scores", {}).items():
947
- variable_importance.append({"feature": feat, "importance": float(imp)})
948
- # Sort and take top 20
 
 
 
 
 
 
 
949
  variable_importance.sort(key=lambda x: x["importance"], reverse=True)
950
  variable_importance = variable_importance[:20]
951
  except Exception as e:
@@ -1010,3 +1070,172 @@ async def get_latest_backtest():
1010
  raise HTTPException(status_code=500, detail="Failed to parse backtest report")
1011
 
1012
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  from sqlalchemy import func
23
 
24
  from app.db import init_db, SessionLocal, get_db_type
25
+ from app.models import NewsArticle, PriceBar, DailySentiment, DailySentimentV2, AnalysisSnapshot, NewsSentimentV2, NewsProcessed, NewsRaw
26
  from app.settings import get_settings
27
  from app.lock import is_pipeline_locked
28
  # NOTE: Faz 1 - API is snapshot-only, no report generation
 
455
  from app.models import HeatmapCache
456
  from app.heatmap import refresh_market_heatmap
457
 
458
+ # Stuck refresh safety: if a refresh has been "in progress" for longer than
459
+ # this, assume the worker crashed and allow a fresh background refresh to
460
+ # be kicked off. yfinance batch fetch for the full universe finishes in
461
+ # under ~2 minutes under normal conditions.
462
+ STUCK_REFRESH_SECONDS = 180
463
+
464
  with SessionLocal() as session:
465
  cache = session.query(HeatmapCache).first()
466
  now = datetime.now(timezone.utc)
467
 
468
+ def _payload_count(payload) -> int:
469
+ if not isinstance(payload, dict):
470
+ return 0
471
+ total = 0
472
+ for grp in payload.get("children", []) or []:
473
+ for sub in grp.get("children", []) or []:
474
+ total += len(sub.get("children", []) or [])
475
+ return total
476
+
477
  # If no cache or completely empty payload β€” trigger background refresh
478
+ if not cache or not cache.payload_json or _payload_count(cache.payload_json) == 0:
479
+ # Clear any stale "in progress" flag so we don't deadlock.
480
+ if cache and cache.refresh_started_at is not None:
481
+ started = cache.refresh_started_at
482
+ if started.tzinfo is None:
483
+ started = started.replace(tzinfo=timezone.utc)
484
+ age = (now - started).total_seconds()
485
+ if age > STUCK_REFRESH_SECONDS:
486
+ cache.refresh_started_at = None
487
+ session.commit()
488
  background_tasks.add_task(refresh_market_heatmap)
489
  return {
490
  "name": "CopperMind Universe",
 
494
  "refresh_in_progress": True,
495
  "last_updated_at": None,
496
  "next_refresh_at": None,
497
+ "source_delay_minutes": 15,
498
+ "payload_count": 0,
499
+ "refresh_error": cache.refresh_error if cache else None,
500
+ "cache_state": "empty",
501
+ },
502
  }
503
+
504
  # Check if stale
505
+ expires_at = cache.expires_at
506
+ if expires_at and expires_at.tzinfo is None:
507
+ expires_at = expires_at.replace(tzinfo=timezone.utc)
508
+ is_stale = now > expires_at if expires_at else True
509
+
510
  refresh_in_progress = cache.refresh_started_at is not None
511
+ # Recover from stuck "in progress" flags
512
+ if refresh_in_progress:
513
+ started = cache.refresh_started_at
514
+ if started.tzinfo is None:
515
+ started = started.replace(tzinfo=timezone.utc)
516
+ if (now - started).total_seconds() > STUCK_REFRESH_SECONDS:
517
+ logger.warning(
518
+ "Heatmap refresh appears stuck (started %.0fs ago) β€” clearing flag",
519
+ (now - started).total_seconds(),
520
+ )
521
+ cache.refresh_started_at = None
522
+ session.commit()
523
+ refresh_in_progress = False
524
+
525
  if is_stale and not refresh_in_progress:
 
526
  background_tasks.add_task(refresh_market_heatmap)
 
527
  cache.refresh_started_at = now
528
  session.commit()
529
  refresh_in_progress = True
530
+
531
  payload = cache.payload_json
532
+ payload_count = _payload_count(payload)
533
+ cache_state = "fresh"
534
+ if is_stale:
535
+ cache_state = "stale"
536
+ if refresh_in_progress:
537
+ cache_state = "refreshing"
538
+
539
  if isinstance(payload, dict):
540
  payload["_meta"] = {
541
  "is_stale": is_stale,
542
  "refresh_in_progress": refresh_in_progress,
543
  "last_updated_at": cache.cached_at.isoformat() if cache.cached_at else None,
544
  "next_refresh_at": cache.expires_at.isoformat() if cache.expires_at else None,
545
+ "source_delay_minutes": 15,
546
+ "payload_count": payload_count,
547
+ "refresh_error": cache.refresh_error,
548
+ "cache_state": cache_state,
549
  }
550
+
551
  return payload
552
 
553
 
 
990
  variable_importance = []
991
  try:
992
  import pathlib
993
+ from .features import describe_feature
994
+
995
  artifact_dir = pathlib.Path(config.get("feature_store", {}).get("artifact_dir", "artifacts/feature_store"))
996
  mrmr_path = artifact_dir / "latest" / "mrmr_results.json"
997
  if mrmr_path.exists():
998
  mrmr_data = json.loads(mrmr_path.read_text(encoding="utf-8"))
999
  for feat, imp in mrmr_data.get("scores", {}).items():
1000
+ meta_desc = describe_feature(feat)
1001
+ variable_importance.append({
1002
+ "feature": feat,
1003
+ "importance": float(imp),
1004
+ "label": meta_desc["label"],
1005
+ "description": meta_desc["description"],
1006
+ "category": meta_desc["category"],
1007
+ "time_horizon": meta_desc.get("time_horizon", ""),
1008
+ })
1009
  variable_importance.sort(key=lambda x: x["importance"], reverse=True)
1010
  variable_importance = variable_importance[:20]
1011
  except Exception as e:
 
1070
  raise HTTPException(status_code=500, detail="Failed to parse backtest report")
1071
 
1072
 
1073
+ # =============================================================================
1074
+ # Sentiment Summary β€” Stable, DB-backed, NO LLM on the hot path.
1075
+ # =============================================================================
1076
+ #
1077
+ # Architecture contract (frontend should depend on this shape forever):
1078
+ # - `index`: blended daily sentiment in [-1, +1]
1079
+ # - `label`: Bullish / Neutral / Bearish (derived from `index`)
1080
+ # - `source`: which aggregate layer produced the value
1081
+ # ("daily_v2" | "rolling_v2" | "legacy_v1" | "none")
1082
+ # - `components`: breakdown of LLM vs FinBERT vs rule_sign contributions
1083
+ # - `trend_7d`: list of {date, index, news_count} for sparkline
1084
+ # - `recent_articles`: a small sample of latest processed headlines
1085
+ # - `data_freshness`: {oldest, newest, age_hours, article_count_24h}
1086
+ #
1087
+ # This endpoint NEVER calls an LLM. Commentary generation (which does use
1088
+ # OpenRouter) is pipeline-driven and cached in `AICommentary`.
1089
+ # =============================================================================
1090
+
1091
+ @app.get(
1092
+ "/api/sentiment/summary",
1093
+ summary="Stable sentiment summary (DB-backed, no LLM on hot path)",
1094
+ description=(
1095
+ "Returns a hybrid sentiment summary that blends FinBERT, rule-based "
1096
+ "commodity heuristics and cached LLM impact scores. Falls back "
1097
+ "gracefully when individual sources are missing."
1098
+ ),
1099
+ )
1100
+ async def get_sentiment_summary(
1101
+ days: int = Query(default=7, ge=1, le=30, description="Trend window in days"),
1102
+ recent_limit: int = Query(default=6, ge=1, le=20, description="Recent headlines to include"),
1103
+ ):
1104
+ from sqlalchemy import func, desc
1105
+
1106
+ def _label(idx: float) -> str:
1107
+ if idx > 0.10:
1108
+ return "Bullish"
1109
+ if idx < -0.10:
1110
+ return "Bearish"
1111
+ return "Neutral"
1112
+
1113
+ with SessionLocal() as session:
1114
+ now = datetime.now(timezone.utc)
1115
+ window_start = now - timedelta(days=days)
1116
+
1117
+ # ---- 1) Preferred source: DailySentimentV2 (commodity-aware) ----
1118
+ v2_rows = (
1119
+ session.query(DailySentimentV2)
1120
+ .filter(DailySentimentV2.date >= window_start)
1121
+ .order_by(DailySentimentV2.date.asc())
1122
+ .all()
1123
+ )
1124
+
1125
+ # ---- 2) Component breakdown from NewsSentimentV2 (same window) ----
1126
+ # Published date lives on NewsRaw, so we join processed β†’ raw.
1127
+ component_rows = (
1128
+ session.query(
1129
+ func.avg(NewsSentimentV2.impact_score_llm).label("avg_llm"),
1130
+ func.avg(NewsSentimentV2.finbert_pos - NewsSentimentV2.finbert_neg).label("avg_finbert"),
1131
+ func.avg(NewsSentimentV2.rule_sign).label("avg_rule"),
1132
+ func.avg(NewsSentimentV2.confidence_calibrated).label("avg_conf"),
1133
+ func.avg(NewsSentimentV2.relevance_score).label("avg_rel"),
1134
+ func.count(NewsSentimentV2.id).label("n"),
1135
+ )
1136
+ .join(NewsProcessed, NewsProcessed.id == NewsSentimentV2.news_processed_id)
1137
+ .join(NewsRaw, NewsRaw.id == NewsProcessed.raw_id)
1138
+ .filter(NewsRaw.published_at >= window_start)
1139
+ .one()
1140
+ )
1141
+
1142
+ # ---- 3) Pick the freshest possible index ----
1143
+ index_val: float = 0.0
1144
+ source = "none"
1145
+ avg_confidence: Optional[float] = None
1146
+
1147
+ if v2_rows:
1148
+ latest_v2 = v2_rows[-1]
1149
+ index_val = float(latest_v2.sentiment_index or 0.0)
1150
+ avg_confidence = float(latest_v2.avg_confidence or 0.0) if latest_v2.avg_confidence is not None else None
1151
+ source = "daily_v2"
1152
+ elif component_rows and component_rows.n and component_rows.n > 0:
1153
+ # No daily aggregate yet β€” fall back to rolling per-article avg
1154
+ llm = float(component_rows.avg_llm or 0.0)
1155
+ fb = float(component_rows.avg_finbert or 0.0)
1156
+ rule = float(component_rows.avg_rule or 0.0)
1157
+ index_val = 0.5 * llm + 0.3 * fb + 0.2 * rule
1158
+ avg_confidence = float(component_rows.avg_conf or 0.0)
1159
+ source = "rolling_v2"
1160
+ else:
1161
+ # Last-ditch fallback: legacy DailySentiment
1162
+ legacy = (
1163
+ session.query(DailySentiment)
1164
+ .order_by(DailySentiment.date.desc())
1165
+ .first()
1166
+ )
1167
+ if legacy is not None:
1168
+ index_val = float(legacy.sentiment_index or 0.0)
1169
+ source = "legacy_v1"
1170
+
1171
+ # ---- 4) Build trend series for sparkline ----
1172
+ trend_7d = [
1173
+ {
1174
+ "date": r.date.isoformat() if r.date else None,
1175
+ "index": float(r.sentiment_index or 0.0),
1176
+ "news_count": int(r.news_count or 0),
1177
+ }
1178
+ for r in v2_rows
1179
+ ]
1180
+
1181
+ # ---- 5) Recent articles (hybrid: raw news + processed + V2 score) ----
1182
+ recent_q = (
1183
+ session.query(NewsRaw, NewsProcessed, NewsSentimentV2)
1184
+ .join(NewsProcessed, NewsProcessed.raw_id == NewsRaw.id)
1185
+ .outerjoin(
1186
+ NewsSentimentV2,
1187
+ NewsSentimentV2.news_processed_id == NewsProcessed.id,
1188
+ )
1189
+ .order_by(desc(NewsRaw.published_at))
1190
+ .limit(recent_limit)
1191
+ .all()
1192
+ )
1193
+ recent_articles = []
1194
+ for raw, proc, score in recent_q:
1195
+ recent_articles.append({
1196
+ "title": getattr(raw, "title", None) or getattr(proc, "canonical_title", None) or "",
1197
+ "source": getattr(raw, "source", None),
1198
+ "url": getattr(raw, "url", None),
1199
+ "published_at": raw.published_at.isoformat() if getattr(raw, "published_at", None) else None,
1200
+ "sentiment": {
1201
+ "label": score.label if score else None,
1202
+ "final_score": float(score.final_score) if score else None,
1203
+ "relevance": float(score.relevance_score) if score else None,
1204
+ "confidence": float(score.confidence_calibrated) if score else None,
1205
+ "event_type": score.event_type if score else None,
1206
+ } if score else None,
1207
+ })
1208
+
1209
+ # ---- 6) Data freshness (lives on NewsRaw, not Processed) ----
1210
+ freshness_q = session.query(
1211
+ func.min(NewsRaw.published_at).label("oldest"),
1212
+ func.max(NewsRaw.published_at).label("newest"),
1213
+ func.count(NewsRaw.id).label("n_total"),
1214
+ ).filter(NewsRaw.published_at >= (now - timedelta(hours=24))).one()
1215
+
1216
+ newest = freshness_q.newest
1217
+ age_hours = ((now - newest).total_seconds() / 3600.0) if newest else None
1218
+
1219
+ return {
1220
+ "index": round(float(index_val), 4),
1221
+ "label": _label(index_val),
1222
+ "source": source,
1223
+ "components": {
1224
+ "llm_impact_avg": float(component_rows.avg_llm) if component_rows.avg_llm is not None else None,
1225
+ "finbert_pn_avg": float(component_rows.avg_finbert) if component_rows.avg_finbert is not None else None,
1226
+ "rule_sign_avg": float(component_rows.avg_rule) if component_rows.avg_rule is not None else None,
1227
+ "avg_confidence": avg_confidence,
1228
+ "avg_relevance": float(component_rows.avg_rel) if component_rows.avg_rel is not None else None,
1229
+ "sample_size": int(component_rows.n or 0),
1230
+ },
1231
+ "trend": trend_7d,
1232
+ "recent_articles": recent_articles,
1233
+ "data_freshness": {
1234
+ "newest": newest.isoformat() if newest else None,
1235
+ "oldest": freshness_q.oldest.isoformat() if freshness_q.oldest else None,
1236
+ "age_hours": round(age_hours, 2) if age_hours is not None else None,
1237
+ "article_count_24h": int(freshness_q.n_total or 0),
1238
+ },
1239
+ "generated_at": now.isoformat(),
1240
+ }
1241
+
app/schemas.py CHANGED
@@ -10,9 +10,12 @@ from pydantic import BaseModel, Field
10
 
11
  class Influencer(BaseModel):
12
  """Top feature influencer in the model."""
13
- feature: str = Field(..., description="Feature name")
14
  importance: float = Field(..., ge=0, le=1, description="Normalized importance score")
15
- description: Optional[str] = Field(None, description="Human-readable description")
 
 
 
16
 
17
 
18
  class DataQuality(BaseModel):
 
10
 
11
  class Influencer(BaseModel):
12
  """Top feature influencer in the model."""
13
+ feature: str = Field(..., description="Raw feature identifier (e.g. CL=F_lag_ret1_2)")
14
  importance: float = Field(..., ge=0, le=1, description="Normalized importance score")
15
+ label: Optional[str] = Field(None, description="Short, user-facing label")
16
+ description: Optional[str] = Field(None, description="Longer human-readable description")
17
+ category: Optional[str] = Field(None, description="High-level bucket (Momentum, Macro, ...)")
18
+ time_horizon: Optional[str] = Field(None, description="Lookback horizon, e.g. '14d'")
19
 
20
 
21
  class DataQuality(BaseModel):
config/symbol_sets/challenger.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
- "run_id": "scr-20260413-4fbcb2",
3
- "generated_at": "2026-04-13T15:12:28Z",
4
- "screener_run_id": "scr-20260413-4fbcb2",
5
- "content_fingerprint": "sha256:d79a3182a37a7ade36cb128876b5d0f027aa92b778a8310124f66a8be6d74170",
6
- "output_fingerprint": "sha256:606a643a9e2f90e954fcd544e9f0383d05be0c9a935a96c0e2bb8ed0ef75a346",
7
- "universe_content_fingerprint": "sha256:f2a39293bab0991c7f0ab879e1c24162457fe92aa43fbed21f428e58f53cb2ae",
8
  "selection_rules_version": "3.2.0",
9
  "target": "HG=F",
10
  "total_candidates": 141,
@@ -75,153 +75,153 @@
75
  "category": "macro_currency",
76
  "selection_source": "mandatory",
77
  "rank": 116,
78
- "score_composite": 0.3304,
79
  "is_pearson": -0.4365658515972885,
80
  "is_n_obs": 312,
81
  "is_best_lag": 0,
82
- "oos_pearson": -0.2586489771508439,
83
  "oos_pearson_sign": "negative",
84
- "oos_n_obs": 119,
85
- "oos_partial_corr": 0.15194443516874934,
86
- "oos_rolling_std": 0.0967657690847652,
87
  "oos_frozen_lag": 0,
88
- "oos_lag_corr_at_frozen": -0.2586489771508439
89
  },
90
  {
91
  "ticker": "CL=F",
92
  "category": "commodity_energy",
93
  "selection_source": "mandatory",
94
- "rank": 139,
95
- "score_composite": 0.2342,
96
  "is_pearson": 0.34649573464844113,
97
  "is_n_obs": 312,
98
  "is_best_lag": 0,
99
- "oos_pearson": 0.02938600426282767,
100
  "oos_pearson_sign": "positive",
101
- "oos_n_obs": 119,
102
- "oos_partial_corr": 0.055186240194099075,
103
- "oos_rolling_std": 0.2082867561690246,
104
  "oos_frozen_lag": 0,
105
- "oos_lag_corr_at_frozen": 0.02938600426282767
106
  },
107
  {
108
  "ticker": "COPX",
109
  "category": "etf_copper",
110
  "selection_source": "screener",
111
  "rank": 2,
112
- "score_composite": 0.7599,
113
- "is_pearson": 0.7841777787074203,
114
  "is_n_obs": 312,
115
  "is_best_lag": 0,
116
- "oos_pearson": 0.694054710593407,
117
  "oos_pearson_sign": "positive",
118
- "oos_n_obs": 119,
119
- "oos_partial_corr": 0.6275584695900277,
120
- "oos_rolling_std": 0.12230090858045142,
121
  "oos_frozen_lag": 0,
122
- "oos_lag_corr_at_frozen": 0.694054710593407
123
  },
124
  {
125
  "ticker": "PICK",
126
  "category": "etf_miners",
127
  "selection_source": "screener",
128
  "rank": 3,
129
- "score_composite": 0.7123,
130
- "is_pearson": 0.7269353514133451,
131
  "is_n_obs": 312,
132
  "is_best_lag": 0,
133
- "oos_pearson": 0.6586190466540032,
134
  "oos_pearson_sign": "positive",
135
- "oos_n_obs": 119,
136
- "oos_partial_corr": 0.5759614119140422,
137
- "oos_rolling_std": 0.09680769656403815,
138
  "oos_frozen_lag": 0,
139
- "oos_lag_corr_at_frozen": 0.6586190466540032
140
  },
141
  {
142
  "ticker": "FCX",
143
  "category": "miner_major",
144
  "selection_source": "screener",
145
  "rank": 4,
146
- "score_composite": 0.7064,
147
- "is_pearson": 0.698920319242409,
148
  "is_n_obs": 312,
149
  "is_best_lag": 0,
150
- "oos_pearson": 0.6346227587921998,
151
  "oos_pearson_sign": "positive",
152
- "oos_n_obs": 119,
153
- "oos_partial_corr": 0.5414133683835944,
154
- "oos_rolling_std": 0.17469947275766462,
155
  "oos_frozen_lag": 0,
156
- "oos_lag_corr_at_frozen": 0.6346227587921998
157
  },
158
  {
159
  "ticker": "SCCO",
160
  "category": "miner_major",
161
  "selection_source": "screener",
162
  "rank": 5,
163
- "score_composite": 0.7055,
164
- "is_pearson": 0.7085865985153059,
165
  "is_n_obs": 312,
166
  "is_best_lag": 0,
167
- "oos_pearson": 0.6189101885158909,
168
  "oos_pearson_sign": "positive",
169
- "oos_n_obs": 119,
170
- "oos_partial_corr": 0.5426690161031208,
171
- "oos_rolling_std": 0.15441153706735966,
172
  "oos_frozen_lag": 0,
173
- "oos_lag_corr_at_frozen": 0.6189101885158909
174
  },
175
  {
176
  "ticker": "HBM.TO",
177
  "category": "miner_mid",
178
  "selection_source": "screener",
179
  "rank": 6,
180
- "score_composite": 0.669,
181
- "is_pearson": 0.6560509623548416,
182
  "is_n_obs": 312,
183
  "is_best_lag": 0,
184
- "oos_pearson": 0.5769945541384507,
185
  "oos_pearson_sign": "positive",
186
- "oos_n_obs": 119,
187
- "oos_partial_corr": 0.4897437427483747,
188
- "oos_rolling_std": 0.16243346778426973,
189
  "oos_frozen_lag": 0,
190
- "oos_lag_corr_at_frozen": 0.5769945541384507
191
  },
192
  {
193
  "ticker": "RIO",
194
  "category": "miner_major",
195
  "selection_source": "screener",
196
  "rank": 7,
197
- "score_composite": 0.6689,
198
- "is_pearson": 0.6828596089961047,
199
  "is_n_obs": 312,
200
  "is_best_lag": 0,
201
- "oos_pearson": 0.5938718481026231,
202
  "oos_pearson_sign": "positive",
203
- "oos_n_obs": 119,
204
- "oos_partial_corr": 0.4763511551296136,
205
- "oos_rolling_std": 0.15947979937883844,
206
  "oos_frozen_lag": 0,
207
- "oos_lag_corr_at_frozen": 0.5938718481026231
208
  },
209
  {
210
  "ticker": "BHP",
211
  "category": "miner_major",
212
  "selection_source": "screener",
213
  "rank": 8,
214
- "score_composite": 0.6539,
215
- "is_pearson": 0.6635625032521306,
216
  "is_n_obs": 312,
217
  "is_best_lag": 0,
218
- "oos_pearson": 0.5596809501004598,
219
  "oos_pearson_sign": "positive",
220
- "oos_n_obs": 119,
221
- "oos_partial_corr": 0.44213226349935664,
222
- "oos_rolling_std": 0.19500095315219837,
223
  "oos_frozen_lag": 0,
224
- "oos_lag_corr_at_frozen": 0.5596809501004598
225
  },
226
  {
227
  "ticker": "LUN.TO",
@@ -229,135 +229,135 @@
229
  "selection_source": "screener",
230
  "rank": 9,
231
  "score_composite": 0.6355,
232
- "is_pearson": 0.6287675534487994,
233
  "is_n_obs": 312,
234
  "is_best_lag": 0,
235
- "oos_pearson": 0.4936251016058408,
236
  "oos_pearson_sign": "positive",
237
- "oos_n_obs": 119,
238
- "oos_partial_corr": 0.4060122073892502,
239
- "oos_rolling_std": 0.23876681873727557,
240
  "oos_frozen_lag": 0,
241
- "oos_lag_corr_at_frozen": 0.4936251016058408
242
  },
243
  {
244
  "ticker": "XME",
245
  "category": "etf_metals",
246
  "selection_source": "screener",
247
  "rank": 12,
248
- "score_composite": 0.6164,
249
- "is_pearson": 0.5943770396247202,
250
  "is_n_obs": 312,
251
  "is_best_lag": 0,
252
- "oos_pearson": 0.48081630395323255,
253
  "oos_pearson_sign": "positive",
254
- "oos_n_obs": 119,
255
- "oos_partial_corr": 0.3495914147297897,
256
- "oos_rolling_std": 0.17994461455823774,
257
  "oos_frozen_lag": 0,
258
- "oos_lag_corr_at_frozen": 0.48081630395323255
259
  },
260
  {
261
  "ticker": "FXA",
262
  "category": "macro_currency",
263
  "selection_source": "screener",
264
- "rank": 19,
265
- "score_composite": 0.5757,
266
- "is_pearson": 0.5554664806073706,
267
  "is_n_obs": 312,
268
  "is_best_lag": 0,
269
- "oos_pearson": 0.4858614703578012,
270
  "oos_pearson_sign": "positive",
271
- "oos_n_obs": 119,
272
- "oos_partial_corr": 0.31382731677273945,
273
- "oos_rolling_std": 0.12253800956233246,
274
  "oos_frozen_lag": 0,
275
- "oos_lag_corr_at_frozen": 0.4858614703578012
276
  },
277
  {
278
  "ticker": "VALE",
279
  "category": "miner_diversified",
280
  "selection_source": "screener",
281
  "rank": 25,
282
- "score_composite": 0.5621,
283
- "is_pearson": 0.6012557573170682,
284
  "is_n_obs": 312,
285
  "is_best_lag": 0,
286
- "oos_pearson": 0.4502258109294509,
287
  "oos_pearson_sign": "positive",
288
- "oos_n_obs": 119,
289
- "oos_partial_corr": 0.29703840630751516,
290
- "oos_rolling_std": 0.19244269660322322,
291
  "oos_frozen_lag": 0,
292
- "oos_lag_corr_at_frozen": 0.4502258109294509
293
  },
294
  {
295
  "ticker": "GXC",
296
  "category": "macro_china",
297
  "selection_source": "screener",
298
  "rank": 35,
299
- "score_composite": 0.5082,
300
- "is_pearson": 0.516883166887679,
301
  "is_n_obs": 312,
302
  "is_best_lag": 0,
303
- "oos_pearson": 0.4219665130151076,
304
  "oos_pearson_sign": "positive",
305
- "oos_n_obs": 119,
306
- "oos_partial_corr": 0.30493515682789213,
307
- "oos_rolling_std": 0.10013620451899444,
308
  "oos_frozen_lag": 0,
309
- "oos_lag_corr_at_frozen": 0.4219665130151076
310
  },
311
  {
312
  "ticker": "CEW",
313
  "category": "macro_em",
314
  "selection_source": "screener",
315
  "rank": 37,
316
- "score_composite": 0.5035,
317
- "is_pearson": 0.553621601909239,
318
  "is_n_obs": 312,
319
  "is_best_lag": 0,
320
- "oos_pearson": 0.39117454662183765,
321
  "oos_pearson_sign": "positive",
322
- "oos_n_obs": 119,
323
- "oos_partial_corr": 0.15904988103523168,
324
- "oos_rolling_std": 0.13348318196955938,
325
  "oos_frozen_lag": 0,
326
- "oos_lag_corr_at_frozen": 0.39117454662183765
327
  },
328
  {
329
  "ticker": "PL=F",
330
  "category": "commodity_precious",
331
  "selection_source": "screener",
332
  "rank": 46,
333
- "score_composite": 0.4798,
334
  "is_pearson": 0.48368775603200487,
335
  "is_n_obs": 312,
336
  "is_best_lag": 0,
337
- "oos_pearson": 0.4715259338267748,
338
  "oos_pearson_sign": "positive",
339
- "oos_n_obs": 119,
340
- "oos_partial_corr": 0.40164089670075337,
341
- "oos_rolling_std": 0.11159791597484903,
342
  "oos_frozen_lag": 0,
343
- "oos_lag_corr_at_frozen": 0.4715259338267748
344
  },
345
  {
346
  "ticker": "ALI=F",
347
  "category": "commodity_base",
348
  "selection_source": "screener",
349
- "rank": 76,
350
- "score_composite": 0.3984,
351
  "is_pearson": 0.5018667089390725,
352
  "is_n_obs": 312,
353
  "is_best_lag": 0,
354
- "oos_pearson": 0.45888606735123644,
355
  "oos_pearson_sign": "positive",
356
- "oos_n_obs": 119,
357
- "oos_partial_corr": 0.4042797861710215,
358
- "oos_rolling_std": 0.10216467096575076,
359
  "oos_frozen_lag": 0,
360
- "oos_lag_corr_at_frozen": 0.45888606735123644
361
  }
362
  ]
363
  }
 
1
  {
2
+ "run_id": "scr-20260420-dac38a",
3
+ "generated_at": "2026-04-20T05:41:09Z",
4
+ "screener_run_id": "scr-20260420-dac38a",
5
+ "content_fingerprint": "sha256:6f6c9b4c3ff7c023141aa664f16ffe61ad1df7fb42ba34ba3dc7e9529ae6be0f",
6
+ "output_fingerprint": "sha256:7a578f373ca88e247b195518479faf3e9ffba7924d2383fd52e07ceba67ccaa6",
7
+ "universe_content_fingerprint": "sha256:23c50412c519b2ac51f4dc535fdf131abe158fc515c5ffd511922983b5cb3183",
8
  "selection_rules_version": "3.2.0",
9
  "target": "HG=F",
10
  "total_candidates": 141,
 
75
  "category": "macro_currency",
76
  "selection_source": "mandatory",
77
  "rank": 116,
78
+ "score_composite": 0.3313,
79
  "is_pearson": -0.4365658515972885,
80
  "is_n_obs": 312,
81
  "is_best_lag": 0,
82
+ "oos_pearson": -0.2618243326335814,
83
  "oos_pearson_sign": "negative",
84
+ "oos_n_obs": 120,
85
+ "oos_partial_corr": 0.1430516784223634,
86
+ "oos_rolling_std": 0.09631898911932543,
87
  "oos_frozen_lag": 0,
88
+ "oos_lag_corr_at_frozen": -0.2618243326335814
89
  },
90
  {
91
  "ticker": "CL=F",
92
  "category": "commodity_energy",
93
  "selection_source": "mandatory",
94
+ "rank": 140,
95
+ "score_composite": 0.2287,
96
  "is_pearson": 0.34649573464844113,
97
  "is_n_obs": 312,
98
  "is_best_lag": 0,
99
+ "oos_pearson": 0.011333834274343644,
100
  "oos_pearson_sign": "positive",
101
+ "oos_n_obs": 120,
102
+ "oos_partial_corr": 0.04128550405134411,
103
+ "oos_rolling_std": 0.21276159118024315,
104
  "oos_frozen_lag": 0,
105
+ "oos_lag_corr_at_frozen": 0.011333834274343644
106
  },
107
  {
108
  "ticker": "COPX",
109
  "category": "etf_copper",
110
  "selection_source": "screener",
111
  "rank": 2,
112
+ "score_composite": 0.7604,
113
+ "is_pearson": 0.7841779145024922,
114
  "is_n_obs": 312,
115
  "is_best_lag": 0,
116
+ "oos_pearson": 0.6955611600748269,
117
  "oos_pearson_sign": "positive",
118
+ "oos_n_obs": 120,
119
+ "oos_partial_corr": 0.6257894634034198,
120
+ "oos_rolling_std": 0.12178437440224855,
121
  "oos_frozen_lag": 0,
122
+ "oos_lag_corr_at_frozen": 0.6955611600748269
123
  },
124
  {
125
  "ticker": "PICK",
126
  "category": "etf_miners",
127
  "selection_source": "screener",
128
  "rank": 3,
129
+ "score_composite": 0.713,
130
+ "is_pearson": 0.7269350945404545,
131
  "is_n_obs": 312,
132
  "is_best_lag": 0,
133
+ "oos_pearson": 0.6610773647423791,
134
  "oos_pearson_sign": "positive",
135
+ "oos_n_obs": 120,
136
+ "oos_partial_corr": 0.5745310800650918,
137
+ "oos_rolling_std": 0.09686836763932709,
138
  "oos_frozen_lag": 0,
139
+ "oos_lag_corr_at_frozen": 0.6610773647423791
140
  },
141
  {
142
  "ticker": "FCX",
143
  "category": "miner_major",
144
  "selection_source": "screener",
145
  "rank": 4,
146
+ "score_composite": 0.7067,
147
+ "is_pearson": 0.6989202341728037,
148
  "is_n_obs": 312,
149
  "is_best_lag": 0,
150
+ "oos_pearson": 0.6357360958336581,
151
  "oos_pearson_sign": "positive",
152
+ "oos_n_obs": 120,
153
+ "oos_partial_corr": 0.5369548430749718,
154
+ "oos_rolling_std": 0.17431010065412936,
155
  "oos_frozen_lag": 0,
156
+ "oos_lag_corr_at_frozen": 0.6357360958336581
157
  },
158
  {
159
  "ticker": "SCCO",
160
  "category": "miner_major",
161
  "selection_source": "screener",
162
  "rank": 5,
163
+ "score_composite": 0.7049,
164
+ "is_pearson": 0.7085868370035991,
165
  "is_n_obs": 312,
166
  "is_best_lag": 0,
167
+ "oos_pearson": 0.617011883283589,
168
  "oos_pearson_sign": "positive",
169
+ "oos_n_obs": 120,
170
+ "oos_partial_corr": 0.537098926474806,
171
+ "oos_rolling_std": 0.15370648062554831,
172
  "oos_frozen_lag": 0,
173
+ "oos_lag_corr_at_frozen": 0.617011883283589
174
  },
175
  {
176
  "ticker": "HBM.TO",
177
  "category": "miner_mid",
178
  "selection_source": "screener",
179
  "rank": 6,
180
+ "score_composite": 0.6694,
181
+ "is_pearson": 0.6560510108366006,
182
  "is_n_obs": 312,
183
  "is_best_lag": 0,
184
+ "oos_pearson": 0.5784132187589868,
185
  "oos_pearson_sign": "positive",
186
+ "oos_n_obs": 120,
187
+ "oos_partial_corr": 0.48810118337513614,
188
+ "oos_rolling_std": 0.16167407821610363,
189
  "oos_frozen_lag": 0,
190
+ "oos_lag_corr_at_frozen": 0.5784132187589868
191
  },
192
  {
193
  "ticker": "RIO",
194
  "category": "miner_major",
195
  "selection_source": "screener",
196
  "rank": 7,
197
+ "score_composite": 0.6692,
198
+ "is_pearson": 0.6828595562447017,
199
  "is_n_obs": 312,
200
  "is_best_lag": 0,
201
+ "oos_pearson": 0.5947325298246477,
202
  "oos_pearson_sign": "positive",
203
+ "oos_n_obs": 120,
204
+ "oos_partial_corr": 0.47266763923399996,
205
+ "oos_rolling_std": 0.15873455261146716,
206
  "oos_frozen_lag": 0,
207
+ "oos_lag_corr_at_frozen": 0.5947325298246477
208
  },
209
  {
210
  "ticker": "BHP",
211
  "category": "miner_major",
212
  "selection_source": "screener",
213
  "rank": 8,
214
+ "score_composite": 0.6549,
215
+ "is_pearson": 0.6635623759003635,
216
  "is_n_obs": 312,
217
  "is_best_lag": 0,
218
+ "oos_pearson": 0.5631072301246801,
219
  "oos_pearson_sign": "positive",
220
+ "oos_n_obs": 120,
221
+ "oos_partial_corr": 0.44293609331370554,
222
+ "oos_rolling_std": 0.19417955108947027,
223
  "oos_frozen_lag": 0,
224
+ "oos_lag_corr_at_frozen": 0.5631072301246801
225
  },
226
  {
227
  "ticker": "LUN.TO",
 
229
  "selection_source": "screener",
230
  "rank": 9,
231
  "score_composite": 0.6355,
232
+ "is_pearson": 0.6287677691319059,
233
  "is_n_obs": 312,
234
  "is_best_lag": 0,
235
+ "oos_pearson": 0.4936270291724404,
236
  "oos_pearson_sign": "positive",
237
+ "oos_n_obs": 120,
238
+ "oos_partial_corr": 0.4032586175125347,
239
+ "oos_rolling_std": 0.2376713426233714,
240
  "oos_frozen_lag": 0,
241
+ "oos_lag_corr_at_frozen": 0.4936270291724404
242
  },
243
  {
244
  "ticker": "XME",
245
  "category": "etf_metals",
246
  "selection_source": "screener",
247
  "rank": 12,
248
+ "score_composite": 0.6172,
249
+ "is_pearson": 0.594377223997752,
250
  "is_n_obs": 312,
251
  "is_best_lag": 0,
252
+ "oos_pearson": 0.4833532807101978,
253
  "oos_pearson_sign": "positive",
254
+ "oos_n_obs": 120,
255
+ "oos_partial_corr": 0.34640598070087714,
256
+ "oos_rolling_std": 0.17979569475221757,
257
  "oos_frozen_lag": 0,
258
+ "oos_lag_corr_at_frozen": 0.4833532807101978
259
  },
260
  {
261
  "ticker": "FXA",
262
  "category": "macro_currency",
263
  "selection_source": "screener",
264
+ "rank": 18,
265
+ "score_composite": 0.5769,
266
+ "is_pearson": 0.5554651420877051,
267
  "is_n_obs": 312,
268
  "is_best_lag": 0,
269
+ "oos_pearson": 0.4897520030938691,
270
  "oos_pearson_sign": "positive",
271
+ "oos_n_obs": 120,
272
+ "oos_partial_corr": 0.31370627997002987,
273
+ "oos_rolling_std": 0.12208767366985793,
274
  "oos_frozen_lag": 0,
275
+ "oos_lag_corr_at_frozen": 0.4897520030938691
276
  },
277
  {
278
  "ticker": "VALE",
279
  "category": "miner_diversified",
280
  "selection_source": "screener",
281
  "rank": 25,
282
+ "score_composite": 0.5634,
283
+ "is_pearson": 0.6012556843181778,
284
  "is_n_obs": 312,
285
  "is_best_lag": 0,
286
+ "oos_pearson": 0.4545523193177489,
287
  "oos_pearson_sign": "positive",
288
+ "oos_n_obs": 120,
289
+ "oos_partial_corr": 0.2975836396034435,
290
+ "oos_rolling_std": 0.19205512792965243,
291
  "oos_frozen_lag": 0,
292
+ "oos_lag_corr_at_frozen": 0.4545523193177489
293
  },
294
  {
295
  "ticker": "GXC",
296
  "category": "macro_china",
297
  "selection_source": "screener",
298
  "rank": 35,
299
+ "score_composite": 0.5095,
300
+ "is_pearson": 0.5168833911270554,
301
  "is_n_obs": 312,
302
  "is_best_lag": 0,
303
+ "oos_pearson": 0.4262808689477157,
304
  "oos_pearson_sign": "positive",
305
+ "oos_n_obs": 120,
306
+ "oos_partial_corr": 0.3059948465656659,
307
+ "oos_rolling_std": 0.10106729295855013,
308
  "oos_frozen_lag": 0,
309
+ "oos_lag_corr_at_frozen": 0.4262808689477157
310
  },
311
  {
312
  "ticker": "CEW",
313
  "category": "macro_em",
314
  "selection_source": "screener",
315
  "rank": 37,
316
+ "score_composite": 0.5053,
317
+ "is_pearson": 0.5536228716901678,
318
  "is_n_obs": 312,
319
  "is_best_lag": 0,
320
+ "oos_pearson": 0.3970569168071562,
321
  "oos_pearson_sign": "positive",
322
+ "oos_n_obs": 120,
323
+ "oos_partial_corr": 0.16180889750688798,
324
+ "oos_rolling_std": 0.1331504799515344,
325
  "oos_frozen_lag": 0,
326
+ "oos_lag_corr_at_frozen": 0.3970569168071562
327
  },
328
  {
329
  "ticker": "PL=F",
330
  "category": "commodity_precious",
331
  "selection_source": "screener",
332
  "rank": 46,
333
+ "score_composite": 0.4804,
334
  "is_pearson": 0.48368775603200487,
335
  "is_n_obs": 312,
336
  "is_best_lag": 0,
337
+ "oos_pearson": 0.47326309236331615,
338
  "oos_pearson_sign": "positive",
339
+ "oos_n_obs": 120,
340
+ "oos_partial_corr": 0.4017210211456023,
341
+ "oos_rolling_std": 0.1111207646530568,
342
  "oos_frozen_lag": 0,
343
+ "oos_lag_corr_at_frozen": 0.47326309236331615
344
  },
345
  {
346
  "ticker": "ALI=F",
347
  "category": "commodity_base",
348
  "selection_source": "screener",
349
+ "rank": 75,
350
+ "score_composite": 0.3999,
351
  "is_pearson": 0.5018667089390725,
352
  "is_n_obs": 312,
353
  "is_best_lag": 0,
354
+ "oos_pearson": 0.4637641383653265,
355
  "oos_pearson_sign": "positive",
356
+ "oos_n_obs": 120,
357
+ "oos_partial_corr": 0.406556194126755,
358
+ "oos_rolling_std": 0.10198573409010318,
359
  "oos_frozen_lag": 0,
360
+ "oos_lag_corr_at_frozen": 0.4637641383653265
361
  }
362
  ]
363
  }
deep_learning/inference/predictor.py CHANGED
@@ -133,7 +133,20 @@ class TFTPredictor:
133
 
134
  master_df, tv_unknown, tv_known, target_cols, last_known_price = build_tft_dataframe(session, self.cfg)
135
 
136
- logger.info("TFT predict: baseline_price=%.4f for %s", last_known_price, symbol)
 
 
 
 
 
 
 
 
 
 
 
 
 
137
 
138
  encoder_length = self.cfg.model.max_encoder_length
139
  prediction_length = self.cfg.model.max_prediction_length
@@ -198,6 +211,7 @@ class TFTPredictor:
198
  pred_for_format,
199
  quantiles=self.cfg.model.quantiles,
200
  baseline_price=last_known_price,
 
201
  )
202
 
203
  result["model_info"] = {
 
133
 
134
  master_df, tv_unknown, tv_known, target_cols, last_known_price = build_tft_dataframe(session, self.cfg)
135
 
136
+ # Extract the date of the baseline price so the frontend can show
137
+ # "% change relative to <date> close" unambiguously.
138
+ reference_price_date: Optional[str] = None
139
+ try:
140
+ if hasattr(master_df.index, "max"):
141
+ last_date = master_df.index.max()
142
+ reference_price_date = str(last_date)[:10] if last_date is not None else None
143
+ except Exception:
144
+ reference_price_date = None
145
+
146
+ logger.info(
147
+ "TFT predict: baseline_price=%.4f (date=%s) for %s",
148
+ last_known_price, reference_price_date, symbol,
149
+ )
150
 
151
  encoder_length = self.cfg.model.max_encoder_length
152
  prediction_length = self.cfg.model.max_prediction_length
 
211
  pred_for_format,
212
  quantiles=self.cfg.model.quantiles,
213
  baseline_price=last_known_price,
214
+ reference_price_date=reference_price_date,
215
  )
216
 
217
  result["model_info"] = {
deep_learning/models/tft_copper.py CHANGED
@@ -280,18 +280,25 @@ def format_prediction(
280
  raw_prediction: torch.Tensor,
281
  quantiles: Sequence[float] = (0.02, 0.10, 0.25, 0.50, 0.75, 0.90, 0.98),
282
  baseline_price: float = 1.0,
 
283
  ) -> Dict[str, Any]:
284
  """
285
  Convert raw TFT quantile output to a structured prediction dict.
286
 
287
- Args:
288
- raw_prediction: tensor of shape (prediction_length, n_quantiles)
289
- quantiles: quantile levels
290
- baseline_price: current price for return-to-price conversion
291
-
292
- Returns:
293
- Dict with per-day forecasts, confidence bands, and volatility estimate.
294
- Top-level fields use the *final* day (end of horizon) for backward compat.
 
 
 
 
 
 
295
  """
296
  import math as _math
297
 
@@ -299,44 +306,62 @@ def format_prediction(
299
  n_days = pred.shape[0]
300
  median_idx = len(quantiles) // 2
301
 
302
- # Guard: log if baseline_price is invalid (NaN prices will be sanitised
303
- # to null by the API layer's _sanitize_floats, keeping the chart clean).
304
  if _math.isnan(baseline_price) or _math.isinf(baseline_price) or baseline_price <= 0:
305
  logger.warning(
306
  "format_prediction: invalid baseline_price=%s β€” price fields will be null",
307
  baseline_price,
308
  )
309
 
310
- # Hard clamp: prevents overconfident models (VR >> 1) from producing
311
- # absurd compound prices. Copper's actual daily Οƒ β‰ˆ 0.024; capping at
312
- # ~1.25Οƒ keeps the 5-day compound under β‰ˆ16 %. The clamp is inactive
313
- # once the model is retrained with a healthy VR (0.5–1.5).
314
- _MAX_DAILY_RET = 0.03
315
-
316
- # T+1 quantile spreads (return-space distance from median).
317
- # Used as the base width for confidence bands; scaled by sqrt(d) for
318
- # later days so uncertainty grows realistically instead of compounding
319
- # tail quantiles exponentially (which would produce absurd bands).
320
- med_0 = float(np.clip(pred[0, median_idx], -_MAX_DAILY_RET, _MAX_DAILY_RET))
321
- _raw_med_0 = float(pred[0, median_idx])
322
- spread_q10 = np.clip(float(pred[0, 1]) - _raw_med_0, -_MAX_DAILY_RET, 0) if len(quantiles) > 2 else 0.0
323
- spread_q90 = np.clip(float(pred[0, -2]) - _raw_med_0, 0, _MAX_DAILY_RET) if len(quantiles) > 2 else 0.0
324
- spread_q02 = np.clip(float(pred[0, 0]) - _raw_med_0, -_MAX_DAILY_RET * 1.5, 0)
325
- spread_q98 = np.clip(float(pred[0, -1]) - _raw_med_0, 0, _MAX_DAILY_RET * 1.5)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
326
 
327
  daily_forecasts = []
328
  cum_price_med = baseline_price
329
 
330
  for d in range(n_days):
331
- med = float(np.clip(pred[d, median_idx], -_MAX_DAILY_RET, _MAX_DAILY_RET))
 
332
  cum_price_med *= (1 + med)
333
  cum_return = (cum_price_med / baseline_price) - 1.0
334
 
 
 
335
  scale = (d + 1) ** 0.5
336
 
337
  daily_forecasts.append({
338
  "day": d + 1,
339
  "daily_return": med,
 
340
  "cumulative_return": cum_return,
341
  "price_median": cum_price_med,
342
  "price_q10": cum_price_med * (1 + spread_q10 * scale),
@@ -345,15 +370,16 @@ def format_prediction(
345
  "price_q98": cum_price_med * (1 + spread_q98 * scale),
346
  })
347
 
348
- # T+1 is the primary signal (most reliable, highest signal-to-noise).
349
  first = daily_forecasts[0]
350
  last = daily_forecasts[-1]
351
  vol_estimate = (first["price_q90"] - first["price_q10"]) / (2.0 * baseline_price)
352
 
353
  return {
 
 
354
  "predicted_return_median": first["daily_return"],
355
- "predicted_return_q10": float(np.clip(pred[0, 1], -_MAX_DAILY_RET * 2, _MAX_DAILY_RET * 2)) if len(quantiles) > 2 else first["daily_return"],
356
- "predicted_return_q90": float(np.clip(pred[0, -2], -_MAX_DAILY_RET * 2, _MAX_DAILY_RET * 2)) if len(quantiles) > 2 else first["daily_return"],
357
  "predicted_price_median": first["price_median"],
358
  "predicted_price_q10": first["price_q10"],
359
  "predicted_price_q90": first["price_q90"],
@@ -364,4 +390,11 @@ def format_prediction(
364
  "weekly_price": last["price_median"],
365
  "prediction_horizon_days": n_days,
366
  "daily_forecasts": daily_forecasts,
 
 
 
 
 
 
 
367
  }
 
280
  raw_prediction: torch.Tensor,
281
  quantiles: Sequence[float] = (0.02, 0.10, 0.25, 0.50, 0.75, 0.90, 0.98),
282
  baseline_price: float = 1.0,
283
+ reference_price_date: Optional[str] = None,
284
  ) -> Dict[str, Any]:
285
  """
286
  Convert raw TFT quantile output to a structured prediction dict.
287
 
288
+ The model emits next-day *simple returns* in return-space (target was
289
+ ``close.pct_change().shift(-1)``). We therefore treat every quantile as a
290
+ daily return and compound to price using ``baseline_price`` as the
291
+ reference. The returned dict is the single source of truth: both the
292
+ headline percentage (``predicted_return_median``) and the T+1 price
293
+ (``daily_forecasts[0].price_median``) are derived from the same value.
294
+
295
+ Hard return clamps used to be set to 3% which was the root cause of the
296
+ "stuck at 3%" display bug: any time the raw median exceeded 3% it was
297
+ silently snapped to exactly 3% and the UI rendered the clamp value. We
298
+ now use a calibrated sanity-check anomaly bound (``ANOMALY_DAILY_RET``)
299
+ which only triggers on genuinely implausible moves (> ~5Γ— copper's daily
300
+ Οƒ) and logs loudly when it does. Within a reasonable range the raw
301
+ model output is passed through untouched.
302
  """
303
  import math as _math
304
 
 
306
  n_days = pred.shape[0]
307
  median_idx = len(quantiles) // 2
308
 
 
 
309
  if _math.isnan(baseline_price) or _math.isinf(baseline_price) or baseline_price <= 0:
310
  logger.warning(
311
  "format_prediction: invalid baseline_price=%s β€” price fields will be null",
312
  baseline_price,
313
  )
314
 
315
+ # ------------------------------------------------------------------
316
+ # Calibrated anomaly bound (NOT a "display cap").
317
+ # Copper daily Οƒ β‰ˆ 0.024 (2.4%). A 5Οƒ daily move (~12%) is a genuine
318
+ # regime-break event; if the model outputs that, it is almost
319
+ # certainly a bug in preprocessing / scaling, not a real forecast.
320
+ # Below this level we trust the model output as-is.
321
+ # ------------------------------------------------------------------
322
+ ANOMALY_DAILY_RET = 0.12
323
+ raw_median_0 = float(pred[0, median_idx])
324
+ anomaly_detected = abs(raw_median_0) > ANOMALY_DAILY_RET
325
+ if anomaly_detected:
326
+ logger.error(
327
+ "format_prediction: anomalous raw return detected at T+1: %.4f "
328
+ "(|r| > %.3f). Likely a scaling / target-space bug β€” the value "
329
+ "will be bounded at Β±%.2f and flagged in the response.",
330
+ raw_median_0, ANOMALY_DAILY_RET, ANOMALY_DAILY_RET,
331
+ )
332
+
333
+ def _bound(x: float) -> float:
334
+ """Only clip if outside the anomaly bound; otherwise pass through."""
335
+ if abs(x) > ANOMALY_DAILY_RET:
336
+ return float(np.sign(x) * ANOMALY_DAILY_RET)
337
+ return float(x)
338
+
339
+ # Quantile spreads (distance of each quantile from the median, in
340
+ # return-space). We do NOT clip *spreads* β€” models with a healthy
341
+ # variance ratio produce spreads of ~2Οƒ and that is fine.
342
+ raw_med_0 = raw_median_0
343
+ spread_q10 = float(pred[0, 1]) - raw_med_0 if len(quantiles) > 2 else 0.0
344
+ spread_q90 = float(pred[0, -2]) - raw_med_0 if len(quantiles) > 2 else 0.0
345
+ spread_q02 = float(pred[0, 0]) - raw_med_0
346
+ spread_q98 = float(pred[0, -1]) - raw_med_0
347
 
348
  daily_forecasts = []
349
  cum_price_med = baseline_price
350
 
351
  for d in range(n_days):
352
+ raw_med = float(pred[d, median_idx])
353
+ med = _bound(raw_med)
354
  cum_price_med *= (1 + med)
355
  cum_return = (cum_price_med / baseline_price) - 1.0
356
 
357
+ # √t spread expansion keeps multi-day uncertainty realistic instead
358
+ # of exponentially compounding tail quantiles.
359
  scale = (d + 1) ** 0.5
360
 
361
  daily_forecasts.append({
362
  "day": d + 1,
363
  "daily_return": med,
364
+ "raw_daily_return": raw_med,
365
  "cumulative_return": cum_return,
366
  "price_median": cum_price_med,
367
  "price_q10": cum_price_med * (1 + spread_q10 * scale),
 
370
  "price_q98": cum_price_med * (1 + spread_q98 * scale),
371
  })
372
 
 
373
  first = daily_forecasts[0]
374
  last = daily_forecasts[-1]
375
  vol_estimate = (first["price_q90"] - first["price_q10"]) / (2.0 * baseline_price)
376
 
377
  return {
378
+ # Single source of truth for the UI β€” headline percentage and T+1
379
+ # price are now derived from the *same* value.
380
  "predicted_return_median": first["daily_return"],
381
+ "predicted_return_q10": float(pred[0, 1]) if len(quantiles) > 2 else first["daily_return"],
382
+ "predicted_return_q90": float(pred[0, -2]) if len(quantiles) > 2 else first["daily_return"],
383
  "predicted_price_median": first["price_median"],
384
  "predicted_price_q10": first["price_q10"],
385
  "predicted_price_q90": first["price_q90"],
 
390
  "weekly_price": last["price_median"],
391
  "prediction_horizon_days": n_days,
392
  "daily_forecasts": daily_forecasts,
393
+ # Explicit contract for the frontend β€” no more guessing which price
394
+ # the percentages are relative to.
395
+ "reference_price": float(baseline_price),
396
+ "reference_price_date": reference_price_date,
397
+ "return_basis": "simple_next_day_return",
398
+ "raw_predicted_return_median": raw_median_0,
399
+ "anomaly_detected": bool(anomaly_detected),
400
  }