Sync from GitHub (tests passed)

app/main.py

@@ -1639,17 +1639,21 @@ async def get_pipeline_run_status(
     "/api/analysis/consensus",
     response_model=ConsensusSignal,
     summary="Get consensus signal",
-    description="Combines XGBoost and TFT-ASRO signals into a directional consensus."
+    description=(
+        "Combines the latest XGBoost snapshot and snapshot-first TFT-ASRO "
+        "signal into a directional consensus."
+    ),
 )
 async def get_consensus(
     symbol: str = Query(default=TARGET_SYMBOL, description="Trading symbol")
 ):
-    from deep_learning.inference.predictor import ensemble_directional_vote, generate_tft_analysis
+    from deep_learning.inference.predictor import ensemble_directional_vote
     
-    # 1. Get TFT analysis
+    # 1. Reuse the TFT endpoint's snapshot-first contract. This keeps the
+    # consensus response semantically identical while avoiding a fresh model
+    # inference when the pipeline already persisted a valid prediction.
     try:
-        with SessionLocal() as session:
-            tft_result = generate_tft_analysis(session, symbol)
+        tft_result = await get_tft_analysis(symbol=symbol, source="snapshot")
             
         if "error" in tft_result:
             raise HTTPException(status_code=500, detail=tft_result["error"])

config/symbol_sets/challenger.json

@@ -1,10 +1,10 @@
 {
-  "run_id": "scr-20260907-47e71d",
-  "generated_at": "2026-09-07T07:49:34Z",
-  "screener_run_id": "scr-20260907-47e71d",
-  "content_fingerprint": "sha256:b7d332601e036e5e6ddf2aa399d963bec8d8014df3dbe004b3599ebb3ade1f9a",
-  "output_fingerprint": "sha256:27c40ee31ca83bac542e54d11c139b630915fbb65d757f86ae6422e38939a543",
-  "universe_content_fingerprint": "sha256:9d9fa74eda4fb62f607299c7c411b45dc4d7c9fa1c9087dea82014c39490ecc7",
+  "run_id": "scr-20260914-1b4575",
+  "generated_at": "2026-09-14T08:27:01Z",
+  "screener_run_id": "scr-20260914-1b4575",
+  "content_fingerprint": "sha256:c97b5ba04c019cc9b1cd437062b7ae7f3d29b62c5bd55d3be6d165d0bf851b83",
+  "output_fingerprint": "sha256:d5391b0594cabeeec50e14e9d67208389a33911eaeb04b552632a590caa9c78e",
+  "universe_content_fingerprint": "sha256:0fd8dc62f0d844427de19e4a55e54c1a8cf4129d28d95cbd86c231d20faa0f75",
   "selection_rules_version": "3.2.0",
   "target": "HG=F",
   "total_candidates": 141,
@@ -75,204 +75,204 @@
       "category": "macro_currency",
       "selection_source": "mandatory",
       "rank": 113,
-      "score_composite": 0.3322,
+      "score_composite": 0.3321,
       "is_pearson": -0.4365658515972885,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": -0.26481112850985244,
+      "oos_pearson": -0.26430676889238325,
       "oos_pearson_sign": "negative",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.12771342511863104,
-      "oos_rolling_std": 0.09139103113852662,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.12367444280609856,
+      "oos_rolling_std": 0.09105194847720606,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": -0.26481112850985244
+      "oos_lag_corr_at_frozen": -0.26430676889238325
     },
     {
       "ticker": "CL=F",
       "category": "commodity_energy",
       "selection_source": "mandatory",
       "rank": 139,
-      "score_composite": 0.2283,
+      "score_composite": 0.2302,
       "is_pearson": 0.34649573464844113,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": -0.009725612150347122,
+      "oos_pearson": -0.016127694916842482,
       "oos_pearson_sign": "negative",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.026040931590577132,
-      "oos_rolling_std": 0.2904904934635282,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.0191421576303892,
+      "oos_rolling_std": 0.2913467841041788,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": -0.009725612150347122
+      "oos_lag_corr_at_frozen": -0.016127694916842482
     },
     {
       "ticker": "COPX",
       "category": "etf_copper",
       "selection_source": "screener",
       "rank": 2,
-      "score_composite": 0.7528,
-      "is_pearson": 0.7841777665901276,
+      "score_composite": 0.753,
+      "is_pearson": 0.7841778917513906,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.6702190340297001,
+      "oos_pearson": 0.6711073866179069,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.5892097405886964,
-      "oos_rolling_std": 0.11536717604218495,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.5901986177405685,
+      "oos_rolling_std": 0.11491662730551566,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.6702190340297001
+      "oos_lag_corr_at_frozen": 0.6711073866179069
     },
     {
       "ticker": "PICK",
       "category": "etf_miners",
       "selection_source": "screener",
       "rank": 3,
-      "score_composite": 0.7084,
-      "is_pearson": 0.7269351028456664,
+      "score_composite": 0.7087,
+      "is_pearson": 0.7269352758776466,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.645425266898279,
+      "oos_pearson": 0.646618739434024,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.547376717358926,
-      "oos_rolling_std": 0.10044822702864298,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.5489059575210063,
+      "oos_rolling_std": 0.10030383876885804,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.645425266898279
+      "oos_lag_corr_at_frozen": 0.646618739434024
     },
     {
       "ticker": "FCX",
       "category": "miner_major",
       "selection_source": "screener",
       "rank": 4,
-      "score_composite": 0.6995,
-      "is_pearson": 0.6989203225741722,
+      "score_composite": 0.6997,
+      "is_pearson": 0.6989203826512825,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.6115947330459959,
+      "oos_pearson": 0.6123498748865847,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.5036797627061815,
-      "oos_rolling_std": 0.16570944891424616,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.5040578448259175,
+      "oos_rolling_std": 0.16506359290430392,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.6115947330459959
+      "oos_lag_corr_at_frozen": 0.6123498748865847
     },
     {
       "ticker": "SCCO",
       "category": "miner_major",
       "selection_source": "screener",
       "rank": 5,
-      "score_composite": 0.6975,
-      "is_pearson": 0.7085866952903818,
+      "score_composite": 0.6978,
+      "is_pearson": 0.7085867959347079,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.592165816993722,
+      "oos_pearson": 0.5932490392779859,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.5039968942966784,
-      "oos_rolling_std": 0.14606867230985837,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.5051401730950509,
+      "oos_rolling_std": 0.14549743380882185,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.592165816993722
+      "oos_lag_corr_at_frozen": 0.5932490392779859
     },
     {
       "ticker": "RIO",
       "category": "miner_major",
       "selection_source": "screener",
       "rank": 6,
-      "score_composite": 0.6608,
-      "is_pearson": 0.6828594972987202,
+      "score_composite": 0.6613,
+      "is_pearson": 0.6828593761093451,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.5668930737915945,
+      "oos_pearson": 0.5685016016688824,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.4367660236404706,
-      "oos_rolling_std": 0.14679906832009698,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.4389788782565666,
+      "oos_rolling_std": 0.1462264046748876,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.5668930737915945
+      "oos_lag_corr_at_frozen": 0.5685016016688824
     },
     {
       "ticker": "HBM.TO",
       "category": "miner_mid",
       "selection_source": "screener",
       "rank": 7,
-      "score_composite": 0.6601,
-      "is_pearson": 0.656051010118594,
+      "score_composite": 0.6605,
+      "is_pearson": 0.6560508716990718,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.5475275362874713,
+      "oos_pearson": 0.5487563903427952,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.4546578699383272,
-      "oos_rolling_std": 0.15329984991467707,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.45592625353736893,
+      "oos_rolling_std": 0.15272683664076614,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.5475275362874713
+      "oos_lag_corr_at_frozen": 0.5487563903427952
     },
     {
       "ticker": "BHP",
       "category": "miner_major",
       "selection_source": "screener",
       "rank": 8,
-      "score_composite": 0.6473,
-      "is_pearson": 0.6635622305490159,
+      "score_composite": 0.6478,
+      "is_pearson": 0.6635622138870493,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.5376411945718109,
+      "oos_pearson": 0.5393694863399422,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.41040121050568446,
-      "oos_rolling_std": 0.18622367162062195,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.41269070419122456,
+      "oos_rolling_std": 0.18559142511544605,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.5376411945718109
+      "oos_lag_corr_at_frozen": 0.5393694863399422
     },
     {
       "ticker": "LUN.TO",
       "category": "miner_mid",
       "selection_source": "screener",
       "rank": 9,
-      "score_composite": 0.6332,
-      "is_pearson": 0.6287677490628221,
+      "score_composite": 0.6338,
+      "is_pearson": 0.6287674535731311,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.48622276669490333,
+      "oos_pearson": 0.48823972975553603,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.39539446882770857,
-      "oos_rolling_std": 0.22798193850613174,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.39775226596186225,
+      "oos_rolling_std": 0.2275092586828792,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.48622276669490333
+      "oos_lag_corr_at_frozen": 0.48823972975553603
     },
     {
       "ticker": "XME",
       "category": "etf_metals",
       "selection_source": "screener",
       "rank": 12,
-      "score_composite": 0.613,
-      "is_pearson": 0.5943770811582014,
+      "score_composite": 0.6136,
+      "is_pearson": 0.5943770868985795,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.46939602284833737,
+      "oos_pearson": 0.47157452355251056,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.32592932042220457,
-      "oos_rolling_std": 0.17807951788305412,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.32883502642083673,
+      "oos_rolling_std": 0.17774006238393442,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.46939602284833737
+      "oos_lag_corr_at_frozen": 0.47157452355251056
     },
     {
       "ticker": "FXA",
       "category": "macro_currency",
       "selection_source": "screener",
       "rank": 17,
-      "score_composite": 0.576,
-      "is_pearson": 0.5554676133481979,
+      "score_composite": 0.5763,
+      "is_pearson": 0.5554644517228883,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.4867358920236742,
+      "oos_pearson": 0.4876982661494697,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.3014904458743425,
-      "oos_rolling_std": 0.11602988271302624,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.30302585615698363,
+      "oos_rolling_std": 0.11582806074717024,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.4867358920236742
+      "oos_lag_corr_at_frozen": 0.4876982661494697
     },
     {
       "ticker": "VALE",
@@ -280,84 +280,84 @@
       "selection_source": "screener",
       "rank": 25,
       "score_composite": 0.561,
-      "is_pearson": 0.6012555070028014,
+      "is_pearson": 0.6012555157666103,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.4466655590992748,
+      "oos_pearson": 0.44658191248463813,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.2887976221115926,
-      "oos_rolling_std": 0.1935832334479645,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.28800956163151104,
+      "oos_rolling_std": 0.19298190394297499,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.4466655590992748
+      "oos_lag_corr_at_frozen": 0.44658191248463813
     },
     {
       "ticker": "GXC",
       "category": "macro_china",
       "selection_source": "screener",
       "rank": 35,
-      "score_composite": 0.5116,
-      "is_pearson": 0.5168833751333366,
+      "score_composite": 0.5124,
+      "is_pearson": 0.5168832599054346,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.4333670088956817,
+      "oos_pearson": 0.43581070140912315,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.30581422834854566,
-      "oos_rolling_std": 0.121199484856543,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.30889120868524156,
+      "oos_rolling_std": 0.12280568468881828,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.4333670088956817
+      "oos_lag_corr_at_frozen": 0.43581070140912315
     },
     {
       "ticker": "CEW",
       "category": "macro_em",
       "selection_source": "screener",
       "rank": 37,
-      "score_composite": 0.5054,
-      "is_pearson": 0.5536232108061646,
+      "score_composite": 0.5057,
+      "is_pearson": 0.5536216688550951,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.3976319130333007,
+      "oos_pearson": 0.3985481291989441,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.17028678827216104,
-      "oos_rolling_std": 0.14252821376933172,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.17175256592658594,
+      "oos_rolling_std": 0.14250725378780832,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.3976319130333007
+      "oos_lag_corr_at_frozen": 0.3985481291989441
     },
     {
       "ticker": "PL=F",
       "category": "commodity_precious",
       "selection_source": "screener",
       "rank": 44,
-      "score_composite": 0.4786,
+      "score_composite": 0.4789,
       "is_pearson": 0.48368775603200487,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.4675441187246214,
+      "oos_pearson": 0.4683634854783861,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.3902405085328851,
-      "oos_rolling_std": 0.10480371701034612,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.39111085029770926,
+      "oos_rolling_std": 0.10455541201142945,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.4675441187246214
+      "oos_lag_corr_at_frozen": 0.4683634854783861
     },
     {
       "ticker": "ALI=F",
       "category": "commodity_base",
       "selection_source": "screener",
       "rank": 82,
-      "score_composite": 0.3856,
+      "score_composite": 0.386,
       "is_pearson": 0.5018667089390725,
       "is_n_obs": 312,
       "is_best_lag": 0,
-      "oos_pearson": 0.416083472244696,
+      "oos_pearson": 0.41765145706182727,
       "oos_pearson_sign": "positive",
-      "oos_n_obs": 140,
-      "oos_partial_corr": 0.37599757846352727,
-      "oos_rolling_std": 0.1308183978751278,
+      "oos_n_obs": 141,
+      "oos_partial_corr": 0.37763895895846994,
+      "oos_rolling_std": 0.13058008151506226,
       "oos_frozen_lag": 0,
-      "oos_lag_corr_at_frozen": 0.416083472244696
+      "oos_lag_corr_at_frozen": 0.41765145706182727
     }
   ]
 }

deep_learning/data/embeddings.py

@@ -180,6 +180,58 @@ def bytes_to_embedding(data: bytes, dim: int = _EMBEDDING_DIM) -> np.ndarray:
 # DB backfill logic
 # ---------------------------------------------------------------------------
 
+def _insert_embedding_batch(
+    session,
+    payloads: Sequence[dict[str, object]],
+) -> tuple[int, int]:
+    """Insert a batch without per-row existence queries.
+
+    ``news_processed_id`` is unique, so PostgreSQL and SQLite can safely
+    ignore a concurrent duplicate at the database boundary. The fallback
+    keeps the function usable with other SQLAlchemy dialects while still
+    reducing existence checks to one query per batch.
+
+    Returns:
+        ``(inserted_count, skipped_count)`` for the attempted payloads.
+    """
+    if not payloads:
+        return 0, 0
+
+    from app.models import NewsEmbedding
+
+    dialect = session.get_bind().dialect.name
+    if dialect == "postgresql":
+        from sqlalchemy.dialects.postgresql import insert as dialect_insert
+    elif dialect == "sqlite":
+        from sqlalchemy.dialects.sqlite import insert as dialect_insert
+    else:
+        article_ids = [int(payload["news_processed_id"]) for payload in payloads]
+        existing_ids = {
+            int(row[0])
+            for row in session.query(NewsEmbedding.news_processed_id)
+            .filter(NewsEmbedding.news_processed_id.in_(article_ids))
+            .all()
+        }
+        pending = [
+            NewsEmbedding(**payload)
+            for payload in payloads
+            if int(payload["news_processed_id"]) not in existing_ids
+        ]
+        session.add_all(pending)
+        return len(pending), len(payloads) - len(pending)
+
+    statement = (
+        dialect_insert(NewsEmbedding)
+        .values(list(payloads))
+        .on_conflict_do_nothing(
+            index_elements=[NewsEmbedding.news_processed_id]
+        )
+    )
+    result = session.execute(statement)
+    inserted = max(0, int(result.rowcount or 0))
+    return inserted, len(payloads) - inserted
+
+
 def backfill_embeddings(
     *,
     days: int = 180,
@@ -193,7 +245,7 @@ def backfill_embeddings(
         1. Query news_processed rows that lack a corresponding news_embeddings row.
         2. Extract CLS embeddings in batches.
         3. Fit or load PCA model.
-        4. Store reduced embeddings in news_embeddings table.
+        4. Store reduced embeddings in race-safe batches.
     """
     from app.db import SessionLocal
     from app.models import NewsEmbedding, NewsProcessed, NewsRaw
@@ -205,8 +257,6 @@ def backfill_embeddings(
     stats = {"embedded": 0, "skipped": 0, "pca_fitted": False}
 
     with SessionLocal() as session:
-        from sqlalchemy import text as sa_text
-
         cutoff = datetime.now(timezone.utc) - timedelta(days=days)
 
         rows = (
@@ -258,32 +308,31 @@ def backfill_embeddings(
 
         reduced = reduce_embeddings(full_embeddings, pca)
 
-        try:
-            from app.models import NewsEmbedding
-        except ImportError:
-            logger.error("NewsEmbedding model not found - run DB migration first")
-            return stats
-
-        for idx, article_id in enumerate(ids):
-            existing = session.query(NewsEmbedding).filter(
-                NewsEmbedding.news_processed_id == article_id
-            ).first()
-            if existing:
-                stats["skipped"] += 1
-                continue
-
-            emb = NewsEmbedding(
-                news_processed_id=article_id,
-                embedding_full=embedding_to_bytes(full_embeddings[idx]),
-                embedding_pca=embedding_to_bytes(reduced[idx]),
-                pca_version=f"pca{pca_dim}_v1",
+        payloads = [
+            {
+                "news_processed_id": article_id,
+                "embedding_full": embedding_to_bytes(full_embeddings[idx]),
+                "embedding_pca": embedding_to_bytes(reduced[idx]),
+                "pca_version": f"pca{pca_dim}_v1",
+            }
+            for idx, article_id in enumerate(ids)
+        ]
+
+        # Keep transactions bounded while reducing the previous N existence
+        # queries to one database-level upsert per persistence batch.
+        persist_batch_size = 200
+        for start in range(0, len(payloads), persist_batch_size):
+            batch = payloads[start : start + persist_batch_size]
+            inserted, skipped = _insert_embedding_batch(session, batch)
+            stats["embedded"] += inserted
+            stats["skipped"] += skipped
+            session.commit()
+            logger.info(
+                "Committed embedding batch: %d inserted, %d skipped, %d total",
+                inserted,
+                skipped,
+                stats["embedded"],
             )
-            session.add(emb)
-            stats["embedded"] += 1
-
-            if stats["embedded"] % 200 == 0:
-                session.commit()
-                logger.info("Committed %d embeddings so far", stats["embedded"])
 
         session.commit()