Upload analyst.py with huggingface_hub

analyst.py

@@ -52,16 +52,49 @@ DEBATE_CLOSE = re.compile(r"Judge Round|\[4/5\]|── LLM Debate")
 
 # Per-stage result one-liners (label and result share a line via end=" ").
 # Each becomes a timeline milestone: {step, text, done}.
+def _pct_word(txt):
+    """'+2.99% (score:16/32 conf:100%)' → '+3.0% expected · scorecard 16/32'."""
+    m = re.search(r"([+-]?\d+(?:\.\d+)?)%", txt)
+    return f"{float(m.group(1)):+.1f}% expected" if m else txt.strip()
+
+
+def _ml_words(txt):
+    """'+10.73% (dir_acc:65% RankIC:0.183 OOS:60 MAE:13.9% feat:89)' →
+    '+10.7% expected · 65% direction accuracy out of sample · typical error ±14%'."""
+    r = re.search(r"([+-]?\d+(?:\.\d+)?)%", txt); da = re.search(r"dir_acc:(\d+)%", txt); mae = re.search(r"MAE:(\d+(?:\.\d+)?)%", txt)
+    parts = []
+    if r: parts.append(f"{float(r.group(1)):+.1f}% expected")
+    if da: parts.append(f"{da.group(1)}% direction accuracy out of sample")
+    if mae: parts.append(f"typical error ±{float(mae.group(1)):.0f}%")
+    if "low reliability" in txt: parts.append("low reliability — near-zero weight")
+    return " · ".join(parts) or txt.strip()
+
+
+def _fund_words(txt):
+    r = re.search(r"([+-]?\d+(?:\.\d+)?)%", txt); sc = re.search(r"score:(\d+/\d+)", txt)
+    out = f"{float(r.group(1)):+.1f}% expected" if r else ""
+    if sc: out += f" · scorecard {sc.group(1)}"
+    return out or txt.strip()
+
+
+def _data_words(m):
+    step = m.group(2).strip().lower()
+    nice = {"daily price history (10yr)": "10 years of daily prices", "company overview": "company profile",
+            "income statement": "income statements", "balance sheet": "balance sheets", "cash flow": "cash-flow statements",
+            "earnings": "earnings history", "news sentiment": "news & sentiment"}
+    return f"{m.group(1)} of 17 sources · {nice.get(step, step)}"
+
+
 MILESTONES = [
-    ("data", re.compile(r"\[(\d+)/17\]\s*(.+?)\.\.\."), lambda m: f"{m.group(1)}/17 · {m.group(2).strip()}", False),
-    ("features", re.compile(r"PIT fundamental features\.{3}\s*(.+)"), lambda m: m.group(1).strip(), True),
-    ("tech", re.compile(r"Technical analysis \(yearly\)\.{3}\s*(.+)"), lambda m: m.group(1).strip(), True),
-    ("fund", re.compile(r"Fundamental scorecard\.{3}\s*(.+)"), lambda m: m.group(1).strip(), True),
-    ("sent", re.compile(r"Sentiment analysis\.{3}\s*(.+)"), lambda m: m.group(1).strip(), True),
-    ("fundml", re.compile(r"Blended fair value:\s*(.+?)(?:\s{2,}|$)"), lambda m: f"fair value {m.group(1).strip()}", True),
-    ("macro", re.compile(r"Macro analysis \(yearly \+ yield curve\)\.{3}\s*(.+)"), lambda m: m.group(1).strip(), True),
-    ("ml", re.compile(r"ML ensemble \(fund\+macro features\)\.{3}\s*(.+)"), lambda m: m.group(1).strip(), True),
-    ("screen", re.compile(r"composite screen:\s*(.+)"), lambda m: m.group(1).strip(), True),
+    ("data", re.compile(r"\[(\d+)/17\]\s*(.+?)\.\.\."), _data_words, False),
+    ("features", re.compile(r"PIT fundamental features\.{3}\s*(.+)"), lambda m: "point-in-time features built — no look-ahead", True),
+    ("tech", re.compile(r"Technical analysis \(yearly\)\.{3}\s*(.+)"), lambda m: _pct_word(m.group(1)), True),
+    ("fund", re.compile(r"Fundamental scorecard\.{3}\s*(.+)"), lambda m: _fund_words(m.group(1)), True),
+    ("sent", re.compile(r"Sentiment analysis\.{3}\s*(.+)"), lambda m: _pct_word(m.group(1)), True),
+    ("fundml", re.compile(r"Blended fair value:\s*(.+?)(?:\s{2,}|$)"), lambda m: f"market composite {m.group(1).strip()}", True),
+    ("macro", re.compile(r"Macro analysis \(yearly \+ yield curve\)\.{3}\s*(.+)"), lambda m: ("yield curve normal" if "NORMAL" in m.group(1) else "yield curve inverted — caution") , True),
+    ("ml", re.compile(r"ML ensemble \(fund\+macro features\)\.{3}\s*(.+)"), lambda m: _ml_words(m.group(1)), True),
+    ("screen", re.compile(r"Panel-ML rank \(12m cross-sectional[^)]*\):\s*(.+)"), lambda m: re.sub(r"decile \d+/10 · ", "", m.group(1).split("(as-of")[0]).replace("exp ", "expected ").strip(), True),
     ("blend", re.compile(r"Quant blend:\s*(.+?)(?:\s{2,}|$)"), lambda m: m.group(1).strip(), True),
 ]
 
@@ -125,10 +158,18 @@ def _extract_debate(ld: dict) -> dict | None:
             "key_points": entry.get("key_points", []),
         }
 
+    _roles_tmp = {
+        "neutral": role("round1", "gemini"), "bull_initial": role("round1", "gpt"),
+        "bear_initial": role("round1", "kimi"), "pm": role("round2", "claude"),
+    }
+    _missing = [k for k in ("neutral", "bear_initial", "bull_initial", "pm") if not _roles_tmp.get(k)]
     return {
         "recommendation": ld.get("recommendation"),
         "vote_summary": ld.get("vote_summary"),
-        "panel_complete": ld.get("panel_complete", True),  # absent (pre-fix report) → assume complete
+        # complete only when EVERY seat produced a result (audit: AVGO's Neutral seat failed
+        # silently while the old flag — which only tracked Bull/Bear — still said complete)
+        "panel_complete": bool(ld.get("panel_complete", True)) and not _missing,
+        "missing_roles": _missing,
         "bear_model": ld.get("bear_model"),
         "p_bear": ld.get("p_bear"),
         "p_bull": ld.get("p_bull"),
@@ -199,6 +240,24 @@ def _run(job: dict) -> None:
                 job["confidence"] = report.get("confidence")
                 job["debate"] = _extract_debate(report.get("llm_debate") or {})
                 job["damodaran"] = report.get("damodaran")
+                job["blend"] = report.get("blend_sources") or []
+                job["eps_gate"] = ((report.get("fundamental_detail") or {}).get("EPS Consensus Gate")
+                                   or (report.get("pred_fund") or {}).get("eps_consensus_gate"))
+                # Uncertainty band from the price model's OUT-OF-SAMPLE error (an 80% band =
+                # ±1.28 × MAE-as-sigma proxy), only when enough independent observations exist.
+                _ml = report.get("ml_detail") or {}
+                _mae = _ml.get("mae"); _neff = _ml.get("n_oos_eff") or 0
+                if _mae and _neff and _neff >= 3 and job["expected_return"] is not None:
+                    _hw = round(1.28 * float(_mae) * 100, 1)
+                    job["band"] = {"low": round(job["expected_return"] - _hw, 1),
+                                   "high": round(job["expected_return"] + _hw, 1),
+                                   "basis": f"80% band from the price model's out-of-sample error (MAE {float(_mae)*100:.0f}%, ~{int(_neff)} independent obs)"}
+                else:
+                    job["band"] = {"low": None, "high": None,
+                                   "basis": "not estimable — too few independent out-of-sample observations"}
+                _c = job.get("confidence") or 0
+                job["confidence_label"] = ("high" if _c >= 0.65 else "moderate" if _c >= 0.30 else
+                                           "low" if _c >= 0.12 else "very low — treat the point estimate as indicative only")
             except (json.JSONDecodeError, KeyError, ValueError, OSError):
                 pass
     finally:
@@ -330,12 +389,15 @@ def _save_report(job: dict) -> None:
             "confidence": job["confidence"], "rating": job["rating"],
             "debate": job["debate"], "damodaran": job["damodaran"],
             "n_lines": len(job.get("lines") or []),
+            "blend": job.get("blend") or [], "band": job.get("band"),
+            "confidence_label": job.get("confidence_label"), "eps_gate": job.get("eps_gate"),
             "cached_at": time.time(),
             "price_at_analysis": (job.get("damodaran") or {}).get("current_price"),
             "source": job.get("source", "user"),
         }
         f = _REPORTS_DIR / f"{job['ticker']}.json"
         f.write_text(json.dumps(payload))
+        _ledger_append(payload)
         if _HF_TOKEN:
             try:
                 from huggingface_hub import HfApi
@@ -347,6 +409,78 @@ def _save_report(job: dict) -> None:
         pass
 
 
+_LEDGER = _REPORTS_DIR / "_ledger.jsonl"
+
+
+def _ledger_append(payload: dict) -> None:
+    """Append-only record of every call, so realized accuracy can be measured later
+    (the 7-day report cache prunes; the ledger never does). Mirrored to the dataset repo."""
+    try:
+        row = {"ticker": payload["ticker"], "ts": payload["cached_at"],
+               "date": dt_date_str(payload["cached_at"]),
+               "price": payload.get("price_at_analysis"),
+               "expected_return": payload.get("expected_return"), "rating": payload.get("rating"),
+               "value_implied": (payload.get("damodaran") or {}).get("value_implied"),
+               "source": payload.get("source", "user")}
+        if not row["price"] or row["expected_return"] is None:
+            return
+        with open(_LEDGER, "a") as fh:
+            fh.write(json.dumps(row) + "\n")
+        if _HF_TOKEN:
+            try:
+                from huggingface_hub import HfApi
+                HfApi().upload_file(path_or_fileobj=str(_LEDGER), path_in_repo="ledger/_ledger.jsonl",
+                                    repo_id=_REPORTS_REPO, repo_type="dataset", token=_HF_TOKEN)
+            except Exception:
+                pass
+    except Exception:
+        pass
+
+
+def dt_date_str(ts):
+    import datetime as _dt
+    return _dt.datetime.fromtimestamp(ts, _dt.timezone.utc).strftime("%Y-%m-%d")
+
+
+def track_record(min_age_days: int = 30) -> dict:
+    """Realized accuracy of past calls: for ledger rows older than min_age_days, compare the
+    predicted direction/return with the price move since. Empty until the ledger ages."""
+    rows = []
+    try:
+        if _LEDGER.exists():
+            rows = [json.loads(l) for l in _LEDGER.read_text().splitlines() if l.strip()]
+    except Exception:
+        rows = []
+    now = time.time()
+    aged = [r for r in rows if now - r["ts"] >= min_age_days * 86400]
+    total = len(rows)
+    if not aged:
+        return {"calls_logged": total, "evaluated": 0, "min_age_days": min_age_days,
+                "note": f"track record builds automatically — first reading when calls are {min_age_days} days old"}
+    # dedupe to latest aged call per ticker to avoid overweighting repeats
+    latest = {}
+    for r in aged:
+        if r["ticker"] not in latest or r["ts"] > latest[r["ticker"]]["ts"]:
+            latest[r["ticker"]] = r
+    hits = 0; errs = []; det = []
+    for r in latest.values():
+        p1 = _price_now(r["ticker"])
+        if not p1:
+            continue
+        real = p1 / r["price"] - 1.0
+        pred = (r["expected_return"] or 0) / 100.0
+        ok = (real >= 0) == (pred >= 0) if abs(pred) >= 0.02 else abs(real) < 0.05
+        hits += 1 if ok else 0
+        errs.append(abs(real - pred))
+        det.append({"ticker": r["ticker"], "date": r["date"], "rating": r["rating"],
+                    "predicted": round(pred * 100, 1), "realized_to_date": round(real * 100, 1), "direction_ok": ok})
+    n = len(det)
+    return {"calls_logged": total, "evaluated": n, "min_age_days": min_age_days,
+            "direction_hit_rate": round(hits / n, 3) if n else None,
+            "mean_abs_error_pct": round(sum(errs) / n * 100, 1) if n else None,
+            "detail": sorted(det, key=lambda x: x["date"])[-25:]}
+
+
 def _hydrate_reports() -> None:
     """On startup, pull recent reports from the dataset repo so the cache survives redeploys."""
     if not _HF_TOKEN:
@@ -522,5 +656,9 @@ def result(job_id: str) -> dict:
         "rating": job["rating"],
         "debate": job["debate"],
         "damodaran": job["damodaran"],
+        "blend": job.get("blend") or [],
+        "band": job.get("band"),
+        "confidence_label": job.get("confidence_label"),
+        "eps_gate": job.get("eps_gate"),
         "n_lines": len(job["lines"]),
     }