Blame for tests/test_qt3_space.py

"""Quality-Trend 3 Space: data client, ranking, rebalance rules and guards, execution, performance, API auth.
No network: Sharadar and Alpaca are faked."""
from __future__ import annotations

import json
import sys
from datetime import date, datetime
from pathlib import Path

import numpy as np
import pandas as pd
import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from qt3 import config, performance, ranking, rebalance, sharadar  # noqa: E402
from qt3.broker import BrokerError  # noqa: E402


# --------------------------------------------------------------------------- Sharadar client
class _R:
    def __init__(self, code, body):
        self.status_code, self._b = code, body

    def json(self):
        return self._b


class FakeHTTP:
    def __init__(self, pages):
        self.pages, self.calls = pages, []

    def get(self, url, params=None, timeout=None):
        self.calls.append((url, dict(params)))
        return _R(200, self.pages[len(self.calls) - 1])


def test_sharadar_paginates_and_maps_filters():
    cols = [{"name": "ticker"}, {"name": "date"}]
    s = FakeHTTP([{"datatable": {"data": [["A", "2026-10-01"]], "columns": cols}, "meta": {"next_cursor_id": "c2"}},
                  {"datatable": {"data": [["B", "2026-10-01"]], "columns": cols}, "meta": {"next_cursor_id": None}}])
    df = sharadar.query("SEP", "KEY", session=s, ticker="A,B", date__gte="2026-01-01", columns=["ticker", "date"])
    assert list(df["ticker"]) == ["A", "B"]
    p0, p1 = s.calls[0][1], s.calls[1][1]
    assert p0["date.gte"] == "2026-01-01" and p0["qopts.columns"] == "ticker,date" and "qopts.cursor_id" not in p0
    assert p1["qopts.cursor_id"] == "c2"
    with pytest.raises(sharadar.SharadarError):
        sharadar.query("SEP", "", session=s)


# --------------------------------------------------------------------------- ranking (steps 4-5)
def _universe(n=40, seed=0):
    rng = np.random.default_rng(seed)
    u = pd.DataFrame({"ticker": [f"T{i:02d}" for i in range(n)], "price": 50.0, "dv20": 2e7,
                      "currency": "USD", "ma50_200": rng.normal(0, .1, n), "mom_6_1": rng.normal(0, .2, n)})
    for c in ("roe", "roa", "gross_margin", "fcf_conversion", "fcf_yield", "buyback_yield",
              "net_share_reduction_yoy", "basic_diluted_gap", "equity_financing"):
        u[c] = rng.normal(0, 1, n)
    return u


def test_rank_table_pool_and_order():
    u = _universe()
    u.loc[0, "currency"] = "JPY"          # foreign reporter excluded
    u.loc[1, "price"] = 3.0               # below $5 excluded
    pool = ranking.rank_table(u)
    assert "T00" not in set(pool["ticker"]) and "T01" not in set(pool["ticker"])
    assert pool.attrs["liquid_usd"] == 38
    assert len(pool) == pytest.approx(38 * 0.3, abs=1)
    assert pool["trend"].is_monotonic_decreasing and list(pool["rank"]) == list(range(1, len(pool) + 1))


def test_build_uses_api_tables_and_returns_ranked(monkeypatch):
    u = _universe(60)
    days = pd.bdate_range(end="2026-10-09", periods=260)
    def fetch(table, key, **f):
        assert table == "DAILY"
        return pd.DataFrame({"ticker": u["ticker"], "date": "2026-10-09", "marketcap": np.arange(60, 0, -1) * 1e3})
    def fetch_t(table, key, tickers, **f):
        if table == "TICKERS":
            return pd.concat([pd.DataFrame({"table": t, "ticker": tickers, "name": "Co", "sector": "Tech",
                                            "industry": "X", "currency": "USD"}) for t in ("SEP", "SF1")])
        if table == "SEP":
            rows = []
            for i, t in enumerate(tickers):
                c = 100 * np.exp(np.cumsum(np.full(len(days), (i - 30) / 6000)))
                rows.append(pd.DataFrame({"ticker": t, "date": days, "close": c, "volume": 1e6, "closeadj": c}))
            return pd.concat(rows)
        return pd.DataFrame()                         # SF1 empty → no quality inputs → empty pool is fine
    r = ranking.build("KEY", today=datetime(2026, 10, 10), fetch=fetch, fetch_tickers=fetch_t)
    assert r["prices_as_of"] == "2026-10-09" and r["universe"] == 60


# --------------------------------------------------------------------------- rebalance rules
RANKED = ["A", "B", "C", "D", "E", "F", "G", "H"]


def test_target_list_keeps_top6_and_refills():
    final, why = rebalance.target_list(RANKED, ["F", "H", "ZZ"])
    assert final == ["A", "B", "F"]                       # F (#6) kept; H (#8) and ZZ sold; refill A, B
    assert "#8" in why["H"] and "left the quality pool" in why["ZZ"]
    assert rebalance.target_list(RANKED, [])[0] == ["A", "B", "C"]


def test_plan_initial_equal_thirds_with_buffer():
    px = {s: 100.0 for s in "ABC"}
    p = rebalance.plan(["A", "B", "C"], {}, px, cash=10_000.0, fractional=True)
    buys = {o["symbol"]: o["value"] for o in p["orders"]}
    assert set(buys) == {"A", "B", "C"} and all(v == pytest.approx(3300, abs=1) for v in buys.values())
    assert p["cash_after"] == pytest.approx(100, abs=1)


def test_plan_sells_first_exit_and_trim():
    px = {"A": 200.0, "B": 100.0, "C": 100.0, "OLD": 10.0}
    p = rebalance.plan(["A", "B", "C"], {"A": 30, "B": 10, "OLD": 100}, px, cash=0.0,
                       reasons={"OLD": "now ranked #9"}, fractional=True)
    sides = [(o["symbol"], o["side"]) for o in p["orders"]]
    assert sides[0] == ("OLD", "sell") and p["orders"][0]["close_all"]
    assert ("A", "sell") in sides and sides.index(("A", "sell")) < sides.index(("C", "buy"))


def test_plan_whole_shares_and_missing_price():
    p = rebalance.plan(["A", "B", "C"], {}, {s: 333.0 for s in "ABC"}, cash=3000.0, fractional=False)
    assert all(float(o["shares"]).is_integer() for o in p["orders"])
    with pytest.raises(ValueError, match="C"):
        rebalance.plan(["A", "B", "C"], {}, {"A": 1, "B": 1}, cash=10.0)


# --------------------------------------------------------------------------- guards + execution (fake Alpaca)
class FakeAlpaca:
    def __init__(self, open_=True, orders=(), days=None, fill="filled"):
        self.open_, self._orders, self.fill = open_, list(orders), fill
        self._days = days or ["2026-11-02", "2026-11-03", "2026-11-04", "2026-11-05", "2026-11-06", "2026-11-09",
                              "2026-12-01"]
        self.submitted = []
        self.cash = 0.0

    def clock(self):
        return {"is_open": self.open_, "next_open": "2026-11-03T09:30:00-05:00"}

    def trading_days(self, s, e):
        return self._days

    def orders(self, after=None, status="all", limit=500):
        return self._orders

    def submit(self, symbol, side, qty=None, notional=None, client_order_id=None):
        self.submitted.append({"symbol": symbol, "side": side, "qty": qty, "notional": notional, "cid": client_order_id})
        if side == "sell":
            self.cash += 1000.0
        return {"id": f"o{len(self.submitted)}", "status": "accepted"}

    def wait(self, ids, seconds=0, poll=0):
        return {i: {"status": self.fill} for i in ids}

    def account(self):
        return {"cash": self.cash, "non_marginable_buying_power": self.cash, "equity": 10_000.0}


NOV2 = datetime(2026, 11, 2, 10, 30, tzinfo=rebalance.NY)
FRESH = {"ranked": [{"ticker": t} for t in RANKED], "prices_as_of": "2026-10-30"}


def test_check_first_run_and_guards():
    assert rebalance.check(FakeAlpaca(), FRESH, NOV2)["ok"]                          # first run, day 1, open
    assert "market closed" in rebalance.check(FakeAlpaca(open_=False), FRESH, NOV2)["reasons"][0]
    stale = {**FRESH, "prices_as_of": "2026-10-20"}
    assert any("days old" in r for r in rebalance.check(FakeAlpaca(), stale, NOV2)["reasons"])
    done = [{"client_order_id": "qt3-20261102-A-buy", "side": "buy", "status": "filled",
             "created_at": "2026-11-02T15:00:01.123456789Z"}]
    c = rebalance.check(FakeAlpaca(orders=done), FRESH, NOV2)
    assert not c["ok"] and "already rebalanced this month" in c["reasons"] and c["done_this_month"]
    old = [{"client_order_id": "qt3-20261001-A-buy", "side": "buy", "status": "filled",
            "created_at": "2026-10-01T15:00:00Z"}]
    late = datetime(2026, 11, 9, 10, 30, tzinfo=rebalance.NY)                          # 6th trading day
    assert any("window" in r for r in rebalance.check(FakeAlpaca(orders=old), FRESH, late)["reasons"])
    fresh_late = {**FRESH, "prices_as_of": "2026-11-06"}
    assert rebalance.check(FakeAlpaca(orders=old), fresh_late, late, force=True)["ok"]      # force skips window
    assert not rebalance.check(FakeAlpaca(orders=old), FRESH, late, force=True)["ok"]      # never stale data


def test_execute_sells_then_scaled_buys_with_ids():
    a = FakeAlpaca()
    p = {"fractional": True, "orders": [
        {"symbol": "OLD", "side": "sell", "shares": 5.0, "value": 1000.0},
        {"symbol": "A", "side": "buy", "shares": 1.0, "value": 600.0},
        {"symbol": "B", "side": "buy", "shares": 1.0, "value": 600.0}]}
    res = rebalance.execute(a, p, today=date(2026, 11, 2))
    assert [r["side"] for r in res] == ["sell", "buy", "buy"]
    assert a.submitted[0]["cid"] == "qt3-20261102-OLD-sell"
    # cash after sells 1000 − 1% of equity (100) = 900 → buys scaled 900/1200
    assert a.submitted[1]["notional"] == pytest.approx(450.0) and a.submitted[1]["cid"] == "qt3-20261102-A-buy"


def test_execute_defers_buys_if_sells_not_filled():
    a = FakeAlpaca(fill="new")
    res = rebalance.execute(a, {"fractional": True, "orders": [
        {"symbol": "OLD", "side": "sell", "shares": 1.0, "value": 10.0},
        {"symbol": "A", "side": "buy", "shares": 1.0, "value": 10.0}]}, today=date(2026, 11, 2), wait_s=0)
    assert res[-1]["status"].startswith("deferred") and all(s["side"] == "sell" for s in a.submitted)


# --------------------------------------------------------------------------- performance
def test_twr_excludes_deposits_and_drawdown():
    g = performance.twr(["d1", "d2", "d3"], [1000.0, 2100.0, 1890.0], {"d2": 1000.0})
    assert g[1] == pytest.approx(1.10) and g[2] == pytest.approx(0.99)
    assert performance.max_drawdown([1.0, 1.2, 0.9, 1.3]) == pytest.approx(-0.25)


# --------------------------------------------------------------------------- API auth
def test_rebalance_endpoint_requires_token(monkeypatch, tmp_path):
    from fastapi.testclient import TestClient
    monkeypatch.setattr(config, "REBALANCE_TOKEN", "secret-token")
    monkeypatch.setattr(config, "NDL_KEY", "")
    monkeypatch.setattr(config, "CACHE_DIR", str(tmp_path))
    (tmp_path / "ranking.json").write_text(json.dumps({**FRESH, "built": datetime.now().astimezone().isoformat()}))
    import app as space
    c = TestClient(space.app)
    assert c.get("/api/health").json()["rebalance_enabled"]
    r = c.post("/api/rebalance?dry=0", headers={"X-Rebalance-Token": "wrong"})
    assert r.status_code == 403


# --------------------------------------------------------------------------- in-Space scheduler
def test_scheduler_window_and_guarded_run(monkeypatch):
    from qt3 import scheduler
    ny = rebalance.NY
    assert scheduler.due(datetime(2026, 11, 2, 10, 15, tzinfo=ny))          # Monday 10:15 New York
    assert not scheduler.due(datetime(2026, 11, 2, 9, 45, tzinfo=ny))      # before 10:00
    assert not scheduler.due(datetime(2026, 11, 2, 15, 45, tzinfo=ny))     # after 15:30
    assert not scheduler.due(datetime(2026, 11, 7, 11, 0, tzinfo=ny))      # Saturday
    monkeypatch.setattr(ranking, "refresh", lambda force=False: FRESH)
    a = FakeAlpaca(open_=False)
    out = scheduler.run_once(alpaca=a)
    assert not out["executed"] and "market closed" in out["reasons"][0] and a.submitted == []
    assert scheduler._lock.acquire(blocking=False)                        # lock released afterwards
    scheduler._lock.release()


def test_scheduler_not_started_without_keys(monkeypatch):
    from qt3 import scheduler
    monkeypatch.setattr(config, "ALPACA_KEY", "")
    monkeypatch.setattr(scheduler, "STATE", {**scheduler.STATE, "running": False})
    assert scheduler.start() is False


def test_bench_value_same_money_and_deposits():
    # S&P +10% on day 2; a $500 deposit on day 3 with S&P flat → benchmark gets the same deposit
    b = performance.bench_value([1000.0, 1050.0, 1600.0], [100.0, 110.0, 110.0], ["d1", "d2", "d3"], {"d3": 500.0})
    assert b == [1000.0, 1100.0, 1600.0]


def test_rebuild_rolls_back_fills_and_deposits():
    # now: 10 A + $100 cash. A was bought (10 @ 50) on d3 and B sold (5 @ 20) on d3; $200 deposited on d3.
    fills = [{"date": "d3", "symbol": "A", "side": "buy", "qty": 10.0, "price": 50.0},
             {"date": "d3", "symbol": "B", "side": "sell", "qty": 5.0, "price": 20.0}]
    closes = {"A": {"d2": 48.0, "d3": 51.0}, "B": {"d2": 21.0, "d3": 20.0}}
    out = performance.rebuild(["d2", "d3"], 100.0, {"A": 10.0}, fills, {"d3": 200.0}, closes)
    # d2: cash 100 - 200 + 500 - 100 = 300, holding 5 B @ 21 → 405; d3: today's holdings at d3 closes → 610
    assert out == {"d2": 405.0, "d3": 610.0}
    assert performance.rebuild(["d2"], 100.0, {"A": 10.0}, [], {}, {"A": {}}) == {}   # missing close → skipped


def test_series_fills_alpaca_history_gap_and_uses_live_equity(monkeypatch):
    from datetime import datetime as real_dt
    from zoneinfo import ZoneInfo
    ny = ZoneInfo("America/New_York")

    class FixedNow(real_dt):
        @classmethod
        def now(cls, tz=None):
            return real_dt(2026, 10, 13, 11, 0, tzinfo=ny)

    class Fake:
        def portfolio_history(self):          # Alpaca has posted 10-09 (all cash) but not 10-12 yet
            return {"timestamp": [int(real_dt(2026, 10, 9, 20, 0, tzinfo=ny).timestamp())], "equity": [1000.0]}
        def orders(self, after=None, status="all"):
            return [{"client_order_id": "qt3-20261012-A-buy", "created_at": "2026-10-12T14:30:00Z", "symbol": "A",
                     "side": "buy", "filled_qty": "10", "filled_avg_price": "99", "filled_at": "2026-10-12T14:31:00Z"}]
        def account(self):
            return {"equity": 1015.0, "cash": 10.0}
        def cash_flows(self):
            return []
        def positions(self):
            return {"A": {"qty": 10.0}}
        def daily_bars(self, sym, start, adjustment="all"):
            px = {"SPY": {"2026-10-09": 500.0, "2026-10-12": 505.0}, "A": {"2026-10-12": 100.0}}[sym]
            return [{"t": d, "c": c} for d, c in px.items()]
        def latest_prices(self, syms):
            return {"SPY": 510.0}

    monkeypatch.setattr(performance, "datetime", FixedNow)
    monkeypatch.setattr(config, "INCEPTION", "")
    s = performance.series(Fake())
    assert s["dates"] == ["2026-10-09", "2026-10-12", "2026-10-13"] and s["src"] == ["alpaca", "rebuilt", "live"]
    assert s["value"] == [1000.0, 1010.0, 1015.0]                 # 10-12: $10 cash + 10 A @ 100
    assert s["bench_value"] == [1000.0, 1010.0, 1020.0] and s["start"] == "2026-10-12"


def test_ranking_endpoint_while_first_build_runs(monkeypatch, tmp_path):
    from fastapi.testclient import TestClient
    monkeypatch.setattr(config, "NDL_KEY", "")
    monkeypatch.setattr(config, "CACHE_DIR", str(tmp_path))        # no cached ranking yet
    import app as space
    r = TestClient(space.app).get("/api/ranking")
    assert r.status_code == 200 and r.json()["age_hours"] is None and "ranked" not in r.json()


def test_history_replays_rebalances_from_orders():
    from qt3 import history

    def o(day, sym, side, qty, px, at="T14:00:00Z"):
        return {"client_order_id": f"qt3-{day.replace('-', '')}-{sym}-{side}", "symbol": sym, "side": side,
                "filled_qty": str(qty), "filled_avg_price": str(px), "filled_at": day + at}
    orders = [o("2026-10-12", s, "buy", 1, 100) for s in "ABC"] + [
        o("2026-11-02", "C", "sell", 1, 120, "T14:00:00Z"), o("2026-11-02", "D", "buy", 1, 110, "T14:05:00Z"),
        o("2026-11-02", "A", "buy", 0.05, 100, "T14:05:00Z"), o("2026-11-02", "B", "sell", 0.03, 100, "T14:00:00Z"),
        {"client_order_id": "manual-1", "symbol": "Z", "side": "buy", "filled_qty": "5", "filled_avg_price": "10"}]
    pos = {"A": {"market_value": 120.0}, "B": {"market_value": 90.0}, "D": {"market_value": 100.0}}
    h = history.build(orders, pos, {"2026-10-01": 10000.0, "2026-10-20": 1000.0}, {"C": "Cee Corp"})
    new, first = h["events"]
    assert first["first"] and first["bought"] == ["A", "B", "C"] and first["added"] == 0
    assert new["bought"] == ["D"] and new["kept"] == ["A", "B"] and new["added"] == 1000.0
    assert [s["symbol"] for s in new["sold"]] == ["C"] and new["sold"][0]["return"] == pytest.approx(0.2)
    assert new["sold"][0]["days"] == 21 and new["sold"][0]["name"] == "Cee Corp"
    assert {r["symbol"]: r["value"] for r in new["resplit"]} == {"A": 5.0, "B": -3.0}
    op = {r["symbol"]: r for r in h["open"]}
    assert set(op) == {"A", "B", "D"}                                   # the manual order is ignored
    assert op["A"]["return"] == pytest.approx(120 / 105 - 1) and op["B"]["return"] == pytest.approx(93 / 100 - 1)
    s = h["summary"]
    assert s["rebalances"] == 2 and s["swaps_per_rebalance"] == 1 and s["closed"] == 1 and s["win_rate"] == 1


def test_short_company_names():
    import app as space
    assert space._short_name("Micron Technology, Inc. Common Stock") == "Micron Technology, Inc."
    assert space._short_name("Alphabet Inc. Class A Common Stock") == "Alphabet Inc."
    assert space._short_name("HF Sinclair Corporation") == "HF Sinclair Corporation" and space._short_name(None) is None