PolyHedra / tests /test_decision_contracts.py
yf176
fix(strategy): model no-trade as proposal
114fd4a
Raw History Blame Contribute Delete
17.3 kB
"""Research, backtest, risk, and strategy contracts enforce decision boundaries."""
import copy
import pytest
from pydantic import ValidationError
from polyhedra.agents.quant.decision_contracts import (
BacktestPayload, BacktesterInput, BacktesterOutput, CommitteeAggregation,
QuantResearcherInput, QuantResearcherOutput, QuantStrategistInput, QuantStrategistLLMPayload,
QuantStrategistOutput, RiskAnalystInput, RiskAnalystOutput,
enrich_strategist_payload,
)
from polyhedra.agents.quant.analysis_contracts import (
SentimentAnalystOutput, TrendAnalystOutput,
)
from polyhedra.agents.quant.asset_contracts import AssetAnalysisCollection
from polyhedra.core.artifacts import JsonArtifactRef
AT = "2026-09-10T12:00:00Z"
def ref(producer, filename, run="run1"):
return JsonArtifactRef(
schema_version="8.0", artifact_id=f"{producer}-{run}-{filename}", run_id=run,
session_id="session1", producer=producer, path=f"quant/runs/{run}/{filename}",
)
def evidence(ticker="SPY"):
return {"artifact": ref("DataFetcher", f"raw/{ticker}.json"), "field_path": "/ohlcv/0/close"}
def envelope(producer, filename, payload, inputs, scope=("SPY",), **updates):
data = dict(
schema_version="8.0", artifact_id=f"{producer}-run1", run_id="run1",
session_id="session1", producer=producer, ticker_scope=list(scope), created_at=AT,
status="COMPLETE", input_refs=[ref(name, path) for name, path in inputs],
baseline_refs=[], data_gaps=[], payload=payload, render_ref=None,
)
data.update(updates)
return data
def signal(signal_id="sig1", ticker="SPY"):
return dict(
signal_id=signal_id, ticker=ticker, thesis="Breakout may persist",
direction="BULLISH", entry_conditions=["Close above resistance"],
exit_conditions=["Target or stop is reached"], confidence="MEDIUM",
assumptions=["Liquidity remains normal"],
falsification_conditions=["Close below support"], evidence_refs=[evidence(ticker)],
delta_state="NEW",
)
def research(signals=None, scope=("SPY",)):
signals = [signal(ticker=scope[0])] if signals is None else signals
return QuantResearcherOutput.model_validate(envelope(
"QuantResearcher", "signals.json", dict(
signals=signals, prioritized_signal_ids=[item["signal_id"] for item in signals],
synthesis="Evidence supports one testable hypothesis." if signals else "No signal survived.",
delta_state="NEW", data_gaps=[],
), [("TrendAnalyst", "trend_analysis.json")], scope,
))
def backtest(method="HISTORICAL_SIMULATION", strength="SUFFICIENT", **updates):
result = dict(
signal_id="sig1", ticker="SPY", observations=90, occurrences=8, win_rate=0.625,
average_return=0.012, cumulative_return=0.096, maximum_drawdown=-0.04,
sharpe_ratio=0.8, evidence_strength=strength, evidence_refs=[evidence()], data_gaps=[],
)
if strength == "NOT_TESTABLE":
result.update(observations=None, occurrences=None, win_rate=None, average_return=None,
cumulative_return=None, maximum_drawdown=None, sharpe_ratio=None,
evidence_refs=[], data_gaps=[dict(
component="OHLCV", code="INSUFFICIENT_HISTORY",
message="No valid test window", ticker="SPY")])
result.update(updates.pop("result_updates", {}))
payload = dict(
method=method,
tested_from="2026-06-01" if method == "HISTORICAL_SIMULATION" else None,
tested_through="2026-09-01" if method == "HISTORICAL_SIMULATION" else None,
results=[result], assumptions=["Daily close execution"],
limitations=["Paper simulation"], data_gaps=[],
)
payload.update(updates)
return BacktesterOutput.model_validate(envelope(
"Backtester", "backtest_results.json", payload,
[("QuantResearcher", "signals.json")],
))
def assessment(verdict="ACCEPTABLE", **updates):
data = dict(
signal_id="sig1", ticker="SPY", verdict=verdict, rationale="Risk is bounded.",
suggested_paper_position_limit=0.05 if verdict != "REJECT" else 0.0,
required_mitigation="Reduce size" if verdict == "ELEVATED" else None,
evidence_refs=[evidence()],
)
return {**data, **updates}
def risk(verdict="ACCEPTABLE"):
return RiskAnalystOutput.model_validate(envelope(
"RiskAnalyst", "risk_assessment.json", dict(
signal_assessments=[assessment(verdict)], portfolio_heat=0.1,
concentration_assessment="Low concentration.", correlation_assessment="Low overlap.",
maximum_loss_estimate=0.02, stress_scenarios=[dict(
name="Gap down", estimated_loss=0.03, assumptions=["Two percent overnight gap"]
)], overall_risk_summary="Risk is acceptable for paper review.", data_gaps=[],
), [("Backtester", "backtest_results.json")],
))
def proposal(**updates):
data = dict(
proposal_id="proposal1", ticker="SPY", direction="LONG",
entry_condition="Close above resistance", entry_level=105.0, target=112.0, stop=99.0,
confirmation="Volume expands", invalidation="Close below support",
human_approval_required=True, time_horizon="Five trading days", confidence="MEDIUM",
source_signal_ids=["sig1"], risk_verdict="ACCEPTABLE", mitigation=None,
rationale="Backtest and risk evidence align.",
)
return {**data, **updates}
def decision_proposal(outcome="NO_TRADE", **updates):
data = dict(
proposal_id="proposal-no-trade",
outcome=outcome,
ticker_scope=["SPY"],
confidence="MEDIUM",
rationale="No setup qualifies.",
confirmation_conditions=["A risk-approved signal is published."],
evidence_refs=[evidence()],
)
return {**data, **updates}
def llm_payload(outcome="PAPER_TRADE_SUGGESTION", proposals=None, **updates):
if proposals is None:
proposals = (
[proposal()] if outcome == "PAPER_TRADE_SUGGESTION"
else [] if outcome == "ANALYSIS_UNAVAILABLE"
else [decision_proposal(outcome)]
)
data = dict(
outcome=outcome, proposals=proposals,
no_trade_or_watch_rationale=("No setup qualifies." if outcome in {
"NO_TRADE", "WATCH_FOR_CONFIRMATION"} else None), exclusions=[],
committee_votes=[dict(
member_id="risk_skeptic", vote="BULLISH", confidence="MEDIUM",
thesis="Risk remains bounded", cited_evidence=[evidence()],
proposal_impact="Supports limited paper size",
)], proposal_updates=[], evidence_refs=[evidence()], data_gaps=[],
)
return QuantStrategistLLMPayload.model_validate({**data, **updates})
def aggregation(weight=1.1):
return CommitteeAggregation.model_validate(dict(
weighted_votes=[dict(
member_id="risk_skeptic", vote="BULLISH", trusted_weight=weight,
confidence_multiplier=1.0, weighted_score=weight,
)], tally={"BULLISH": weight}, leading_vote="BULLISH",
ignored_unknown_members=[],
))
def strategy(verdict="ACCEPTABLE"):
llm = llm_payload(proposals=[proposal(risk_verdict=verdict,
mitigation="Reduce size" if verdict == "ELEVATED" else None)])
payload = enrich_strategist_payload(
llm, weights={"risk_skeptic": 1.1}, aggregation=aggregation(), weight_updates=[],
)
return QuantStrategistOutput.model_validate(envelope(
"QuantStrategist", "trade_proposals.json", payload.model_dump(mode="json"),
[("RiskAnalyst", "risk_assessment.json")],
))
def trend_output(scope):
return TrendAnalystOutput.model_validate(envelope(
"TrendAnalyst", "trend_analysis.json", {"ticker_analyses": [dict(
ticker=ticker, price_direction="NEUTRAL", price_change_percentage=0.0,
measurement_days=5, support_levels=[], resistance_levels=[], volume_state="NORMAL",
sma_20_state="UNKNOWN", ema_20_state="UNKNOWN", moving_average_crossover="NONE",
rsi_14_state="UNKNOWN", macd_state="UNKNOWN", confidence="LOW",
evidence_refs=[evidence(ticker)], delta_state="NOT_COMPARED", data_gaps=[],
) for ticker in scope]}, [("DataFetcher", f"raw/{scope[0]}.json")], scope,
))
def sentiment_output(scope):
return SentimentAnalystOutput.model_validate(envelope(
"SentimentAnalyst", "sentiment_analysis.json", {"ticker_analyses": [dict(
ticker=ticker, source_coverage=[], article_count=0, scored_count=0,
unscored_count=0, evidence_sufficiency="THIN", aggregate_sentiment="UNKNOWN",
themes=[], earnings_sentiment=None, trend_alignment="NOT_ASSESSED",
incremental_signal_value=False, confidence="UNKNOWN", evidence_refs=[],
delta_state="NOT_COMPARED", data_gaps=[],
) for ticker in scope]}, [("DataFetcher", f"raw/{scope[0]}.json")], scope,
))
def unavailable_assets(scope):
contributions = []
for ticker in scope:
gap = dict(component="ANALYSIS", code="AGENT_UNAVAILABLE",
message=f"{ticker} analysis unavailable", ticker=ticker)
contributions.append(dict(
ticker=ticker, producer=f"{ticker}Analyst", status="UNAVAILABLE",
artifact_ref=None, analysis=None, data_gaps=[gap],
))
return AssetAnalysisCollection(
schema_version="8.0", artifact_id="asset-collection", run_id="run1",
session_id="session1", created_at=AT, ticker_scope=list(scope),
contributions=contributions,
)
def test_research_signal_ids_are_stable_and_in_scope():
valid = research()
assert valid.payload.prioritized_signal_ids == ["sig1"]
data = valid.model_dump(mode="json")
data["payload"]["prioritized_signal_ids"] = ["missing"]
with pytest.raises(ValidationError, match="does not exist"):
QuantResearcherOutput.model_validate(data)
data = valid.model_dump(mode="json")
data["payload"]["signals"][0]["ticker"] = "QQQ"
with pytest.raises(ValidationError, match="outside"):
QuantResearcherOutput.model_validate(data)
def test_empty_research_signal_list_is_valid():
assert research(signals=[]).payload.signals == []
def test_research_input_selects_general_and_specialized_branches_by_scope():
common = dict(
run_id="run1", session_id="session1", run_mode="BASELINE", as_of=AT,
optional_context={}, baseline_refs=[],
)
QuantResearcherInput.model_validate({
**common, "ticker_scope": ["AAPL"], "dependencies": {
"trend": trend_output(("AAPL",)), "sentiment": sentiment_output(("AAPL",))},
})
QuantResearcherInput.model_validate({
**common, "ticker_scope": ["SPY"], "dependencies": {
"trend": trend_output(("SPY",)), "asset_analyses": unavailable_assets(("SPY",))},
})
with pytest.raises(ValidationError, match="asset analysis collection"):
QuantResearcherInput.model_validate({
**common, "ticker_scope": ["SPY"],
"dependencies": {"trend": trend_output(("SPY",))},
})
def test_backtest_separates_simulation_from_llm_assessment():
assert backtest().payload.method == "HISTORICAL_SIMULATION"
assessed = backtest(method="LLM_ASSESSMENT", strength="NOT_TESTABLE")
assert assessed.payload.results[0].observations is None
with pytest.raises(ValidationError, match="cannot claim"):
BacktestPayload.model_validate({
**assessed.payload.model_dump(mode="json"), "tested_from": "2026-01-01",
"tested_through": "2026-02-01",
})
def test_not_testable_backtest_rejects_sharpe_metric():
with pytest.raises(ValidationError, match="empty metrics"):
backtest(
method="LLM_ASSESSMENT",
strength="NOT_TESTABLE",
result_updates={"sharpe_ratio": 0.8},
)
@pytest.mark.parametrize("updates", [
{"occurrences": 91},
{"evidence_strength": "NOT_TESTABLE"},
])
def test_backtest_rejects_incoherent_metrics(updates):
with pytest.raises(ValidationError):
backtest(result_updates=updates)
def test_risk_requires_mitigation_and_zero_rejected_size():
risk("ELEVATED")
risk("REJECT")
for bad in (
assessment("ELEVATED", required_mitigation=None),
assessment("REJECT", suggested_paper_position_limit=0.01),
):
data = risk().model_dump(mode="json")
data["payload"]["signal_assessments"] = [bad]
with pytest.raises(ValidationError):
RiskAnalystOutput.model_validate(data)
def test_risk_portfolio_context_is_unique_bounded_and_in_scope():
base = dict(
run_id="run1",
session_id="session1",
run_mode="BASELINE",
as_of=AT,
ticker_scope=["SPY"],
dependencies={"backtests": backtest()},
baseline_refs=[],
)
position = dict(
position_id="position-spy",
ticker="SPY",
direction="LONG",
entry_price=650.0,
portfolio_fraction=0.2,
)
RiskAnalystInput.model_validate({
**base,
"optional_context": {"open_positions": [position]},
})
for positions, match in (
([position, position], "unique"),
([{**position, "ticker": "QQQ"}], "outside"),
([position, {
**position,
"position_id": "position-spy-2",
"portfolio_fraction": 0.9,
}], "cannot exceed"),
):
with pytest.raises(ValidationError, match=match):
RiskAnalystInput.model_validate({
**base,
"optional_context": {"open_positions": positions},
})
def test_strategy_trade_contract_and_maximum_three():
assert strategy().payload.proposals[0].ticker == "SPY"
with pytest.raises(ValidationError):
llm_payload(proposals=[proposal(proposal_id=f"p{i}") for i in range(4)])
with pytest.raises(ValidationError, match="decision proposals"):
llm_payload(outcome="NO_TRADE", proposals=[proposal()])
with pytest.raises(ValidationError, match="requires rationale"):
llm_payload(outcome="NO_TRADE", no_trade_or_watch_rationale=None)
def test_unavailable_strategy_cannot_validate_as_no_trade():
data = strategy().model_dump(mode="json")
data["status"] = "UNAVAILABLE"
data["data_gaps"] = [dict(
component="LLM", code="AGENT_FAILED", message="No validated strategy output")]
with pytest.raises(ValidationError, match="contradicts"):
QuantStrategistOutput.model_validate(data)
def test_llm_cannot_author_runtime_owned_weights():
raw = llm_payload().model_dump(mode="json")
raw["committee_weights"] = {"risk_skeptic": 2.0}
with pytest.raises(ValidationError):
QuantStrategistLLMPayload.model_validate(raw)
enriched = enrich_strategist_payload(
llm_payload(), weights={"risk_skeptic": 1.1},
aggregation=aggregation(), weight_updates=[],
)
assert enriched.committee_weights == {"risk_skeptic": 1.1}
def test_runtime_aggregation_must_use_trusted_weights():
bad = aggregation().model_dump(mode="json")
bad["weighted_votes"][0]["weighted_score"] = 1.9
with pytest.raises(ValidationError, match="weighted score"):
enrich_strategist_payload(
llm_payload(), weights={"risk_skeptic": 1.1},
aggregation=CommitteeAggregation.model_validate(bad), weight_updates=[],
)
def test_strategy_rejects_rejected_or_unassessed_signals():
for risk_output, match in ((risk("REJECT"), "rejected"), (
RiskAnalystOutput.model_validate({
**risk().model_dump(mode="json"), "payload": {
**risk().payload.model_dump(mode="json"), "signal_assessments": []},
}), "unassessed")):
with pytest.raises(ValueError, match=match):
strategy().validate_against_risk(risk_output)
def test_downstream_inputs_require_current_dependency_identity():
backtester_input = dict(
run_id="run1", session_id="session1", run_mode="BASELINE", as_of=AT,
ticker_scope=["SPY"], dependencies={"signals": research()},
optional_context={"market_data": []}, baseline_refs=[],
)
BacktesterInput.model_validate(backtester_input)
risk_input = dict(
**{key: value for key, value in backtester_input.items() if key not in {
"dependencies", "optional_context"}}, dependencies={"backtests": backtest()},
optional_context={},
)
RiskAnalystInput.model_validate(risk_input)
strategist_input = dict(
**{key: value for key, value in backtester_input.items() if key not in {
"dependencies", "optional_context"}}, dependencies={"risk": risk()},
optional_context={"committee_performance": {
"weights": {"risk_skeptic": 1.1}, "recorded_at": AT}},
)
QuantStrategistInput.model_validate(strategist_input)
stale = copy.deepcopy(strategist_input)
stale["dependencies"]["risk"] = risk().model_copy(update={"run_id": "prior"})
with pytest.raises(ValidationError, match="run identity|another run"):
QuantStrategistInput.model_validate(stale)