yf176 commited on
Commit ·
1341b26
1
Parent(s): 8440fca
fix(evidence): bind downstream record citations
Browse files
src/polyhedra/agents/quant/backtester.py
CHANGED
|
@@ -144,6 +144,7 @@ class BacktesterAgent(Agent):
|
|
| 144 |
])],
|
| 145 |
})
|
| 146 |
|
|
|
|
| 147 |
_validate_backtest_payload(inputs, payload, input_refs=input_refs)
|
| 148 |
verify_evidence(payload, sources)
|
| 149 |
output_gaps = _dedupe_backtest_gaps([*upstream_gaps, *payload.data_gaps])
|
|
@@ -388,6 +389,34 @@ def _dedupe_backtest_gaps(gaps: list[DataIssue]) -> list[DataIssue]:
|
|
| 388 |
return list(unique.values())
|
| 389 |
|
| 390 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
def _validate_backtest_payload(
|
| 392 |
inputs: BacktesterInput,
|
| 393 |
payload: BacktestPayload,
|
|
|
|
| 144 |
])],
|
| 145 |
})
|
| 146 |
|
| 147 |
+
payload = _bind_current_signal_evidence(payload, input_refs[0], signals)
|
| 148 |
_validate_backtest_payload(inputs, payload, input_refs=input_refs)
|
| 149 |
verify_evidence(payload, sources)
|
| 150 |
output_gaps = _dedupe_backtest_gaps([*upstream_gaps, *payload.data_gaps])
|
|
|
|
| 389 |
return list(unique.values())
|
| 390 |
|
| 391 |
|
| 392 |
+
def _bind_current_signal_evidence(
|
| 393 |
+
payload: BacktestPayload,
|
| 394 |
+
signals_ref: JsonArtifactRef,
|
| 395 |
+
signals: list[Any],
|
| 396 |
+
) -> BacktestPayload:
|
| 397 |
+
indices = {signal.signal_id: index for index, signal in enumerate(signals)}
|
| 398 |
+
results = []
|
| 399 |
+
for result in payload.results:
|
| 400 |
+
values = result.model_dump(mode="json")
|
| 401 |
+
index = indices.get(result.signal_id)
|
| 402 |
+
if index is not None:
|
| 403 |
+
evidence_refs = [
|
| 404 |
+
evidence.model_dump(mode="json")
|
| 405 |
+
for evidence in result.evidence_refs
|
| 406 |
+
if evidence.artifact != signals_ref
|
| 407 |
+
]
|
| 408 |
+
evidence_refs.append({
|
| 409 |
+
"artifact": signals_ref.model_dump(mode="json"),
|
| 410 |
+
"field_path": f"/payload/signals/{index}",
|
| 411 |
+
})
|
| 412 |
+
values["evidence_refs"] = evidence_refs
|
| 413 |
+
results.append(values)
|
| 414 |
+
return BacktestPayload.model_validate({
|
| 415 |
+
**payload.model_dump(mode="json"),
|
| 416 |
+
"results": results,
|
| 417 |
+
})
|
| 418 |
+
|
| 419 |
+
|
| 420 |
def _validate_backtest_payload(
|
| 421 |
inputs: BacktesterInput,
|
| 422 |
payload: BacktestPayload,
|
src/polyhedra/agents/quant/riskanalyst.py
CHANGED
|
@@ -124,6 +124,11 @@ class RiskAnalystAgent(Agent):
|
|
| 124 |
else:
|
| 125 |
payload = _empty_risk_payload(inputs, upstream_gaps)
|
| 126 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
_validate_risk_payload(inputs, payload, input_refs=input_refs)
|
| 128 |
verify_evidence(payload, sources)
|
| 129 |
output_gaps = _dedupe_risk_gaps([*upstream_gaps, *payload.data_gaps])
|
|
@@ -408,6 +413,34 @@ def _dedupe_risk_gaps(gaps: list[DataIssue]) -> list[DataIssue]:
|
|
| 408 |
return list(unique.values())
|
| 409 |
|
| 410 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 411 |
def _portfolio_heat(inputs: RiskAnalystInput) -> float:
|
| 412 |
return round(sum(
|
| 413 |
position.portfolio_fraction
|
|
|
|
| 124 |
else:
|
| 125 |
payload = _empty_risk_payload(inputs, upstream_gaps)
|
| 126 |
|
| 127 |
+
payload = _bind_current_backtest_evidence(
|
| 128 |
+
payload,
|
| 129 |
+
input_refs[0],
|
| 130 |
+
backtest_results,
|
| 131 |
+
)
|
| 132 |
_validate_risk_payload(inputs, payload, input_refs=input_refs)
|
| 133 |
verify_evidence(payload, sources)
|
| 134 |
output_gaps = _dedupe_risk_gaps([*upstream_gaps, *payload.data_gaps])
|
|
|
|
| 413 |
return list(unique.values())
|
| 414 |
|
| 415 |
|
| 416 |
+
def _bind_current_backtest_evidence(
|
| 417 |
+
payload: RiskPayload,
|
| 418 |
+
backtest_ref: JsonArtifactRef,
|
| 419 |
+
results: list[Any],
|
| 420 |
+
) -> RiskPayload:
|
| 421 |
+
indices = {result.signal_id: index for index, result in enumerate(results)}
|
| 422 |
+
assessments = []
|
| 423 |
+
for assessment in payload.signal_assessments:
|
| 424 |
+
values = assessment.model_dump(mode="json")
|
| 425 |
+
index = indices.get(assessment.signal_id)
|
| 426 |
+
if index is not None:
|
| 427 |
+
evidence_refs = [
|
| 428 |
+
evidence.model_dump(mode="json")
|
| 429 |
+
for evidence in assessment.evidence_refs
|
| 430 |
+
if evidence.artifact != backtest_ref
|
| 431 |
+
]
|
| 432 |
+
evidence_refs.append({
|
| 433 |
+
"artifact": backtest_ref.model_dump(mode="json"),
|
| 434 |
+
"field_path": f"/payload/results/{index}",
|
| 435 |
+
})
|
| 436 |
+
values["evidence_refs"] = evidence_refs
|
| 437 |
+
assessments.append(values)
|
| 438 |
+
return RiskPayload.model_validate({
|
| 439 |
+
**payload.model_dump(mode="json"),
|
| 440 |
+
"signal_assessments": assessments,
|
| 441 |
+
})
|
| 442 |
+
|
| 443 |
+
|
| 444 |
def _portfolio_heat(inputs: RiskAnalystInput) -> float:
|
| 445 |
return round(sum(
|
| 446 |
position.portfolio_fraction
|
tests/test_backtester_typed.py
CHANGED
|
@@ -325,17 +325,22 @@ async def test_typed_backtester_rejects_simulation_claim_from_llm(tmp_path: Path
|
|
| 325 |
|
| 326 |
|
| 327 |
@pytest.mark.asyncio
|
| 328 |
-
async def
|
| 329 |
-
|
| 330 |
payload = assessment_payload(signals_ref, field_path="/payload/synthesis")
|
| 331 |
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
|
| 340 |
|
| 341 |
@pytest.mark.asyncio
|
|
|
|
| 325 |
|
| 326 |
|
| 327 |
@pytest.mark.asyncio
|
| 328 |
+
async def test_typed_backtester_binds_exact_signal_citation(tmp_path: Path):
|
| 329 |
+
session, publisher, inputs, files, signals_ref = setup_backtester(tmp_path)
|
| 330 |
payload = assessment_payload(signals_ref, field_path="/payload/synthesis")
|
| 331 |
|
| 332 |
+
await BacktesterAgent(ControlledLLM(payload)).execute_typed(
|
| 333 |
+
inputs,
|
| 334 |
+
files=files,
|
| 335 |
+
publisher=publisher,
|
| 336 |
+
now=lambda: AT,
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
output = BacktesterOutput.model_validate_json(
|
| 340 |
+
(session / "quant/runs/run1/backtest_results.json").read_text(),
|
| 341 |
+
strict=True,
|
| 342 |
+
)
|
| 343 |
+
assert output.payload.results[0].evidence_refs[0].field_path == "/payload/signals/0"
|
| 344 |
|
| 345 |
|
| 346 |
@pytest.mark.asyncio
|
tests/test_riskanalyst_typed.py
CHANGED
|
@@ -246,17 +246,25 @@ async def test_not_testable_signal_requires_rejection(tmp_path: Path):
|
|
| 246 |
|
| 247 |
|
| 248 |
@pytest.mark.asyncio
|
| 249 |
-
async def
|
| 250 |
-
|
| 251 |
payload = risk_payload(backtest_ref, field_path="/payload/limitations")
|
| 252 |
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
|
| 261 |
|
| 262 |
@pytest.mark.asyncio
|
|
|
|
| 246 |
|
| 247 |
|
| 248 |
@pytest.mark.asyncio
|
| 249 |
+
async def test_risk_binds_exact_backtest_result_citation(tmp_path: Path):
|
| 250 |
+
session, publisher, inputs, files, backtest_ref = setup_risk(tmp_path)
|
| 251 |
payload = risk_payload(backtest_ref, field_path="/payload/limitations")
|
| 252 |
|
| 253 |
+
await RiskAnalystAgent(ControlledLLM(payload)).execute_typed(
|
| 254 |
+
inputs,
|
| 255 |
+
files=files,
|
| 256 |
+
publisher=publisher,
|
| 257 |
+
now=lambda: AT,
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
output = RiskAnalystOutput.model_validate_json(
|
| 261 |
+
(session / "quant/runs/run1/risk_assessment.json").read_text(),
|
| 262 |
+
strict=True,
|
| 263 |
+
)
|
| 264 |
+
assert (
|
| 265 |
+
output.payload.signal_assessments[0].evidence_refs[0].field_path
|
| 266 |
+
== "/payload/results/0"
|
| 267 |
+
)
|
| 268 |
|
| 269 |
|
| 270 |
@pytest.mark.asyncio
|