yf176 commited on
Commit
1341b26
·
1 Parent(s): 8440fca

fix(evidence): bind downstream record citations

Browse files
src/polyhedra/agents/quant/backtester.py CHANGED
@@ -144,6 +144,7 @@ class BacktesterAgent(Agent):
144
  ])],
145
  })
146
 
 
147
  _validate_backtest_payload(inputs, payload, input_refs=input_refs)
148
  verify_evidence(payload, sources)
149
  output_gaps = _dedupe_backtest_gaps([*upstream_gaps, *payload.data_gaps])
@@ -388,6 +389,34 @@ def _dedupe_backtest_gaps(gaps: list[DataIssue]) -> list[DataIssue]:
388
  return list(unique.values())
389
 
390
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
391
  def _validate_backtest_payload(
392
  inputs: BacktesterInput,
393
  payload: BacktestPayload,
 
144
  ])],
145
  })
146
 
147
+ payload = _bind_current_signal_evidence(payload, input_refs[0], signals)
148
  _validate_backtest_payload(inputs, payload, input_refs=input_refs)
149
  verify_evidence(payload, sources)
150
  output_gaps = _dedupe_backtest_gaps([*upstream_gaps, *payload.data_gaps])
 
389
  return list(unique.values())
390
 
391
 
392
+ def _bind_current_signal_evidence(
393
+ payload: BacktestPayload,
394
+ signals_ref: JsonArtifactRef,
395
+ signals: list[Any],
396
+ ) -> BacktestPayload:
397
+ indices = {signal.signal_id: index for index, signal in enumerate(signals)}
398
+ results = []
399
+ for result in payload.results:
400
+ values = result.model_dump(mode="json")
401
+ index = indices.get(result.signal_id)
402
+ if index is not None:
403
+ evidence_refs = [
404
+ evidence.model_dump(mode="json")
405
+ for evidence in result.evidence_refs
406
+ if evidence.artifact != signals_ref
407
+ ]
408
+ evidence_refs.append({
409
+ "artifact": signals_ref.model_dump(mode="json"),
410
+ "field_path": f"/payload/signals/{index}",
411
+ })
412
+ values["evidence_refs"] = evidence_refs
413
+ results.append(values)
414
+ return BacktestPayload.model_validate({
415
+ **payload.model_dump(mode="json"),
416
+ "results": results,
417
+ })
418
+
419
+
420
  def _validate_backtest_payload(
421
  inputs: BacktesterInput,
422
  payload: BacktestPayload,
src/polyhedra/agents/quant/riskanalyst.py CHANGED
@@ -124,6 +124,11 @@ class RiskAnalystAgent(Agent):
124
  else:
125
  payload = _empty_risk_payload(inputs, upstream_gaps)
126
 
 
 
 
 
 
127
  _validate_risk_payload(inputs, payload, input_refs=input_refs)
128
  verify_evidence(payload, sources)
129
  output_gaps = _dedupe_risk_gaps([*upstream_gaps, *payload.data_gaps])
@@ -408,6 +413,34 @@ def _dedupe_risk_gaps(gaps: list[DataIssue]) -> list[DataIssue]:
408
  return list(unique.values())
409
 
410
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
411
  def _portfolio_heat(inputs: RiskAnalystInput) -> float:
412
  return round(sum(
413
  position.portfolio_fraction
 
124
  else:
125
  payload = _empty_risk_payload(inputs, upstream_gaps)
126
 
127
+ payload = _bind_current_backtest_evidence(
128
+ payload,
129
+ input_refs[0],
130
+ backtest_results,
131
+ )
132
  _validate_risk_payload(inputs, payload, input_refs=input_refs)
133
  verify_evidence(payload, sources)
134
  output_gaps = _dedupe_risk_gaps([*upstream_gaps, *payload.data_gaps])
 
413
  return list(unique.values())
414
 
415
 
416
+ def _bind_current_backtest_evidence(
417
+ payload: RiskPayload,
418
+ backtest_ref: JsonArtifactRef,
419
+ results: list[Any],
420
+ ) -> RiskPayload:
421
+ indices = {result.signal_id: index for index, result in enumerate(results)}
422
+ assessments = []
423
+ for assessment in payload.signal_assessments:
424
+ values = assessment.model_dump(mode="json")
425
+ index = indices.get(assessment.signal_id)
426
+ if index is not None:
427
+ evidence_refs = [
428
+ evidence.model_dump(mode="json")
429
+ for evidence in assessment.evidence_refs
430
+ if evidence.artifact != backtest_ref
431
+ ]
432
+ evidence_refs.append({
433
+ "artifact": backtest_ref.model_dump(mode="json"),
434
+ "field_path": f"/payload/results/{index}",
435
+ })
436
+ values["evidence_refs"] = evidence_refs
437
+ assessments.append(values)
438
+ return RiskPayload.model_validate({
439
+ **payload.model_dump(mode="json"),
440
+ "signal_assessments": assessments,
441
+ })
442
+
443
+
444
  def _portfolio_heat(inputs: RiskAnalystInput) -> float:
445
  return round(sum(
446
  position.portfolio_fraction
tests/test_backtester_typed.py CHANGED
@@ -325,17 +325,22 @@ async def test_typed_backtester_rejects_simulation_claim_from_llm(tmp_path: Path
325
 
326
 
327
  @pytest.mark.asyncio
328
- async def test_typed_backtester_requires_exact_signal_citation(tmp_path: Path):
329
- _, publisher, inputs, files, signals_ref = setup_backtester(tmp_path)
330
  payload = assessment_payload(signals_ref, field_path="/payload/synthesis")
331
 
332
- with pytest.raises(ValueError, match="exact current signal"):
333
- await BacktesterAgent(ControlledLLM(payload)).execute_typed(
334
- inputs,
335
- files=files,
336
- publisher=publisher,
337
- now=lambda: AT,
338
- )
 
 
 
 
 
339
 
340
 
341
  @pytest.mark.asyncio
 
325
 
326
 
327
  @pytest.mark.asyncio
328
+ async def test_typed_backtester_binds_exact_signal_citation(tmp_path: Path):
329
+ session, publisher, inputs, files, signals_ref = setup_backtester(tmp_path)
330
  payload = assessment_payload(signals_ref, field_path="/payload/synthesis")
331
 
332
+ await BacktesterAgent(ControlledLLM(payload)).execute_typed(
333
+ inputs,
334
+ files=files,
335
+ publisher=publisher,
336
+ now=lambda: AT,
337
+ )
338
+
339
+ output = BacktesterOutput.model_validate_json(
340
+ (session / "quant/runs/run1/backtest_results.json").read_text(),
341
+ strict=True,
342
+ )
343
+ assert output.payload.results[0].evidence_refs[0].field_path == "/payload/signals/0"
344
 
345
 
346
  @pytest.mark.asyncio
tests/test_riskanalyst_typed.py CHANGED
@@ -246,17 +246,25 @@ async def test_not_testable_signal_requires_rejection(tmp_path: Path):
246
 
247
 
248
  @pytest.mark.asyncio
249
- async def test_risk_requires_exact_backtest_result_citation(tmp_path: Path):
250
- _, publisher, inputs, files, backtest_ref = setup_risk(tmp_path)
251
  payload = risk_payload(backtest_ref, field_path="/payload/limitations")
252
 
253
- with pytest.raises(ValueError, match="exact current backtest"):
254
- await RiskAnalystAgent(ControlledLLM(payload)).execute_typed(
255
- inputs,
256
- files=files,
257
- publisher=publisher,
258
- now=lambda: AT,
259
- )
 
 
 
 
 
 
 
 
260
 
261
 
262
  @pytest.mark.asyncio
 
246
 
247
 
248
  @pytest.mark.asyncio
249
+ async def test_risk_binds_exact_backtest_result_citation(tmp_path: Path):
250
+ session, publisher, inputs, files, backtest_ref = setup_risk(tmp_path)
251
  payload = risk_payload(backtest_ref, field_path="/payload/limitations")
252
 
253
+ await RiskAnalystAgent(ControlledLLM(payload)).execute_typed(
254
+ inputs,
255
+ files=files,
256
+ publisher=publisher,
257
+ now=lambda: AT,
258
+ )
259
+
260
+ output = RiskAnalystOutput.model_validate_json(
261
+ (session / "quant/runs/run1/risk_assessment.json").read_text(),
262
+ strict=True,
263
+ )
264
+ assert (
265
+ output.payload.signal_assessments[0].evidence_refs[0].field_path
266
+ == "/payload/results/0"
267
+ )
268
 
269
 
270
  @pytest.mark.asyncio