Buckets:

download
raw
19.1 kB
# tests/test_replay.py
"""Tests for POST /replay endpoint and ReplayEngine."""
from __future__ import annotations
import time
import pytest
from fastapi.testclient import TestClient
from veronica.api.app import create_app
from veronica.ingest.event_ingestor import CPStepOutcomeStore
from veronica.replay.engine import ReplayEngine, _build_override_policy
from veronica.replay.models import DecisionDiff, ReplayRequest, ReplayResult
from veronica.distribution.policy_distributor import PolicyDistributor
from veronica.schemas.events import StepOutcome
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_NOW = time.time()
_PAST = _NOW - 3600.0
def _make_outcome(
step_id: str,
chain_id: str = "chain-a",
decision: str = "allow",
cost_usd: float = 0.01,
tokens: int = 100,
ts: float | None = None,
) -> StepOutcome:
return StepOutcome(
step_id=step_id,
chain_id=chain_id,
operation_name="test-op",
decision=decision,
cost_usd=cost_usd,
tokens=tokens,
duration_ms=10.0,
policy_hash="abc123",
audit_id="audit-" + step_id,
timestamp=ts if ts is not None else _NOW - 100.0,
)
@pytest.fixture()
def client() -> TestClient:
app = create_app()
with TestClient(app) as c:
yield c
@pytest.fixture()
def populated_client() -> TestClient:
"""Client with pre-populated event store."""
app = create_app()
with TestClient(app) as c:
store = app.state.ingestor.store
store.put_many(
[
_make_outcome(
"step-1",
chain_id="chain-a",
decision="allow",
cost_usd=0.01,
ts=_NOW - 200.0,
),
_make_outcome(
"step-2",
chain_id="chain-a",
decision="allow",
cost_usd=0.01,
ts=_NOW - 100.0,
),
_make_outcome(
"step-3",
chain_id="chain-a",
decision="halt",
cost_usd=0.05,
ts=_NOW - 50.0,
),
_make_outcome(
"step-x",
chain_id="chain-b",
decision="allow",
cost_usd=0.01,
ts=_NOW - 100.0,
),
]
)
yield c
# ---------------------------------------------------------------------------
# Unit: ReplayEngine
# ---------------------------------------------------------------------------
class TestReplayEngineNoOverride:
def _engine(self, outcomes: list[StepOutcome]) -> ReplayEngine:
store = CPStepOutcomeStore()
store.put_many(outcomes)
return ReplayEngine(store=store, distributor=PolicyDistributor())
def test_no_override_decisions_unchanged(self) -> None:
outcomes = [
_make_outcome("s1", decision="allow"),
_make_outcome("s2", decision="halt"),
]
engine = self._engine(outcomes)
req = ReplayRequest(
chain_id="chain-a", from_timestamp=_NOW - 300.0, to_timestamp=_NOW
)
result = engine.replay(req)
assert result.event_count == 2
assert result.changed_count == 0
for diff in result.diffs:
assert diff.changed is False
assert diff.original_decision == diff.replayed_decision
def test_empty_range_returns_empty_result(self) -> None:
outcomes = [_make_outcome("s1", ts=_NOW - 1000.0)]
engine = self._engine(outcomes)
req = ReplayRequest(
chain_id="chain-a", from_timestamp=_NOW - 10.0, to_timestamp=_NOW
)
result = engine.replay(req)
assert result.event_count == 0
assert result.diffs == []
assert result.changed_count == 0
def test_chain_not_found_returns_empty(self) -> None:
outcomes = [_make_outcome("s1", chain_id="other")]
engine = self._engine(outcomes)
req = ReplayRequest(
chain_id="chain-a", from_timestamp=_NOW - 300.0, to_timestamp=_NOW
)
result = engine.replay(req)
assert result.event_count == 0
assert "No events found" in result.summary
def test_store_not_mutated_after_replay(self) -> None:
outcomes = [_make_outcome("s1")]
store = CPStepOutcomeStore()
store.put_many(outcomes)
engine = ReplayEngine(store=store, distributor=PolicyDistributor())
req = ReplayRequest(
chain_id="chain-a", from_timestamp=_NOW - 300.0, to_timestamp=_NOW
)
engine.replay(req)
# Store must still have exactly the original record
assert len(store.snapshot()) == 1
def test_result_fields_present(self) -> None:
outcomes = [_make_outcome("s1")]
engine = self._engine(outcomes)
req = ReplayRequest(
chain_id="chain-a", from_timestamp=_NOW - 300.0, to_timestamp=_NOW
)
result = engine.replay(req)
assert isinstance(result, ReplayResult)
assert result.chain_id == "chain-a"
assert result.event_count == 1
assert len(result.diffs) == 1
assert isinstance(result.diffs[0], DecisionDiff)
class TestReplayEngineWithOverride:
def _engine(self, outcomes: list[StepOutcome]) -> ReplayEngine:
store = CPStepOutcomeStore()
store.put_many(outcomes)
return ReplayEngine(store=store, distributor=PolicyDistributor())
def _req(self, override: dict) -> ReplayRequest:
return ReplayRequest(
chain_id="chain-a",
from_timestamp=_NOW - 300.0,
to_timestamp=_NOW,
override_policy=override,
)
def test_stricter_ceiling_halts_expensive_step(self) -> None:
# Original: allow at 0.01 each. Override: ceiling_usd=0.005 -> halt
outcomes = [_make_outcome("s1", decision="allow", cost_usd=0.01)]
engine = self._engine(outcomes)
req = self._req(
{"chain_id": "chain-a", "ceiling_usd": 0.005, "on_exceed": "halt"}
)
result = engine.replay(req)
assert result.changed_count == 1
assert result.diffs[0].replayed_decision == "halt"
assert result.diffs[0].original_decision == "allow"
assert result.diffs[0].changed is True
def test_looser_ceiling_allows_originally_halted(self) -> None:
# Original: halt. Override: ceiling_usd=100 -> allow
outcomes = [_make_outcome("s1", decision="halt", cost_usd=0.01)]
engine = self._engine(outcomes)
req = self._req(
{"chain_id": "chain-a", "ceiling_usd": 100.0, "on_exceed": "halt"}
)
result = engine.replay(req)
assert result.changed_count == 1
assert result.diffs[0].replayed_decision == "allow"
assert result.diffs[0].original_decision == "halt"
def test_cumulative_cost_triggers_halt_mid_chain(self) -> None:
# Two steps each 0.01. Override ceiling=0.015 -> step-2 halts
outcomes = [
_make_outcome("s1", decision="allow", cost_usd=0.01, ts=_NOW - 200.0),
_make_outcome("s2", decision="allow", cost_usd=0.01, ts=_NOW - 100.0),
]
engine = self._engine(outcomes)
req = self._req(
{"chain_id": "chain-a", "ceiling_usd": 0.015, "on_exceed": "halt"}
)
result = engine.replay(req)
assert result.diffs[0].replayed_decision == "allow"
assert result.diffs[1].replayed_decision == "halt"
assert result.changed_count == 1
def test_step_ceiling_enforced(self) -> None:
# Three steps, override ceiling_steps=2
outcomes = [
_make_outcome(
f"s{i}", decision="allow", cost_usd=0.001, ts=_NOW - (300 - i * 10)
)
for i in range(3)
]
engine = self._engine(outcomes)
req = self._req(
{
"chain_id": "chain-a",
"ceiling_usd": 100.0,
"ceiling_steps": 2,
"on_exceed": "halt",
}
)
result = engine.replay(req)
assert result.diffs[0].replayed_decision == "allow"
assert result.diffs[1].replayed_decision == "allow"
assert result.diffs[2].replayed_decision == "halt"
def test_on_exceed_degrade_propagates(self) -> None:
outcomes = [_make_outcome("s1", decision="allow", cost_usd=0.01)]
engine = self._engine(outcomes)
req = self._req(
{"chain_id": "chain-a", "ceiling_usd": 0.005, "on_exceed": "degrade"}
)
result = engine.replay(req)
assert result.diffs[0].replayed_decision == "degrade"
def test_summary_mentions_chain_id(self) -> None:
outcomes = [_make_outcome("s1")]
engine = self._engine(outcomes)
req = self._req(
{"chain_id": "chain-a", "ceiling_usd": 100.0, "on_exceed": "halt"}
)
result = engine.replay(req)
assert "chain-a" in result.summary
def test_invalid_override_policy_raises_value_error(self) -> None:
outcomes = [_make_outcome("s1")]
engine = self._engine(outcomes)
req = ReplayRequest(
chain_id="chain-a",
from_timestamp=_NOW - 300.0,
to_timestamp=_NOW,
override_policy={"ceiling_usd": 1.0}, # missing chain_id
)
with pytest.raises(ValueError, match="Invalid override_policy"):
engine.replay(req)
def test_override_missing_ceiling_usd_raises(self) -> None:
with pytest.raises(ValueError, match="ceiling_usd"):
_build_override_policy({"chain_id": "x", "on_exceed": "halt"})
# ---------------------------------------------------------------------------
# API: POST /replay
# ---------------------------------------------------------------------------
class TestReplayAPIBasic:
def test_empty_store_returns_200_with_empty_result(
self, client: TestClient
) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "chain-x",
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
},
)
assert resp.status_code == 200
data = resp.json()
assert data["event_count"] == 0
assert data["diffs"] == []
assert data["changed_count"] == 0
def test_response_shape(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "chain-x",
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
},
)
data = resp.json()
required = {
"chain_id",
"event_count",
"diffs",
"changed_count",
"summary",
"store_unchanged",
}
assert required.issubset(data.keys())
def test_store_unchanged_is_true(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "chain-x",
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
},
)
assert resp.json()["store_unchanged"] is True
def test_populated_chain_returns_events(self, populated_client: TestClient) -> None:
resp = populated_client.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
},
)
assert resp.status_code == 200
assert resp.json()["event_count"] == 3
def test_chain_isolation(self, populated_client: TestClient) -> None:
resp = populated_client.post(
"/replay",
json={
"chain_id": "chain-b",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
},
)
assert resp.json()["event_count"] == 1
def test_time_range_filtering(self, populated_client: TestClient) -> None:
# Only step-3 (ts=_NOW-50) is within this range
resp = populated_client.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 80.0,
"to_timestamp": _NOW,
},
)
assert resp.json()["event_count"] == 1
def test_no_override_all_unchanged(self, populated_client: TestClient) -> None:
resp = populated_client.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
},
)
data = resp.json()
assert data["changed_count"] == 0
for d in data["diffs"]:
assert d["changed"] is False
def test_override_stricter_changes_decisions(
self, populated_client: TestClient
) -> None:
# ceiling_usd=0.005 -> first step (0.01) already exceeds
resp = populated_client.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
"override_policy": {
"chain_id": "chain-a",
"ceiling_usd": 0.005,
"on_exceed": "halt",
},
},
)
data = resp.json()
assert data["changed_count"] > 0
def test_override_looser_ceiling_no_changes_if_all_allowed(
self, populated_client: TestClient
) -> None:
# Very large ceiling: all steps already within budget -> no changes for "allow" steps
resp = populated_client.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
"override_policy": {
"chain_id": "chain-a",
"ceiling_usd": 9999.0,
"on_exceed": "halt",
},
},
)
data = resp.json()
# Original "halt" step would now be "allow" -> changed
assert data["changed_count"] >= 1
class TestReplayAPIValidation:
def test_missing_chain_id_returns_422(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
},
)
assert resp.status_code == 422
def test_from_equals_to_returns_422(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "c",
"from_timestamp": _NOW,
"to_timestamp": _NOW,
},
)
assert resp.status_code == 422
def test_from_after_to_returns_422(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "c",
"from_timestamp": _NOW,
"to_timestamp": _NOW - 1.0,
},
)
assert resp.status_code == 422
def test_negative_from_timestamp_returns_422(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "c",
"from_timestamp": -1.0,
"to_timestamp": _NOW,
},
)
assert resp.status_code == 422
def test_invalid_override_missing_ceiling_usd_returns_422(
self, client: TestClient
) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "c",
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
"override_policy": {"chain_id": "c", "on_exceed": "halt"},
},
)
assert resp.status_code == 422
def test_invalid_override_bad_on_exceed_returns_422(
self, client: TestClient
) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "c",
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
"override_policy": {
"chain_id": "c",
"ceiling_usd": 1.0,
"on_exceed": "explode",
},
},
)
assert resp.status_code == 422
def test_empty_chain_id_returns_422(self, client: TestClient) -> None:
resp = client.post(
"/replay",
json={
"chain_id": "",
"from_timestamp": _NOW - 100.0,
"to_timestamp": _NOW,
},
)
assert resp.status_code == 422
class TestReplayAPISideEffects:
def test_store_not_modified_after_replay(self) -> None:
app = create_app()
with TestClient(app) as c:
store = app.state.ingestor.store
store.put_many([_make_outcome("s1")])
before = len(store.snapshot())
c.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
},
)
after = len(store.snapshot())
assert before == after
def test_multiple_replays_independent(self) -> None:
app = create_app()
with TestClient(app) as c:
store = app.state.ingestor.store
store.put_many([_make_outcome("s1", cost_usd=0.10)])
r1 = c.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
"override_policy": {
"chain_id": "chain-a",
"ceiling_usd": 0.05,
"on_exceed": "halt",
},
},
)
r2 = c.post(
"/replay",
json={
"chain_id": "chain-a",
"from_timestamp": _NOW - 300.0,
"to_timestamp": _NOW,
"override_policy": {
"chain_id": "chain-a",
"ceiling_usd": 100.0,
"on_exceed": "halt",
},
},
)
assert r1.json()["diffs"][0]["replayed_decision"] == "halt"
assert r2.json()["diffs"][0]["replayed_decision"] == "allow"

Xet Storage Details

Size:
19.1 kB
·
Xet hash:
e7e89f038458c3d9f3bbdfdb284120f617babe18ca9e29058535b1a4824907c4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.