File size: 2,394 Bytes
78ac017 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | from experiments.materialize_third_view_disagreement_agent import materialize
IDENTITY = {
"benchmark": "demo",
"model_manifest_sha256": "m",
"annotation_sha256": "a",
"evaluation_protocol_sha256": "p",
"annotation": "annotation.json",
"evaluation_protocol": {},
"model": "model",
"processor": "processor",
}
def _result(rows: list[dict]) -> dict:
return {**IDENTITY, "rows": rows}
def test_third_view_commits_only_on_acquired_disagreement() -> None:
recent = _result(
[
{"uid": "u0", "prediction": "A", "visual_tokens": 10},
{"uid": "u1", "prediction": "D", "visual_tokens": 10},
]
)
event = _result(
[
{
"uid": "u0",
"prediction": "B",
"error": None,
"visual_tokens": 20,
"sampling": {"frame_indices": list(range(20))},
}
]
)
base = _result(
[
{
"uid": "u0",
"prediction": "A",
"agent_policy": {"escalated": True},
"media_calls": 2,
"num_turns": 2,
"model_processed_frames": 24,
"sampling": {"frame_indices": list(range(20))},
},
{"uid": "u1", "prediction": "D", "media_calls": 1},
]
)
third = _result(
[
{
"uid": "u0",
"prediction": "C",
"raw_response": "C",
"error": None,
"query_time": 31.0,
"frames_observed": 32,
"visual_tokens": 30,
"sampling": {
"frame_indices": list(range(32)),
"frame_timestamps": list(map(float, range(32))),
},
}
]
)
result = materialize(recent, event, base, third)
rows = {row["uid"]: row for row in result["rows"]}
assert rows["u0"]["prediction"] == "C"
assert rows["u0"]["media_calls"] == 3
assert rows["u0"]["model_processed_frames"] == 56
assert rows["u0"]["unique_frames"] == 32
assert rows["u0"]["visual_tokens"] == 60
assert rows["u1"]["prediction"] == "D"
assert rows["u1"]["third_view_policy"]["triggered"] is False
assert result["policy"]["question_task_group_or_benchmark_router"] is False
|