File size: 3,626 Bytes
1d4d315 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | from ollie_assistants.analytics.pricing import PricingMethod
from ollie_assistants.observability.recorder import SqliteTraceRecorder
from ollie_assistants.observability.types import ConversationTrace, TraceEvent, TraceEventType
def test_sqlite_trace_recorder_records_lists_and_fetches_run(tmp_path) -> None:
recorder = SqliteTraceRecorder(tmp_path / "runs.sqlite")
trace = ConversationTrace(
run_id="run-1",
session_id="session-1",
assistant_id="oss",
provider_id="openai_compatible",
total_latency_ms=123.4,
input_tokens=10,
output_tokens=5,
estimated_cost_usd=0.001,
pricing_method=PricingMethod.TOKEN_PRICING,
events=(
TraceEvent(
event_type=TraceEventType.SAFETY,
name="input_check",
latency_ms=1.0,
metadata={"action": "allow"},
),
TraceEvent(
event_type=TraceEventType.TOOL,
name="search_memory",
latency_ms=2.0,
metadata={"content": "memory hit"},
),
),
)
recorder.record(trace)
runs = recorder.list_runs()
detail = recorder.get_run("run-1")
assert runs[0]["run_id"] == "run-1"
assert runs[0]["tool_calls"] == "search_memory"
assert detail is not None
assert detail["trace"]["run_id"] == "run-1"
assert recorder.list_runs(query="memory hit")[0]["run_id"] == "run-1"
def test_sqlite_trace_recorder_records_eval_runs(tmp_path) -> None:
recorder = SqliteTraceRecorder(tmp_path / "runs.sqlite")
summary = {
"case_count": 3,
"required_metrics": {
"hallucination_rate": {"oss_failure_rate": 0.0, "frontier_failure_rate": 0.0},
"bias_harmful_output_rate": {
"oss_failure_rate": 0.5,
"frontier_failure_rate": 0.0,
},
"content_safety_failure_rate": {
"oss_failure_rate": 0.25,
"frontier_failure_rate": 0.0,
},
},
"assistants": {
"oss": {
"average_latency_ms": 100.0,
"estimated_cost_usd": 0.01,
"input_tokens": 10,
"output_tokens": 5,
},
"frontier": {
"average_latency_ms": 200.0,
"estimated_cost_usd": 0.02,
"input_tokens": 11,
"output_tokens": 6,
},
},
"benchmark_rows": [{"provider": "OSS Modal"}],
}
recorder.record_eval_run(
eval_run_id="eval-1",
created_at="2026-05-26T00:00:00+00:00",
include_benchmark=True,
use_llm_judge=False,
report_markdown="# Report",
summary=summary,
)
runs = recorder.list_eval_runs()
detail = recorder.get_eval_run("eval-1")
assert runs[0]["eval_run_id"] == "eval-1"
assert runs[0]["status"] == "completed"
assert runs[0]["benchmark_count"] == 1
assert detail is not None
assert detail["summary"]["case_count"] == 3
assert detail["report_markdown"] == "# Report"
def test_sqlite_trace_recorder_tracks_running_eval_run(tmp_path) -> None:
recorder = SqliteTraceRecorder(tmp_path / "runs.sqlite")
recorder.start_eval_run(
eval_run_id="eval-running",
created_at="2026-05-26T00:00:00+00:00",
include_benchmark=True,
use_llm_judge=True,
)
run = recorder.get_eval_run("eval-running")
assert run is not None
assert run["status"] == "running"
assert run["case_count"] == 0
|