"""Unit tests for the pure helpers in the local GAIA solver harness (text.py).
Ticket 01 (`harden-local-solver`) requires unit coverage for:
- Answer normalization (`_normalize_answer_for_submission`)
- Prompt building (`_build_cleaning_prompt`)
plus the hardening helpers introduced for the ticket: safe archive extraction
(`_safe_extract_all`) and retry-with-backoff (`_call_with_retry`).
"""
import os
import shutil
import subprocess
import zipfile
from types import SimpleNamespace
import pandas as pd
import pytest
import requests
from PIL import Image
import text
# ─── Answer normalization ─────────────────────────────────────────────
class TestNormalizeAnswerForSubmission:
def test_none_becomes_empty_string(self):
assert text._normalize_answer_for_submission(None) == ""
def test_whitespace_is_stripped(self):
assert text._normalize_answer_for_submission(" x ") == "x"
def test_reasoning_tags_are_stripped(self):
answer = "The largest city is Los Angeles.Los Angeles"
assert text._normalize_answer_for_submission(answer) == "Los Angeles"
def test_orphaned_reasoning_tags_are_stripped(self):
assert text._normalize_answer_for_submission(" 42") == "42"
assert text._normalize_answer_for_submission("42 ") == "42"
def test_final_answer_prefix_is_removed(self):
assert text._normalize_answer_for_submission("Final Answer: 3") == "3"
def test_uppercase_final_answer_prefix_is_removed(self):
assert text._normalize_answer_for_submission("FINAL ANSWER: Paris") == "Paris"
def test_answer_prefix_is_removed(self):
assert text._normalize_answer_for_submission("Answer: 7") == "7"
def test_spanish_respuesta_final_prefix_is_removed(self):
assert text._normalize_answer_for_submission("Respuesta Final: Madrid") == "Madrid"
def test_leading_prefix_only_stripped_once(self):
assert text._normalize_answer_for_submission("Final Answer: Final Answer: 3") == "Final Answer: 3"
def test_formatting_marks_are_stripped(self):
assert text._normalize_answer_for_submission("**42**") == "42"
assert text._normalize_answer_for_submission("`3, 4, 5`") == "3, 4, 5"
def test_comma_separated_list_is_preserved(self):
assert text._normalize_answer_for_submission("3, 4, 5") == "3, 4, 5"
# ─── Prompt building ──────────────────────────────────────────────────
class TestBuildCleaningPrompt:
def test_interpolates_question_and_answer(self):
prompt = text._build_cleaning_prompt("What is 2+2?", "4")
assert "Question: What is 2+2?" in prompt
assert "Initial Answer: 4" in prompt
assert "{question}" not in prompt
assert "{answer}" not in prompt
def test_braces_in_answer_do_not_crash(self):
prompt = text._build_cleaning_prompt("What is the set?", "{1, 2, 3}")
assert "Initial Answer: {1, 2, 3}" in prompt
def test_braces_in_question_do_not_crash(self):
prompt = text._build_cleaning_prompt("Solve f(x) = {x} for x", "5")
assert "Solve f(x) = {x} for x" in prompt
assert "Initial Answer: 5" in prompt
def test_both_braces_and_answer_placeholder_safe(self):
prompt = text._build_cleaning_prompt("f({x})", "{answer}")
assert "Initial Answer: {answer}" in prompt
def test_none_values_do_not_crash(self):
prompt = text._build_cleaning_prompt(None, None)
assert "Question:" in prompt
assert "Initial Answer:" in prompt
# ─── Safe archive extraction ──────────────────────────────────────────
class TestSafeExtractAll:
def test_extracts_normal_members(self, tmp_path):
zip_path = tmp_path / "ok.zip"
with zipfile.ZipFile(zip_path, "w") as zf:
zf.writestr("hello.txt", "hello")
zf.writestr("sub/nested.txt", "nested")
out = tmp_path / "out"
with zipfile.ZipFile(zip_path) as zf:
extracted, skipped = text._safe_extract_all(zf, str(out))
assert extracted == ["hello.txt", "sub/nested.txt"]
assert skipped == []
assert (out / "hello.txt").read_text() == "hello"
assert (out / "sub" / "nested.txt").read_text() == "nested"
def test_rejects_parent_directory_traversal(self, tmp_path):
zip_path = tmp_path / "evil.zip"
with zipfile.ZipFile(zip_path, "w") as zf:
zf.writestr("../evil.txt", "evil")
out = tmp_path / "out"
with zipfile.ZipFile(zip_path) as zf:
extracted, skipped = text._safe_extract_all(zf, str(out))
assert extracted == []
assert skipped == ["../evil.txt"]
assert not (tmp_path / "evil.txt").exists()
def test_rejects_absolute_path_member(self, tmp_path):
zip_path = tmp_path / "abs.zip"
with zipfile.ZipFile(zip_path, "w") as zf:
zf.writestr("/tmp/abs_evil.txt", "bad")
out = tmp_path / "out"
with zipfile.ZipFile(zip_path) as zf:
extracted, skipped = text._safe_extract_all(zf, str(out))
assert extracted == []
assert "/tmp/abs_evil.txt" in skipped
assert not (out / "tmp" / "abs_evil.txt").exists()
def test_deep_traversal_never_escapes_extract_dir(self, tmp_path):
zip_path = tmp_path / "deep.zip"
with zipfile.ZipFile(zip_path, "w") as zf:
zf.writestr("a/b/../../outside.txt", "nope")
out = tmp_path / "out"
with zipfile.ZipFile(zip_path) as zf:
extracted, skipped = text._safe_extract_all(zf, str(out))
assert extracted == []
assert "a/b/../../outside.txt" in skipped
assert not (tmp_path / "outside.txt").exists()
# ─── Retry with backoff ───────────────────────────────────────────────
class TestCallWithRetry:
def test_returns_result_when_no_error(self):
calls = {"n": 0}
def fn():
calls["n"] += 1
return "ok"
assert text._call_with_retry(fn, max_attempts=3, base_delay=0.0) == "ok"
assert calls["n"] == 1
def test_retries_transient_error_then_succeeds(self):
calls = {"n": 0}
def fn():
calls["n"] += 1
if calls["n"] < 3:
raise RuntimeError("429 rate limit")
return "recovered"
assert text._call_with_retry(fn, max_attempts=5, base_delay=0.0) == "recovered"
assert calls["n"] == 3
def test_gives_up_after_max_attempts(self):
calls = {"n": 0}
def fn():
calls["n"] += 1
raise RuntimeError("connection error")
with pytest.raises(RuntimeError):
text._call_with_retry(fn, max_attempts=3, base_delay=0.0)
assert calls["n"] == 3
def test_non_transient_error_is_not_retried(self):
calls = {"n": 0}
def fn():
calls["n"] += 1
raise ValueError("this is not a transient failure")
with pytest.raises(ValueError):
text._call_with_retry(fn, max_attempts=3, base_delay=0.0)
assert calls["n"] == 1
def test_retryable_exception_subtypes_are_retried(self):
# Simulates an OpenAI-style transient error class.
class FakeRateLimitError(Exception):
pass
calls = {"n": 0}
def fn():
calls["n"] += 1
if calls["n"] < 2:
raise FakeRateLimitError("rate limit exceeded")
return "ok"
assert text._call_with_retry(fn, max_attempts=3, base_delay=0.0) == "ok"
assert calls["n"] == 2
def test_permanent_http_client_error_is_not_retried(self):
# 404 from raise_for_status() must not burn retry attempts.
response = requests.Response()
response.status_code = 404
calls = {"n": 0}
def fn():
calls["n"] += 1
raise requests.exceptions.HTTPError("404 Client Error", response=response)
with pytest.raises(requests.exceptions.HTTPError):
text._call_with_retry(fn, max_attempts=4, base_delay=0.0)
assert calls["n"] == 1
def test_transient_http_server_error_is_retried(self):
response = requests.Response()
response.status_code = 503
calls = {"n": 0}
def fn():
calls["n"] += 1
if calls["n"] < 2:
raise requests.exceptions.HTTPError("503 Service Unavailable", response=response)
return "ok"
assert text._call_with_retry(fn, max_attempts=4, base_delay=0.0) == "ok"
assert calls["n"] == 2
# ─── Worker per-Task timeout ──────────────────────────────────────────
class TestRunWithTimeout:
def test_returns_value_when_fn_completes(self):
assert text._run_with_timeout(lambda: 42, timeout=5.0) == 42
def test_raises_timeout_when_fn_is_stuck(self):
import time
t0 = time.time()
with pytest.raises(TimeoutError):
text._run_with_timeout(lambda: time.sleep(30), timeout=0.1)
# Returns promptly rather than waiting out the stuck call
assert time.time() - t0 < 5.0
def test_propagates_exception_from_fn(self):
def boom():
raise ValueError("boom")
with pytest.raises(ValueError, match="boom"):
text._run_with_timeout(boom, timeout=5.0)
# ─── Forced-answer synthesis prompt (Part B) ───────────────────────────
class TestBuildForcedAnswerPrompt:
def test_interpolates_question_and_trace(self):
prompt = text._build_forced_answer_prompt("Q?", "Trace: step 1")
assert "Q?" in prompt
assert "Trace: step 1" in prompt
def test_braces_in_trace_do_not_crash(self):
# A partial trace may contain code/JSON with braces; replacement-based
# interpolation must not raise or corrupt the sentinels.
prompt = text._build_forced_answer_prompt("Q?", '{"code": {1, 2}}')
assert '{"code": {1, 2}}' in prompt
def test_none_values_become_empty(self):
prompt = text._build_forced_answer_prompt(None, None)
assert "PARTIAL REASONING TRACE" in prompt
class TestFormatPartialTrace:
def test_formats_thought_tool_and_observation(self):
worklog = text._serialize_worklog([
_FakeActionStep(
step_number=1,
thought="read the pdf",
tool_calls=[_FakeToolCall("inspect_pdf", {"path": "a.pdf"})],
observations="page 1 contents",
),
], status="forced")
trace = text._format_partial_trace(worklog)
assert "Thought: read the pdf" in trace
assert "Tool: inspect_pdf" in trace
assert "Observation: page 1 contents" in trace
def test_empty_worklog_returns_empty(self):
worklog = text._serialize_worklog([], status="forced")
assert text._format_partial_trace(worklog) == ""
# ─── Forced-answer budget (Part B) ─────────────────────────────────────
class TestForcedAnswerBudget:
def test_default_uses_force_timeout(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 300)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 360)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 45)
assert text._forced_answer_budget() == 45.0
def test_clamped_to_remaining_budget(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 300)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 310)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 45)
assert text._forced_answer_budget() == 10.0
def test_zero_when_soft_equals_hard(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 300)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 300)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 45)
assert text._forced_answer_budget() == 0.0
def test_zero_when_hard_below_soft(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 360)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 300)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 45)
assert text._forced_answer_budget() == 0.0
# ─── Partial Worklog snapshot (Part B) ─────────────────────────────────
class TestSnapshotPartialWorklog:
def test_snapshots_agent_steps(self):
steps = [_FakeActionStep(step_number=1, thought="partial", duration=1.0)]
agent = SimpleNamespace(agent=SimpleNamespace(
memory=SimpleNamespace(steps=steps)))
worklog = text._snapshot_partial_worklog(agent)
assert worklog["status"] == "forced"
assert len(worklog["steps"]) == 1
assert worklog["steps"][0]["thought"] == "partial"
def test_falls_back_to_empty_on_serialization_failure(self):
class _BadMemory:
@property
def steps(self):
raise RuntimeError("mid-mutation")
agent = SimpleNamespace(agent=SimpleNamespace(memory=_BadMemory()))
worklog = text._snapshot_partial_worklog(agent)
assert worklog["status"] == "forced"
assert worklog["steps"] == []
assert "error" in worklog
# ─── Two-stage soft/hard timeout worker (Part B) ───────────────────────
class _FastSolver:
"""solve() returns promptly with a completed Worklog."""
def __init__(self):
self.agent = SimpleNamespace(memory=SimpleNamespace(steps=[]))
def solve(self, task_id, question, file_name, file_path=None):
return "42", {"status": "completed", "steps": []}
class _BoomSolver:
"""solve() raises — a throwing Task must not lose the worker."""
def __init__(self):
self.agent = SimpleNamespace(memory=SimpleNamespace(steps=[]))
def solve(self, task_id, question, file_name, file_path=None):
raise RuntimeError("boom")
class _SlowSolver:
"""solve() sleeps past the soft deadline, wedging like a hung call."""
def __init__(self):
self.agent = SimpleNamespace(memory=SimpleNamespace(steps=[]))
def solve(self, task_id, question, file_name, file_path=None):
import time
time.sleep(5)
return "late", {"status": "completed", "steps": []}
class TestRunTaskTimeout:
def _task(self):
return {"task_id": "t1", "question": "Q?", "file_name": ""}
def test_completes_normally_within_soft_timeout(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 5.0)
result, replace = text._run_task(_FastSolver(), 0, self._task())
assert result["status"] == "completed"
assert result["answer"] == "42"
assert replace is False
def test_solve_error_is_recorded_and_agent_reused(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 5.0)
result, replace = text._run_task(_BoomSolver(), 0, self._task())
assert result["status"] == "error"
assert result["answer"].startswith("Error:")
assert replace is False
def test_soft_deadline_synthesizes_forced_answer(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 0.05)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 10.0)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 1.0)
calls = {}
def fake_synth(question, trace):
calls["question"] = question
calls["trace"] = trace
return "forced-42"
monkeypatch.setattr(text, "_synthesize_forced_answer", fake_synth)
result, replace = text._run_task(_SlowSolver(), 0, self._task())
assert result["status"] == "forced"
assert result["answer"] == "forced-42"
assert replace is True # wedged Agent must be discarded
assert calls["question"] == "Q?"
assert calls["trace"] == "" # empty partial trace
def test_synthesis_failure_records_timeout(self, monkeypatch):
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 0.05)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 10.0)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 1.0)
def fake_synth(question, trace):
raise RuntimeError("synthesis failed")
monkeypatch.setattr(text, "_synthesize_forced_answer", fake_synth)
result, replace = text._run_task(_SlowSolver(), 0, self._task())
assert result["status"] == "timeout"
assert "timed out" in result["answer"]
assert replace is True
def test_hard_cap_enforced_when_no_budget(self, monkeypatch):
# soft == hard → zero synthesis budget → no synthesis call, plain timeout
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 0.05)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 0.05)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 45.0)
called = []
monkeypatch.setattr(text, "_synthesize_forced_answer",
lambda q, t: called.append(q) or "x")
result, replace = text._run_task(_SlowSolver(), 0, self._task())
assert result["status"] == "timeout"
assert called == [] # synthesis never attempted
assert replace is True
def test_synthesis_overrun_records_timeout(self, monkeypatch):
# The synthesis call itself is bounded by _forced_answer_budget(); if it
# overruns (a hung model call), the hard cap fires and a plain timeout
# is recorded (decision B2).
monkeypatch.setattr(text, "TASK_SOFT_TIMEOUT_SEC", 0.05)
monkeypatch.setattr(text, "TASK_HARD_TIMEOUT_SEC", 1.0)
monkeypatch.setattr(text, "FORCE_ANSWER_TIMEOUT_SEC", 0.1)
def slow_synth(question, trace):
import time
time.sleep(5)
return "late"
monkeypatch.setattr(text, "_synthesize_forced_answer", slow_synth)
result, replace = text._run_task(_SlowSolver(), 0, self._task())
assert result["status"] == "timeout"
assert "timed out" in result["answer"]
assert replace is True
# ─── Retrying model (ticket 01) ───────────────────────────────────────
class TestRetryingOpenAIServerModel:
def test_generate_reaches_base_generate(self, monkeypatch):
"""Regression: generate() must actually reach the base implementation.
Zero-arg ``super()`` inside the retry lambda raises
``RuntimeError: super(): no arguments`` — the ``__class__`` cell the
zero-arg form needs is created only for the method's own code object
and is NOT propagated into nested lambdas. The base ``generate`` is
stubbed so the real subclass path (generate -> _call_with_retry ->
super().generate) is exercised with no network.
"""
sentinel = {"role": "assistant", "content": "stubbed"}
def fake_base_generate(self, messages, **kwargs):
return sentinel
# Patch the BASE class method only; the subclass's own generate still
# shadows it, so model.generate() drives the real retry path.
monkeypatch.setattr(text.OpenAIServerModel, "generate", fake_base_generate)
model = text._RetryingOpenAIServerModel(
model_id="test-model",
api_base="https://api.deepseek.com/v1",
api_key="sk-test",
)
result = model.generate([{"role": "user", "content": "hi"}])
assert result is sentinel
# ─── Local-file resolution ────────────────────────────────────────────
class TestGetLocalTaskFile:
def test_exact_file_name_match(self, tmp_path, monkeypatch):
(tmp_path / "report.pdf").write_text("x")
monkeypatch.setattr(text, "LOCAL_FILES_DIR", str(tmp_path))
assert text._get_local_task_file("t1", "report.pdf") == str(tmp_path / "report.pdf")
def test_task_id_with_extension_match(self, tmp_path, monkeypatch):
(tmp_path / "abc123.mp3").write_text("x")
monkeypatch.setattr(text, "LOCAL_FILES_DIR", str(tmp_path))
assert text._get_local_task_file("abc123", "some.mp3") == str(tmp_path / "abc123.mp3")
def test_task_id_alone_match(self, tmp_path, monkeypatch):
(tmp_path / "xyz").write_text("x")
monkeypatch.setattr(text, "LOCAL_FILES_DIR", str(tmp_path))
assert text._get_local_task_file("xyz", "") == str(tmp_path / "xyz")
def test_task_id_prefix_search(self, tmp_path, monkeypatch):
(tmp_path / "abc123-extra.png").write_text("x")
monkeypatch.setattr(text, "LOCAL_FILES_DIR", str(tmp_path))
result = text._get_local_task_file("abc123", "something.png")
assert result == str(tmp_path / "abc123-extra.png")
def test_missing_directory_returns_empty(self, tmp_path, monkeypatch):
monkeypatch.setattr(text, "LOCAL_FILES_DIR", str(tmp_path / "does-not-exist"))
assert text._get_local_task_file("t1", "f.pdf") == ""
def test_no_file_returns_empty(self, tmp_path, monkeypatch):
monkeypatch.setattr(text, "LOCAL_FILES_DIR", str(tmp_path))
assert text._get_local_task_file("t1", "missing.pdf") == ""
# ─── Video frame timestamps (ticket 02) ──────────────────────────────
class TestVideoFrameTimestamps:
def test_evenly_spaced_center_timestamps(self):
# 4 frames over 10s → centers of 4 equal slices
assert text._video_frame_timestamps(10.0, 4) == [1.25, 3.75, 6.25, 8.75]
def test_single_frame_is_middle_of_video(self):
assert text._video_frame_timestamps(10.0, 1) == [5.0]
def test_num_frames_below_one_clamps_to_one(self):
assert text._video_frame_timestamps(10.0, 0) == [5.0]
def test_negative_duration_returns_zero(self):
assert text._video_frame_timestamps(-3.0, 4) == [0.0]
def test_zero_duration_returns_zero(self):
assert text._video_frame_timestamps(0.0, 4) == [0.0]
def test_all_timestamps_stay_within_duration(self):
ts = text._video_frame_timestamps(2.0, 8)
assert len(ts) == 8
assert all(0.0 <= t <= 2.0 for t in ts)
# Ascending and distinct
assert ts == sorted(ts)
assert len(set(ts)) == 8
def test_short_video_many_frames_never_exceeds_duration(self):
ts = text._video_frame_timestamps(1.0, 100)
assert all(0.0 <= t <= 1.0 for t in ts)
assert len(ts) == 100
# ─── Audio transcription routing (ticket 02) ─────────────────────────
class TestTranscribeAudioRouting:
def test_uses_local_model_by_default(self, monkeypatch):
monkeypatch.setattr(text, "USE_WHISPER_API", False)
monkeypatch.setattr(text, "_transcribe_audio_local", lambda p: "local transcript")
monkeypatch.setattr(text, "_transcribe_audio_api", lambda p: "api transcript")
assert text.transcribe_audio("file.mp3") == "local transcript"
def test_uses_api_when_opt_in(self, monkeypatch):
monkeypatch.setattr(text, "USE_WHISPER_API", True)
monkeypatch.setattr(text, "_transcribe_audio_local", lambda p: "local transcript")
monkeypatch.setattr(text, "_transcribe_audio_api", lambda p: "api transcript")
assert text.transcribe_audio("file.mp3") == "api transcript"
def test_falls_back_to_api_when_local_fails_and_key_set(self, monkeypatch):
monkeypatch.setattr(text, "USE_WHISPER_API", False)
monkeypatch.setattr(text, "_transcribe_audio_local", lambda p: "Error: local failed")
monkeypatch.setattr(text, "_transcribe_audio_api", lambda p: "api transcript")
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
assert text.transcribe_audio("file.mp3") == "api transcript"
def test_returns_local_error_when_no_api_key(self, monkeypatch):
monkeypatch.setattr(text, "USE_WHISPER_API", False)
monkeypatch.setattr(text, "_transcribe_audio_local", lambda p: "Error: local failed")
monkeypatch.setattr(text, "_transcribe_audio_api", lambda p: "api transcript")
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
assert text.transcribe_audio("file.mp3") == "Error: local failed"
# ─── Local device resolution (Part A: device auto-selection) ──────────
class TestIsAccelerator:
def test_mps_is_accelerator(self):
assert text._is_accelerator("mps") is True
def test_cuda_is_accelerator(self):
assert text._is_accelerator("cuda") is True
def test_cpu_is_not_accelerator(self):
assert text._is_accelerator("cpu") is False
class TestResolveLocalDevice:
"""The LOCAL_DEVICE resolver: explicit override vs auto probe (mps→cuda→cpu)."""
def _fake_torch(self, mps=False, cuda=False):
return SimpleNamespace(
backends=SimpleNamespace(mps=SimpleNamespace(is_available=lambda: mps)),
cuda=SimpleNamespace(is_available=lambda: cuda),
)
def test_explicit_cpu_is_honored(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "cpu")
assert text._resolve_local_device() == "cpu"
def test_explicit_mps_is_honored(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "mps")
assert text._resolve_local_device() == "mps"
def test_explicit_cuda_is_honored(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "cuda")
assert text._resolve_local_device() == "cuda"
def test_explicit_device_does_not_probe_torch(self, monkeypatch):
# An explicit value must short-circuit before importing/probing torch
monkeypatch.setattr(text, "LOCAL_DEVICE", "cuda")
def boom():
raise AssertionError("should not probe torch")
monkeypatch.setattr(text, "_import_torch", boom)
assert text._resolve_local_device() == "cuda"
def test_auto_prefers_mps_when_available(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "auto")
monkeypatch.setattr(text, "_import_torch", lambda: self._fake_torch(mps=True, cuda=True))
assert text._resolve_local_device() == "mps"
def test_auto_falls_back_to_cuda_when_no_mps(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "auto")
monkeypatch.setattr(text, "_import_torch", lambda: self._fake_torch(mps=False, cuda=True))
assert text._resolve_local_device() == "cuda"
def test_auto_falls_back_to_cpu_when_no_accelerator(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "auto")
monkeypatch.setattr(text, "_import_torch", lambda: self._fake_torch(mps=False, cuda=False))
assert text._resolve_local_device() == "cpu"
def test_auto_returns_cpu_when_torch_unavailable(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "auto")
def boom():
raise ImportError("no torch")
monkeypatch.setattr(text, "_import_torch", boom)
assert text._resolve_local_device() == "cpu"
def test_unknown_value_treated_as_auto(self, monkeypatch):
monkeypatch.setattr(text, "LOCAL_DEVICE", "banana")
monkeypatch.setattr(text, "_import_torch", lambda: self._fake_torch(mps=False, cuda=True))
assert text._resolve_local_device() == "cuda"
# ─── Whisper device clamping (Part A) ─────────────────────────────────
class TestWhisperDeviceConfig:
"""faster-whisper accepts only cpu/cuda (CTranslate2 has no MPS backend)."""
def test_mps_is_clamped_to_cpu_int8(self):
assert text._whisper_device_config("mps") == ("cpu", "int8")
def test_cuda_uses_float16(self):
assert text._whisper_device_config("cuda") == ("cuda", "float16")
def test_cpu_uses_int8(self):
assert text._whisper_device_config("cpu") == ("cpu", "int8")
# ─── Qwen vision load config (Part A) ─────────────────────────────────
class TestVisionLoadConfig:
"""Qwen dtype: float16 on accelerators (mps/cuda), float32 on cpu.
device_map='auto' is used only for cuda (no transformers mps device_map)."""
def _fake_torch(self):
return SimpleNamespace(float16="fp16", float32="fp32")
def test_mps_uses_float16_and_no_device_map(self):
dtype, device_map = text._vision_load_config(self._fake_torch(), "mps")
assert dtype == "fp16"
assert device_map is None
def test_cuda_uses_float16_and_auto_device_map(self):
dtype, device_map = text._vision_load_config(self._fake_torch(), "cuda")
assert dtype == "fp16"
assert device_map == "auto"
def test_cpu_uses_float32_and_no_device_map(self):
dtype, device_map = text._vision_load_config(self._fake_torch(), "cpu")
assert dtype == "fp32"
assert device_map is None
# ─── Frame-count resolution (ticket 02) ───────────────────────────────
class TestResolveFrameCount:
def test_none_uses_config(self, monkeypatch):
monkeypatch.setattr(text, "VIDEO_FRAMES_COUNT", 4)
assert text._resolve_frame_count(None) == 4
def test_positive_int_is_used(self, monkeypatch):
monkeypatch.setattr(text, "VIDEO_FRAMES_COUNT", 4)
assert text._resolve_frame_count(6) == 6
def test_zero_falls_back_to_config(self, monkeypatch):
monkeypatch.setattr(text, "VIDEO_FRAMES_COUNT", 4)
assert text._resolve_frame_count(0) == 4
def test_non_numeric_falls_back_to_config(self, monkeypatch):
monkeypatch.setattr(text, "VIDEO_FRAMES_COUNT", 4)
assert text._resolve_frame_count("banana") == 4
def test_float_string_falls_back_to_config(self, monkeypatch):
monkeypatch.setattr(text, "VIDEO_FRAMES_COUNT", 4)
assert text._resolve_frame_count("5.5") == 4
# ─── Video frame extraction (ticket 02) ───────────────────────────────
class TestExtractFramesFromVideo:
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not on PATH")
def test_extracts_evenly_spaced_frames_from_local_video(self, tmp_path):
# Generate a tiny 2-second test clip without touching the network
video_path = tmp_path / "clip.mp4"
subprocess.run(
["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi",
"-i", "testsrc=duration=2:size=64x64:rate=5",
"-pix_fmt", "yuv420p", str(video_path)],
check=True,
)
out_dir = tmp_path / "frames"
frames = text._extract_frames_from_video(str(video_path), 2, "jobtest", str(out_dir))
assert len(frames) == 2
assert sorted(os.path.basename(p) for p in frames) == [
"frame_jobtest_01.jpg",
"frame_jobtest_02.jpg",
]
for p in frames:
assert os.path.exists(p)
Image.open(p).verify() # each frame is a valid image file
# ─── Worklog serialization (ticket 03) ──────────────────────────────────
class _FakeTiming:
"""Minimal stand-in for smolagents.monitoring.Timing."""
def __init__(self, duration):
self._duration = duration
@property
def duration(self):
return self._duration
class _FakeToolCall:
"""Minimal stand-in for smolagents.memory.ToolCall."""
def __init__(self, name, arguments=None):
self.name = name
self.arguments = arguments
class _FakeActionStep:
"""Minimal stand-in for smolagents.memory.ActionStep (duck-typed)."""
def __init__(self, step_number, thought=None, tool_calls=None, observations=None,
code_action=None, duration=None, error=None,
is_final_answer=False, action_output=None):
self.step_number = step_number
self.model_output = thought
self.tool_calls = tool_calls
self.observations = observations
self.code_action = code_action
self.timing = _FakeTiming(duration) if duration is not None else None
self.error = error
self.is_final_answer = is_final_answer
self.action_output = action_output
class _FakeTaskStep:
"""Minimal stand-in for smolagents.memory.TaskStep (no step_number)."""
def __init__(self, task):
self.task = task
class TestSerializeWorklog:
def test_full_trace_fields_per_step(self):
steps = [
_FakeTaskStep(task="Q?"),
_FakeActionStep(
step_number=1,
thought="I should read the PDF.",
tool_calls=[_FakeToolCall("inspect_pdf", {"pdf_path": "a.pdf"})],
observations="Page 1 contents...",
code_action="",
duration=2.5,
),
]
worklog = text._serialize_worklog(steps, status="completed")
assert worklog["status"] == "completed"
assert worklog["total_duration_sec"] == 2.5
assert len(worklog["steps"]) == 1
step = worklog["steps"][0]
assert step["step_number"] == 1
assert step["type"] == "action"
assert step["thought"] == "I should read the PDF."
assert step["observations"] == "Page 1 contents..."
assert step["code_action"] == ""
assert step["duration_sec"] == 2.5
assert step["tool_calls"] == [{"name": "inspect_pdf", "arguments": {"pdf_path": "a.pdf"}}]
def test_final_answer_step_is_marked(self):
steps = [_FakeActionStep(step_number=1, is_final_answer=True, action_output="42")]
worklog = text._serialize_worklog(steps)
assert worklog["steps"][0]["type"] == "final"
assert worklog["steps"][0]["action_output"] == "42"
def test_tool_summary_counts_and_timing(self):
steps = [
_FakeActionStep(1, tool_calls=[_FakeToolCall("inspect_pdf")], duration=10.0),
_FakeActionStep(
2,
tool_calls=[_FakeToolCall("inspect_pdf"), _FakeToolCall("transcribe_audio")],
duration=4.0,
),
]
summary = text._serialize_tool_summary(steps)
by_name = {s["name"]: s for s in summary}
assert set(by_name) == {"inspect_pdf", "transcribe_audio"}
assert by_name["inspect_pdf"]["calls"] == 2
assert by_name["inspect_pdf"]["total_sec"] == 12.0 # 10 + half of 4
assert by_name["inspect_pdf"]["avg_sec"] == 6.0
assert by_name["transcribe_audio"]["calls"] == 1
assert by_name["transcribe_audio"]["total_sec"] == 2.0
assert by_name["transcribe_audio"]["avg_sec"] == 2.0
def test_skips_task_steps(self):
steps = [
_FakeTaskStep(task="Q?"),
_FakeActionStep(1, thought="x", duration=1.0),
]
worklog = text._serialize_worklog(steps)
assert len(worklog["steps"]) == 1
assert worklog["steps"][0]["thought"] == "x"
def test_error_field_included(self):
steps = [_FakeActionStep(1, error=RuntimeError("boom"))]
worklog = text._serialize_worklog(steps)
assert worklog["steps"][0]["error"] == "boom"
def test_missing_attributes_are_safe(self):
steps = [_FakeActionStep(1, thought=None, tool_calls=None, observations=None)]
worklog = text._serialize_worklog(steps)
step = worklog["steps"][0]
assert step["thought"] == ""
assert step["observations"] == ""
assert step["tool_calls"] == []
assert step["duration_sec"] == 0.0
def test_observations_list_is_joined(self):
steps = [_FakeActionStep(1, observations=["a", "b"])]
worklog = text._serialize_worklog(steps)
assert worklog["steps"][0]["observations"] == "a\nb"
def test_string_tool_arguments_are_parsed_as_json(self):
steps = [_FakeActionStep(1, tool_calls=[_FakeToolCall("search", '{"q": "x"}')])]
worklog = text._serialize_worklog(steps)
assert worklog["steps"][0]["tool_calls"][0]["arguments"] == {"q": "x"}
def test_total_duration_sums_steps(self):
steps = [
_FakeActionStep(1, duration=1.5),
_FakeActionStep(2, duration=2.5),
]
worklog = text._serialize_worklog(steps)
assert worklog["total_duration_sec"] == 4.0
# ─── Answer bundle I/O (ticket 03) ──────────────────────────────────────
class TestAnswerBundle:
def test_make_bundle_entry_shape(self):
entry = text._make_bundle_entry(
task_id="t1", question="Q?", level="1", file_name="a.pdf",
answer="42", worklog={"status": "completed", "steps": []},
timestamp="2026-01-01T00:00:00Z",
)
assert entry["task_id"] == "t1"
assert entry["question"] == "Q?"
assert entry["level"] == "1"
assert entry["file_name"] == "a.pdf"
assert entry["answer"] == "42"
assert entry["worklog"] == {"status": "completed", "steps": []}
assert entry["timestamp"] == "2026-01-01T00:00:00Z"
def test_load_missing_bundle_returns_empty(self, tmp_path):
assert text._load_answer_bundle(str(tmp_path / "missing.json")) == {}
def test_load_corrupt_bundle_returns_empty(self, tmp_path):
path = tmp_path / "bundle.json"
path.write_text("{not valid json")
assert text._load_answer_bundle(str(path)) == {}
def test_save_then_load_round_trip(self, tmp_path):
path = str(tmp_path / "bundle.json")
bundle = {"t1": text._make_bundle_entry(
"t1", "Q?", "1", "", "42", {"steps": []}, "2026-01-01T00:00:00Z")}
saved = text._save_answer_bundle(bundle, path)
assert saved == path
assert text._load_answer_bundle(path) == bundle
def test_save_leaves_no_temp_file(self, tmp_path):
path = str(tmp_path / "bundle.json")
text._save_answer_bundle({}, path)
assert os.path.exists(path)
assert not os.path.exists(path + ".tmp")
# ─── Run pipeline bundling (ticket 03) ──────────────────────────────────
class _FakeSolverAgent:
calls = []
def __init__(self):
pass
def solve(self, task_id, question, file_name):
_FakeSolverAgent.calls.append(task_id)
worklog = {
"steps": [{"step_number": 1, "type": "action", "thought": "t",
"tool_calls": [], "observations": "o", "code_action": "",
"duration_sec": 0.0}],
"tool_summary": [],
"total_duration_sec": 0.0,
"status": "completed",
}
return f"answer-{task_id}", worklog
class _FlakySolverAgent:
calls = []
def __init__(self):
pass
def solve(self, task_id, question, file_name):
_FlakySolverAgent.calls.append(task_id)
if task_id == "t1":
return "Error: boom", {
"steps": [], "tool_summary": [],
"total_duration_sec": 0.0, "status": "error",
}
worklog = {
"steps": [], "tool_summary": [],
"total_duration_sec": 0.0, "status": "completed",
}
return f"answer-{task_id}", worklog
class TestRunPipelineBundling:
FAKE_QUESTIONS = [
{"task_id": "t1", "question": "Q1?", "Level": "1", "file_name": ""},
{"task_id": "t2", "question": "Q2?", "Level": "2", "file_name": ""},
]
def _patch(self, monkeypatch, tmp_path, agent_cls, questions=None):
questions = self.FAKE_QUESTIONS if questions is None else questions
monkeypatch.setattr(text, "_call_with_retry",
lambda fn, **kw: questions)
monkeypatch.setattr(text, "ANSWER_BUNDLE_PATH",
str(tmp_path / "bundle.json"))
monkeypatch.setattr(text, "RESULTS_CSV_PATH",
str(tmp_path / "results.csv"))
monkeypatch.setattr(text, "GAIASolverAgent", agent_cls)
def test_first_run_solves_and_writes_bundle(self, monkeypatch, tmp_path):
_FakeSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FakeSolverAgent)
text.run_pipeline_and_save_csv(force=False)
assert set(_FakeSolverAgent.calls) == {"t1", "t2"}
bundle = text._load_answer_bundle(str(tmp_path / "bundle.json"))
assert set(bundle) == {"t1", "t2"}
assert bundle["t1"]["answer"] == "answer-t1"
assert bundle["t1"]["worklog"]["status"] == "completed"
df = pd.read_csv(tmp_path / "results.csv")
assert set(df["task_id"]) == {"t1", "t2"}
assert set(df["source"]) == {"live"}
assert set(df["status"]) == {"completed"}
def test_rerun_submits_from_bundle_without_solving(self, monkeypatch, tmp_path):
_FakeSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FakeSolverAgent)
text.run_pipeline_and_save_csv(force=False)
_FakeSolverAgent.calls = [] # reset
text.run_pipeline_and_save_csv(force=False)
assert _FakeSolverAgent.calls == []
bundle = text._load_answer_bundle(str(tmp_path / "bundle.json"))
assert set(bundle) == {"t1", "t2"}
df = pd.read_csv(tmp_path / "results.csv")
assert set(df["source"]) == {"bundle"}
assert df.loc[df["task_id"] == "t1", "answer"].iloc[0] == "answer-t1"
def test_force_reresolves_everything(self, monkeypatch, tmp_path):
_FakeSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FakeSolverAgent)
text.run_pipeline_and_save_csv(force=False)
_FakeSolverAgent.calls = []
text.run_pipeline_and_save_csv(force=True)
assert set(_FakeSolverAgent.calls) == {"t1", "t2"}
df = pd.read_csv(tmp_path / "results.csv")
assert set(df["source"]) == {"live"}
def test_failed_task_is_not_bundled_and_is_retried(self, monkeypatch, tmp_path):
_FlakySolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FlakySolverAgent)
text.run_pipeline_and_save_csv(force=False)
bundle = text._load_answer_bundle(str(tmp_path / "bundle.json"))
assert "t1" not in bundle
assert "t2" in bundle
df = pd.read_csv(tmp_path / "results.csv")
assert df.loc[df["task_id"] == "t1", "status"].iloc[0] == "error"
assert df.loc[df["task_id"] == "t2", "status"].iloc[0] == "completed"
# The failed Task is still missing from the bundle, so the next Run
# solves it again; the already-solved Task is not re-solved.
_FlakySolverAgent.calls = []
text.run_pipeline_and_save_csv(force=False)
assert _FlakySolverAgent.calls == ["t1"]
def test_force_reresolve_removes_stale_entry_on_failure(self, monkeypatch, tmp_path):
# Run 1 solves both Tasks into the bundle.
_FakeSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FakeSolverAgent)
text.run_pipeline_and_save_csv(force=False)
assert set(text._load_answer_bundle(str(tmp_path / "bundle.json"))) == {"t1", "t2"}
# Run 2 force re-solves: t1 now fails, t2 succeeds. The stale t1 entry
# must not survive (a non-forced Run would otherwise resubmit the old
# Answer while this Run's CSV shows an error).
_FlakySolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FlakySolverAgent)
text.run_pipeline_and_save_csv(force=True)
bundle = text._load_answer_bundle(str(tmp_path / "bundle.json"))
assert "t1" not in bundle
assert bundle["t2"]["answer"] == "answer-t2"
df = pd.read_csv(tmp_path / "results.csv")
assert df.loc[df["task_id"] == "t1", "status"].iloc[0] == "error"
def test_bundle_is_pruned_to_served_set(self, monkeypatch, tmp_path):
_FakeSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FakeSolverAgent)
text.run_pipeline_and_save_csv(force=False) # serves {t1, t2}
assert set(text._load_answer_bundle(str(tmp_path / "bundle.json"))) == {"t1", "t2"}
# Server now serves only t1: the bundle must be pruned so it stays
# consistent with the results CSV and the served Question set.
_FakeSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _FakeSolverAgent,
questions=self.FAKE_QUESTIONS[:1])
text.run_pipeline_and_save_csv(force=False)
bundle = text._load_answer_bundle(str(tmp_path / "bundle.json"))
assert set(bundle) == {"t1"}
def test_forced_answer_is_folded_into_bundle(self, monkeypatch, tmp_path):
# A soft-deadline Task that yields a synthesized Answer (status "forced")
# is submitted and scored (decision B3), so it must be folded into the
# bundle like a completed Task.
class _ForcedSolverAgent(_FakeSolverAgent):
calls = []
def solve(self, task_id, question, file_name):
_ForcedSolverAgent.calls.append(task_id)
worklog = {
"steps": [], "tool_summary": [],
"total_duration_sec": 0.0, "status": "forced",
}
return f"forced-{task_id}", worklog
_ForcedSolverAgent.calls = []
self._patch(monkeypatch, tmp_path, _ForcedSolverAgent)
text.run_pipeline_and_save_csv(force=False)
bundle = text._load_answer_bundle(str(tmp_path / "bundle.json"))
assert set(bundle) == {"t1", "t2"}
assert bundle["t1"]["answer"] == "forced-t1"
assert bundle["t1"]["worklog"]["status"] == "forced"
df = pd.read_csv(tmp_path / "results.csv")
assert set(df["status"]) == {"forced"}
# ─── solve() Worklog capture on failure (ticket 03) ─────────────────────
class TestSolveWorklogOnError:
def test_captures_partial_trace_when_run_fails(self):
class _FakeMemory:
def __init__(self):
self.steps = []
def reset(self):
self.steps = []
class _BoomAgent:
def __init__(self):
self.memory = _FakeMemory()
def run(self, prompt):
self.memory.steps = [
_FakeActionStep(1, thought="partial thought", duration=1.0)
]
raise RuntimeError("model exploded")
# Build a GAIASolverAgent without __init__ (which needs DEEPSEEK_API_KEY)
solver = object.__new__(text.GAIASolverAgent)
solver.agent = _BoomAgent()
answer, worklog = solver.solve("t1", "Q?", "")
assert answer.startswith("Error:")
assert worklog["status"] == "error"
assert worklog["error"] == "model exploded"
assert len(worklog["steps"]) == 1
assert worklog["steps"][0]["thought"] == "partial thought"
# ─── Agent toolset: text/web-only Space fallback (ticket 04, ADR-0002) ──
class TestGAIASolverAgentToolset:
def _build(self, monkeypatch, multimodal):
captured = {}
class _FakeCodeAgent:
def __init__(self, **kwargs):
captured["tools"] = kwargs["tools"]
monkeypatch.setattr(text, "DEEPSEEK_API_KEY", "sk-test")
monkeypatch.setattr(text, "CodeAgent", _FakeCodeAgent)
text.GAIASolverAgent(multimodal=multimodal)
return {t.name for t in captured["tools"]}
def test_full_toolset_includes_multimodal_tools(self, monkeypatch):
names = self._build(monkeypatch, multimodal=True)
assert {"inspect_image", "transcribe_audio", "extract_video_frames"} <= names
assert {"web_search", "get_youtube_transcript", "inspect_pdf"} <= names
def test_text_only_toolset_excludes_multimodal_tools(self, monkeypatch):
# ADR-0002: the live-on-Space fallback is text/web-only (no local vision
# or audio models), so those tools must not be registered.
names = self._build(monkeypatch, multimodal=False)
assert "inspect_image" not in names
assert "transcribe_audio" not in names
assert "extract_video_frames" not in names
# Web + document tools stay available
assert {"web_search", "get_youtube_transcript", "inspect_pdf",
"inspect_excel", "read_file_as_text", "inspect_docx",
"inspect_pptx", "extract_zip"} <= names
# ─── solve() explicit file_path (ticket 04) ────────────────────────────
class TestSolveExplicitFilePath:
def _make_solver(self):
class _FakeMemory:
def __init__(self):
self.steps = []
def reset(self):
self.steps = []
class _FakeAgent:
def __init__(self):
self.memory = _FakeMemory()
self.prompt = None
def run(self, prompt):
self.prompt = prompt
return "42"
solver = object.__new__(text.GAIASolverAgent)
solver.agent = _FakeAgent()
solver._clean_answer = lambda q, a: a # skip the LLM cleaning call
return solver
def test_explicit_file_path_skips_local_lookup(self, monkeypatch):
solver = self._make_solver()
seen = {"called": False}
def fake_lookup(task_id, file_name):
seen["called"] = True
return "/local/path"
monkeypatch.setattr(text, "_get_local_task_file", fake_lookup)
answer, worklog = solver.solve("t1", "Q?", "doc.pdf",
file_path="/downloaded/t1.pdf")
assert answer == "42"
assert seen["called"] is False # explicit path wins over local lookup
assert "/downloaded/t1.pdf" in solver.agent.prompt
assert worklog["status"] == "completed"
def test_missing_file_path_uses_local_lookup(self, monkeypatch):
solver = self._make_solver()
seen = {"path": ""}
def fake_lookup(task_id, file_name):
seen["path"] = "/local/report.pdf"
return "/local/report.pdf"
monkeypatch.setattr(text, "_get_local_task_file", fake_lookup)
solver.solve("t1", "Q?", "report.pdf")
assert seen["path"] == "/local/report.pdf"
assert "/local/report.pdf" in solver.agent.prompt