Spaces:
Configuration error
Configuration error
File size: 5,295 Bytes
e59ae36 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | """
Evaluation runner.
Runs both the baseline (single prompt) and the full agent (multi-persona
debate graph) on every scenario in eval_data/scenarios.json, and saves
raw outputs + basic run metadata (timing, retry counts) to a results
file. This does NOT auto-score quality -- per the hackathon brief, a
"good final result" is user-defined, and forcing an LLM to grade its own
sibling's outputs would be circular evidence. Instead this script produces
the paired baseline/agent transcripts that a human (you) scores against
the rubric in eval_data/rubric.md, and that go into the submission as
raw evidence.
Usage:
python evaluate.py # run all scenarios
python evaluate.py --limit 2 # quick smoke test on first 2
"""
import json
import time
import shutil
import argparse
import tempfile
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
from baseline import run_baseline
from graph import run_debate
from memory.vector_store import DecisionMemory
SCENARIOS_PATH = Path("eval_data/scenarios.json")
RESULTS_PATH = Path("eval_data/results.json")
def load_scenarios():
with open(SCENARIOS_PATH, "r", encoding="utf-8") as f:
return json.load(f)
def run_one_scenario(scenario: dict) -> dict:
"""Each scenario gets a FRESH, isolated memory store.
Discovered issue: running all benchmark scenarios against one shared
persistent memory (as this originally did) caused cross-scenario
contamination -- e.g. the 'used_vs_new_car' scenario's agent output
referenced "your upcoming housing goal", which the user never
mentioned in that query. It leaked in because the unrelated
'rent_vs_buy' scenario, run earlier in the same batch, had been
stored in memory and got retrieved as a "similar past decision" by
TF-IDF's loose similarity matching.
This is a real failure mode, not just a benchmark artifact: in
production, an unrelated prior decision (even the user's own) being
injected as unstated context could distort advice on a fresh
question. The fix here is narrow and honest about its limits: for
evaluation, we isolate memory per scenario so the debate quality is
judged on its own merits, uncontaminated. Memory's actual value
(recalling a user's own genuinely related past decisions) would need
a separate, deliberately-related test case to demonstrate -- it is
not validated by this benchmark run.
"""
query = scenario["query"]
print(f"\n=== Scenario: {scenario['id']} ===")
print(f"Query: {query}")
temp_dir = tempfile.mkdtemp(prefix="eval_memory_")
memory = DecisionMemory(persist_dir=temp_dir)
try:
print("Running baseline...")
t0 = time.time()
baseline_output = run_baseline(query)
baseline_time = time.time() - t0
print(f" baseline done in {baseline_time:.1f}s")
print("Running agent (multi-persona debate)...")
t0 = time.time()
agent_result = run_debate(query, memory)
agent_time = time.time() - t0
print(f" agent done in {agent_time:.1f}s (verification retries: {agent_result.get('retry_count', 0) - 1})")
return {
"id": scenario["id"],
"category": scenario.get("category"),
"query": query,
"note": scenario.get("note", ""),
"baseline": {
"output": baseline_output,
"time_seconds": round(baseline_time, 2),
},
"agent": {
"optimist_view": agent_result["optimist_view"],
"skeptic_view": agent_result["skeptic_view"],
"analyst_view": agent_result["analyst_view"],
"moderator_output": agent_result["moderator_output"],
"verification_retries": agent_result.get("retry_count", 1) - 1,
"time_seconds": round(agent_time, 2),
},
}
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--limit", type=int, default=None, help="Only run the first N scenarios")
args = parser.parse_args()
scenarios = load_scenarios()
if args.limit:
scenarios = scenarios[: args.limit]
results = []
for scenario in scenarios:
try:
result = run_one_scenario(scenario)
except RuntimeError as e:
if "DAILY quota exhausted" in str(e):
print(f"\n STOPPING: {e}")
print(f" Saving {len(results)} completed results so far before exiting.")
break
print(f" ERROR on {scenario['id']}: {e}")
result = {"id": scenario["id"], "query": scenario["query"], "error": str(e)}
results.append(result)
continue
except Exception as e:
print(f" ERROR on {scenario['id']}: {e}")
result = {"id": scenario["id"], "query": scenario["query"], "error": str(e)}
results.append(result)
RESULTS_PATH.parent.mkdir(exist_ok=True)
with open(RESULTS_PATH, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2, ensure_ascii=False)
print(f"\nSaved {len(results)} results to {RESULTS_PATH}")
if __name__ == "__main__":
main() |