| from vela_guard import evaluate | |
| def test_build_request_uses_task_question(): | |
| row = {"task": "injection", "state": {"request": "r", "tools": "t"}} | |
| state, questions = evaluate.build_request(row) | |
| assert state == row["state"] | |
| assert questions["injection"]["type"] == "noul" | |
| def test_parse_span_response(): | |
| resp = {"answers": {"secrets": {"type": "noul", "noul": 0.9}}, | |
| "spans": {"secrets": [{"label": "api_key", "start": 3, "end": 9, "text": "abc", "probability": 0.8}]}} | |
| pred = evaluate.parse_response("secrets", resp) | |
| assert pred == {"spans": [{"label": "api_key", "start": 3, "end": 9}], "score": 0.9} | |
| def test_parse_span_response_without_spans(): | |
| assert evaluate.parse_response("secrets", {"answers": {"secrets": {"noul": 0.1}}}) == {"spans": [], "score": 0.1} | |
| def test_parse_noul_and_choice(): | |
| assert evaluate.parse_response("injection", {"answers": {"injection": {"noul": 0.7}}}) == {"score": 0.7} | |
| resp = {"answers": {"toolcall": {"choice": "block", "probabilities": {"allow": 0.1, "confirm": 0.2, "block": 0.7}}}} | |
| assert evaluate.parse_response("toolcall", resp) == {"choice": "block", "probabilities": {"allow": 0.1, "confirm": 0.2, "block": 0.7}} | |
| def test_score_task_reports_all_and_hard_subsets(): | |
| rows = [ | |
| {"task": "injection", "label": True, "hard": False}, | |
| {"task": "injection", "label": False, "hard": True}, | |
| ] | |
| preds = [{"score": 0.9}, {"score": 0.8}] | |
| report = evaluate.score_task("injection", rows, preds) | |
| assert report["all"]["n"] == 2 and report["hard"]["n"] == 1 | |
| assert report["hard"]["accuracy"] == 0.0 | |
| def test_score_secrets_includes_detection_view(): | |
| rows = [{"task": "secrets", "label": True, "hard": False, "spans": [{"label": "api_key", "start": 0, "end": 4}]}, | |
| {"task": "secrets", "label": False, "hard": True, "spans": []}] | |
| preds = [{"spans": [{"label": "api_key", "start": 0, "end": 4}], "score": 0.9}, {"spans": [], "score": 0.1}] | |
| report = evaluate.score_task("secrets", rows, preds) | |
| assert report["all"]["exact"]["f1"] == 1.0 | |
| assert report["all"]["detection"]["accuracy"] == 1.0 | |