| { | |
| "artifacts": "/home/simon/Projects/mverify/artifacts", | |
| "eval": { | |
| "n": 500, | |
| "accuracy": 0.86, | |
| "pass": { | |
| "precision": 0.4848, | |
| "recall": 0.9697, | |
| "f1": 0.6465, | |
| "tp": 64, | |
| "fp": 68, | |
| "fn": 2 | |
| }, | |
| "fail": { | |
| "precision": 0.9946, | |
| "recall": 0.8433, | |
| "f1": 0.9127, | |
| "tp": 366, | |
| "fp": 2, | |
| "fn": 68 | |
| }, | |
| "fpr_good": 0.0303, | |
| "blatant_fail_recall": 0.9563, | |
| "blatant_n": 183, | |
| "per_category": { | |
| "pass": { | |
| "n": 66, | |
| "accuracy": 0.9697, | |
| "fail_recall": null | |
| }, | |
| "off_topic": { | |
| "n": 62, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| }, | |
| "empty": { | |
| "n": 62, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| }, | |
| "truncated": { | |
| "n": 63, | |
| "accuracy": 0.9524, | |
| "fail_recall": 0.9524 | |
| }, | |
| "ignored_constraint": { | |
| "n": 59, | |
| "accuracy": 0.8644, | |
| "fail_recall": 0.8644 | |
| }, | |
| "wrong_format": { | |
| "n": 62, | |
| "accuracy": 0.8387, | |
| "fail_recall": 0.8387 | |
| }, | |
| "different_question": { | |
| "n": 63, | |
| "accuracy": 0.254, | |
| "fail_recall": 0.254 | |
| }, | |
| "confident_refusal": { | |
| "n": 63, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| } | |
| }, | |
| "latency_ms": { | |
| "mean": 0.127, | |
| "p95": 0.217, | |
| "max": 2.349 | |
| }, | |
| "targets": { | |
| "fpr_good_lt": 0.05, | |
| "blatant_fail_recall_gt": 0.95, | |
| "p95_ms_lt": 200.0 | |
| }, | |
| "pass_targets": true | |
| }, | |
| "eval_bayes": { | |
| "n": 500, | |
| "accuracy": 0.794, | |
| "pass": { | |
| "precision": 0.2319, | |
| "recall": 0.2424, | |
| "f1": 0.237, | |
| "tp": 16, | |
| "fp": 53, | |
| "fn": 50 | |
| }, | |
| "fail": { | |
| "precision": 0.884, | |
| "recall": 0.8779, | |
| "f1": 0.8809, | |
| "tp": 381, | |
| "fp": 50, | |
| "fn": 53 | |
| }, | |
| "fpr_good": 0.7576, | |
| "blatant_fail_recall": 0.918, | |
| "blatant_n": 183, | |
| "per_category": { | |
| "pass": { | |
| "n": 66, | |
| "accuracy": 0.2424, | |
| "fail_recall": null | |
| }, | |
| "off_topic": { | |
| "n": 62, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| }, | |
| "empty": { | |
| "n": 62, | |
| "accuracy": 0.9355, | |
| "fail_recall": 0.9355 | |
| }, | |
| "truncated": { | |
| "n": 63, | |
| "accuracy": 0.8254, | |
| "fail_recall": 0.8254 | |
| }, | |
| "ignored_constraint": { | |
| "n": 59, | |
| "accuracy": 0.8136, | |
| "fail_recall": 0.8136 | |
| }, | |
| "wrong_format": { | |
| "n": 62, | |
| "accuracy": 0.8065, | |
| "fail_recall": 0.8065 | |
| }, | |
| "different_question": { | |
| "n": 63, | |
| "accuracy": 0.7619, | |
| "fail_recall": 0.7619 | |
| }, | |
| "confident_refusal": { | |
| "n": 63, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| } | |
| }, | |
| "latency_ms": { | |
| "mean": 0.0, | |
| "p95": 0.0, | |
| "max": 0.0 | |
| }, | |
| "targets": { | |
| "fpr_good_lt": 0.05, | |
| "blatant_fail_recall_gt": 0.95, | |
| "p95_ms_lt": 200.0 | |
| }, | |
| "pass_targets": false | |
| }, | |
| "test": { | |
| "n": 151, | |
| "accuracy": 0.8874, | |
| "pass": { | |
| "precision": 0.5294, | |
| "recall": 0.9474, | |
| "f1": 0.6792, | |
| "tp": 18, | |
| "fp": 16, | |
| "fn": 1 | |
| }, | |
| "fail": { | |
| "precision": 0.9915, | |
| "recall": 0.8788, | |
| "f1": 0.9317, | |
| "tp": 116, | |
| "fp": 1, | |
| "fn": 16 | |
| }, | |
| "fpr_good": 0.0526, | |
| "blatant_fail_recall": 0.9643, | |
| "blatant_n": 56, | |
| "per_category": { | |
| "pass": { | |
| "n": 19, | |
| "accuracy": 0.9474, | |
| "fail_recall": null | |
| }, | |
| "off_topic": { | |
| "n": 19, | |
| "accuracy": 0.9474, | |
| "fail_recall": 0.9474 | |
| }, | |
| "empty": { | |
| "n": 19, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| }, | |
| "truncated": { | |
| "n": 19, | |
| "accuracy": 0.9474, | |
| "fail_recall": 0.9474 | |
| }, | |
| "ignored_constraint": { | |
| "n": 18, | |
| "accuracy": 0.9444, | |
| "fail_recall": 0.9444 | |
| }, | |
| "wrong_format": { | |
| "n": 19, | |
| "accuracy": 0.8421, | |
| "fail_recall": 0.8421 | |
| }, | |
| "different_question": { | |
| "n": 19, | |
| "accuracy": 0.4737, | |
| "fail_recall": 0.4737 | |
| }, | |
| "confident_refusal": { | |
| "n": 19, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| } | |
| }, | |
| "latency_ms": { | |
| "mean": 0.087, | |
| "p95": 0.195, | |
| "max": 0.235 | |
| }, | |
| "targets": { | |
| "fpr_good_lt": 0.05, | |
| "blatant_fail_recall_gt": 0.95, | |
| "p95_ms_lt": 200.0 | |
| }, | |
| "pass_targets": false | |
| }, | |
| "gold": { | |
| "n": 48, | |
| "accuracy": 0.8333, | |
| "pass": { | |
| "precision": 0.6667, | |
| "recall": 0.7692, | |
| "f1": 0.7143, | |
| "tp": 10, | |
| "fp": 5, | |
| "fn": 3 | |
| }, | |
| "fail": { | |
| "precision": 0.9091, | |
| "recall": 0.8571, | |
| "f1": 0.8824, | |
| "tp": 30, | |
| "fp": 3, | |
| "fn": 5 | |
| }, | |
| "fpr_good": 0.2308, | |
| "blatant_fail_recall": 0.875, | |
| "blatant_n": 16, | |
| "per_category": { | |
| "pass": { | |
| "n": 13, | |
| "accuracy": 0.7692, | |
| "fail_recall": null | |
| }, | |
| "off_topic": { | |
| "n": 5, | |
| "accuracy": 0.8, | |
| "fail_recall": 0.8 | |
| }, | |
| "empty": { | |
| "n": 4, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| }, | |
| "truncated": { | |
| "n": 4, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| }, | |
| "ignored_constraint": { | |
| "n": 7, | |
| "accuracy": 0.8571, | |
| "fail_recall": 0.8571 | |
| }, | |
| "wrong_format": { | |
| "n": 5, | |
| "accuracy": 0.8, | |
| "fail_recall": 0.8 | |
| }, | |
| "different_question": { | |
| "n": 4, | |
| "accuracy": 0.5, | |
| "fail_recall": 0.5 | |
| }, | |
| "confident_refusal": { | |
| "n": 6, | |
| "accuracy": 1.0, | |
| "fail_recall": 1.0 | |
| } | |
| }, | |
| "latency_ms": { | |
| "mean": 0.083, | |
| "p95": 0.155, | |
| "max": 0.196 | |
| }, | |
| "targets": { | |
| "fpr_good_lt": 0.05, | |
| "blatant_fail_recall_gt": 0.95, | |
| "p95_ms_lt": 200.0 | |
| }, | |
| "pass_targets": false | |
| } | |
| } | |