Buckets:
| import unittest | |
| from uniar_adapter.eval_cv_bench import compute_metrics, official_extract_choice, parse_choice | |
| class CVBenchParserTest(unittest.TestCase): | |
| def test_parses_explicit_and_choice_text_answers(self): | |
| self.assertEqual(parse_choice("The answer is (C).", ["3", "2", "1", "0"])[0], "C") | |
| self.assertEqual(parse_choice("B", ["table", "bookcase"])[0], "B") | |
| self.assertEqual(parse_choice("bookcase", ["table", "bookcase"])[0], "B") | |
| def test_explicit_answer_wins_over_echoed_options(self): | |
| output = "(A) red (B) blue (C) green. Final answer: (B)" | |
| self.assertEqual(parse_choice(output, ["red", "blue", "green"])[0], "B") | |
| def test_does_not_parse_article_as_option_a(self): | |
| output = "Based on a careful examination, there is a wooden structure." | |
| self.assertIsNone(parse_choice(output, ["red", "blue"])[0]) | |
| def test_released_official_extractor(self): | |
| self.assertEqual(official_extract_choice("(C) 1"), "C") | |
| self.assertEqual(official_extract_choice("Based on the image"), "B") | |
| class CVBenchMetricsTest(unittest.TestCase): | |
| def test_official_source_weighting(self): | |
| records = [ | |
| {"source": "ADE20K", "type": "2D", "task": "Count", "correct": True, "robust_correct": True, "pred_letter": "A", "official_pred_letter": "A", "parse_method": "leading_letter"}, | |
| {"source": "ADE20K", "type": "2D", "task": "Relation", "correct": False, "robust_correct": False, "pred_letter": "B", "official_pred_letter": "B", "parse_method": "leading_letter"}, | |
| {"source": "COCO", "type": "2D", "task": "Count", "correct": True, "robust_correct": True, "pred_letter": "A", "official_pred_letter": "A", "parse_method": "leading_letter"}, | |
| {"source": "Omni3D", "type": "3D", "task": "Depth", "correct": False, "robust_correct": False, "pred_letter": None, "official_pred_letter": None, "parse_method": "empty"}, | |
| ] | |
| metrics = compute_metrics(records) | |
| self.assertEqual(metrics["accuracy_2d"], 0.75) | |
| self.assertEqual(metrics["accuracy_3d"], 0.0) | |
| self.assertEqual(metrics["cv_bench_accuracy"], 0.375) | |
| self.assertEqual(metrics["robust_task_accuracy"]["2D:Count"], 1.0) | |
| self.assertEqual(metrics["parse_failures"], 1) | |
| if __name__ == "__main__": | |
| unittest.main() | |
Xet Storage Details
- Size:
- 2.34 kB
- Xet hash:
- f4c411261e33d4263726e7dd995833ebee451ff82317d82feccbb289b54b3790
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.