File size: 965 Bytes
99bf65f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | from lmms_eval.filters.extraction import MultiChoiceRegexFilter
def parse_multi_choice_answer(answer):
# Example responses and documents
model_responses = [["The answer is (B)", "I believe it is (A)", "(C) seems correct"], ["Answer is: B!", "Answer: B", "Answer: B"]] # Model response set 1 # Model response set 2
documents = [{"choices": ["A. Apple", "B. Banana", "C. Cherry"]}, {"choices": ["A. Alpha", "B. Beta", "C. Gamma"]}] # Multiple choice options for question 1 # Multiple choice options for question 2
# Instantiate the filter
multi_choice_filter = MultiChoiceRegexFilter(regex_pattern=r"\(([A-D])\)", group_select=0, ignore_case=False, ignore_punctuation=True)
filtered_responses = multi_choice_filter.apply(model_responses, documents)
# Print the filtered answers
for i, filtered in enumerate(filtered_responses):
print(f"Question {i+1} filtered responses: {filtered}")
parse_multi_choice_answer("a")
|