| import pandas as pd |
| from tqdm import tqdm |
| import logging |
| import re |
| from word2number import w2n |
| from dataflow.utils.utils import get_logger |
|
|
| class StringProcessor: |
| """ |
| A class that encapsulates various string processing functions for mathematical expressions. |
| """ |
|
|
| @staticmethod |
| def _fix_fracs(string): |
| """ |
| Fixes fraction expressions in the string, ensuring they are properly formatted as \frac{a}{b}. |
| """ |
| substrs = string.split("\\frac") |
| new_str = substrs[0] |
| if len(substrs) > 1: |
| for substr in substrs[1:]: |
| new_str += "\\frac" |
| if len(substr) > 0 and substr[0] == "{": |
| new_str += substr |
| else: |
| if len(substr) >= 2: |
| a, b = substr[0], substr[1] |
| if b != "{": |
| new_str += f"{{{a}}}{{{b}}}{substr[2:]}" if len(substr) > 2 else f"{{{a}}}{{{b}}}" |
| else: |
| new_str += f"{{{a}}}{b}{substr[2:]}" if len(substr) > 2 else f"{{{a}}}{b}" |
| else: |
| return string |
| return new_str |
|
|
| @staticmethod |
| def _fix_a_slash_b(string): |
| """ |
| Fixes cases where a fraction is represented as a simple division (e.g., a/b) and converts it to \frac{a}{b}. |
| """ |
| if len(string.split("/")) != 2: |
| return string |
| a, b = string.split("/") |
| try: |
| a, b = int(a) if "sqrt" not in a else a, int(b) if "sqrt" not in b else b |
| assert string == f"{a}/{b}" |
| return f"\\frac{{{a}}}{{{b}}}" |
| except: |
| return string |
|
|
| @staticmethod |
| def _fix_sqrt(string): |
| """ |
| Ensures that square root expressions are properly formatted as \sqrt{...}. |
| """ |
| return re.sub(r"\\sqrt(\w+)", r"\\sqrt{\1}", string) |
|
|
| @staticmethod |
| def convert_word_number(text: str) -> str: |
| """ |
| Converts a word representation of a number to a digit. |
| """ |
| try: |
| return str(w2n.word_to_num(text)) |
| except: |
| return text |
|
|
|
|
| |
| class UnitTextManager: |
| """ |
| A class that encapsulates unit text management to remove unwanted unit terms from strings. |
| """ |
|
|
| def __init__(self): |
| """ |
| Initializes the unit texts and their plural forms. |
| """ |
| self.unit_texts = [ |
| "east", "degree", "mph", "kmph", "ft", "m sqaure", "m east", "sq m", "deg", "mile", "q .", "monkey", "prime", |
| "ratio", "profit of rs", "rd", "o", "gm", "p . m", "lb", "tile", "per", "dm", "lt", "gain", "ab", "way", "west", |
| "a .", "b .", "c .", "d .", "e .", "f .", "g .", "h .", "t", "a", "h", "no change", "men", "soldier", "pie", "bc", |
| "excess", "st", "inches", "noon", "percent", "by", "gal", "kmh", "c", "acre", "rise", "a . m", "th", "π r 2", "sq", |
| "mark", "l", "toy", "coin", "sq . m", "gallon", "° f", "profit", "minw", "yr", "women", "feet", "am", "pm", "hr", |
| "cu cm", "square", "v â € ™", "are", "rupee", "rounds", "cubic", "cc", "mtr", "s", "ohm", "number", "kmph", "day", |
| "hour", "minute", "min", "second", "man", "woman", "sec", "cube", "mt", "sq inch", "mp", "∏ cm ³", "hectare", |
| "more", "sec", "unit", "cu . m", "cm 2", "rs .", "rs", "kg", "g", "month", "km", "m", "cm", "mm", "apple", "liter", |
| "loss", "yard", "pure", "year", "increase", "decrease", "d", "less", "Surface", "litre", "pi sq m", "s .", "metre", |
| "meter", "inch", |
| ] |
| self.unit_texts.extend([t + "s" for t in self.unit_texts]) |
|
|
| def clean_units(self, string: str): |
| """ |
| Cleans the string by removing unit terms from it. |
| """ |
| for unit_text in self.unit_texts: |
| string = re.sub(r"(^|\W)" + unit_text + r"($|\W)", r"\1\2", string) |
| return string |
|
|
|
|
| |
| class StringCleaner: |
| """ |
| A class responsible for cleaning and formatting strings in mathematical expressions. |
| """ |
|
|
| def __init__(self, unit_manager: UnitTextManager): |
| """ |
| Initializes the StringCleaner class with a unit manager. |
| """ |
| self.unit_manager = unit_manager |
|
|
| def strip_string(self, string, skip_unit=False): |
| """ |
| Strips unwanted characters and units from the string. |
| """ |
| string = str(string).strip().replace("\n", "").rstrip(".").replace("\\!", "") |
| string = re.sub(r"\\begin\{array\}\{.*?\}", r"\\begin{pmatrix}", string) |
| string = re.sub(r"\\end\{array\}", r"\\end{pmatrix}", string).replace("bmatrix", "pmatrix") |
| string = string.replace("tfrac", "frac").replace("dfrac", "frac").replace("\\neq", "\\ne").replace("\\leq", "\\le").replace("\\geq", "\\ge") |
| string = string.replace("\\left", "").replace("\\right", "").replace("\\{", "{").replace("\\}", "}") |
| |
| |
| if not skip_unit: |
| string = self.unit_manager.clean_units(string) |
|
|
| string = string.replace("^{\\circ}", "").replace("^\\circ", "").replace("\\$", "").replace("$", "").replace("\\(", "").replace("\\)", "") |
| string = StringProcessor.convert_word_number(string) |
| string = re.sub(r"\\text\{(.*?)\}", r"\1", string) |
| |
| for key in ["x=", "y=", "z=", "x\\in", "y\\in", "z\\in", "x\\to", "y\\to", "z\\to"]: |
| string = string.replace(key, "") |
| |
| string = string.replace("\\emptyset", r"{}").replace("(-\\infty,\\infty)", "\\mathbb{R}") |
| string = string.replace("%", "").replace(" .", " 0.").replace("{.", "{0.") |
| |
| return string |
|
|
|
|
| |
| class AnswerExtractor: |
| """ |
| A class responsible for extracting the final answer from a prediction string. |
| """ |
|
|
| def __init__(self, string_cleaner: StringCleaner): |
| """ |
| Initializes the AnswerExtractor class with a string cleaner. |
| """ |
| self.string_cleaner = string_cleaner |
| self.logger = get_logger() |
|
|
| def extract_answer(self, pred_str, data_name, use_last_number=True): |
| """ |
| Extracts the final answer from the prediction string, processing various formats. |
| """ |
| if not pred_str: |
| pred_str = "" |
| pred_str = str(pred_str).replace("\u043a\u0438", "") |
| |
| |
| if "final answer is $" in pred_str and "$. I hope" in pred_str: |
| pred = pred_str.split("final answer is $", 1)[1].split("$. I hope", 1)[0].strip() |
| elif "boxed" in pred_str: |
| pred = self._extract_boxed_answer(pred_str) |
| elif "he answer is" in pred_str: |
| pred = pred_str.split("he answer is")[-1].strip() |
| else: |
| pred = self._get_last_number_answer(pred_str, use_last_number) |
| |
| pred = self.string_cleaner.strip_string(pred, skip_unit=data_name in ["carp_en", "minerva_math"]) |
| return pred |
|
|
| def _extract_boxed_answer(self, pred_str): |
| """ |
| Extracts answers enclosed in 'boxed' notation. |
| """ |
| ans = pred_str.split("boxed")[-1] |
| if ans.startswith("{"): |
| return self._extract_bracketed_answer(ans) |
| else: |
| return ans.split("$")[0].strip() |
|
|
| def _extract_bracketed_answer(self, ans): |
| """ |
| Handles answers that are enclosed within brackets. |
| """ |
| stack = 1 |
| result = "" |
| for c in ans[1:]: |
| if c == "{": |
| stack += 1 |
| result += c |
| elif c == "}": |
| stack -= 1 |
| if stack == 0: |
| break |
| result += c |
| else: |
| result += c |
| return result |
|
|
| def _get_last_number_answer(self, pred_str, use_last_number): |
| """ |
| Extracts the last number from the string if use_last_number is True. |
| """ |
| if use_last_number: |
| pattern = "-?\d*\.?\d+" |
| pred = re.findall(pattern, pred_str.replace(",", "")) |
| return pred[-1] if pred else "" |
| return "" |
|
|
