import re from collections import Counter def extract(text): text = text.split('')[-1].strip() index = text.rfind('\\boxed') if index < 0: index = text.rfind('\\fbox') if index < 0: return None key = '\\boxed' if text.startswith('\\boxed', index) else '\\fbox' rest = text[index + len(key):] stripped = rest.lstrip() if stripped.startswith('{'): depth = 0 for i, char in enumerate(stripped): depth += (char == '{') - (char == '}') if depth == 0: return stripped[1:i].strip() or None elif key == '\\boxed' and rest.startswith(' '): return rest.strip().split('$')[0].split('\n')[0].strip() or None return None def correct(prediction, gold): if prediction is None: return False from math_verify import parse, verify reference = parse('\\boxed{' + gold + '}', parsing_timeout=5) candidate = parse('\\boxed{' + prediction + '}', parsing_timeout=5) return bool(reference and candidate and verify(reference, candidate, timeout_seconds=5)) def repetition(text): words = text.split()[-1000:] grams = [tuple(words[i:i + 8]) for i in range(max(0, len(words) - 7))] duplicate = 1 - len(set(grams)) / max(1, len(grams)) tail = words[-200:] four = [tuple(tail[i:i + 4]) for i in range(max(0, len(tail) - 3))] counts = Counter(four) repetitive = bool(four) and sum(counts[g] >= 3 for g in four) / len(four) >= .5 return {'duplicate_8gram_tail': duplicate, 'repetitive_end': repetitive, 'whitespace_tail': bool(text) and not text[-1000:].strip(), 'thinking_closed': '' in text} def grade_ifeval(row, text): import nltk from common import ROOT from langdetect import DetectorFactory from instruction_following_eval import instructions_registry as registry nltk.data.path[:] = [str(ROOT / 'data/nltk')] DetectorFactory.seed = 0 response = text.split('')[-1].strip() if '' in text else '' flags = [] for instruction_id, kwargs in zip(row['instruction_id_list'], row['kwargs']): checker = registry.INSTRUCTION_DICT[instruction_id](instruction_id) checker.build_description(**{k: v for k, v in (kwargs or {}).items() if v is not None}) args = checker.get_instruction_args() if args and 'prompt' in args: checker.build_description(prompt=row['prompt']) flags.append(bool(response) and bool(checker.check_following(response))) return all(flags)