File size: 2,558 Bytes
fc32096 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import re
from collections import Counter
def extract(text):
text = text.split('</think>')[-1].strip()
index = text.rfind('\\boxed')
if index < 0:
index = text.rfind('\\fbox')
if index < 0:
return None
key = '\\boxed' if text.startswith('\\boxed', index) else '\\fbox'
rest = text[index + len(key):]
stripped = rest.lstrip()
if stripped.startswith('{'):
depth = 0
for i, char in enumerate(stripped):
depth += (char == '{') - (char == '}')
if depth == 0:
return stripped[1:i].strip() or None
elif key == '\\boxed' and rest.startswith(' '):
return rest.strip().split('$')[0].split('\n')[0].strip() or None
return None
def correct(prediction, gold):
if prediction is None:
return False
from math_verify import parse, verify
reference = parse('\\boxed{' + gold + '}', parsing_timeout=5)
candidate = parse('\\boxed{' + prediction + '}', parsing_timeout=5)
return bool(reference and candidate and verify(reference, candidate, timeout_seconds=5))
def repetition(text):
words = text.split()[-1000:]
grams = [tuple(words[i:i + 8]) for i in range(max(0, len(words) - 7))]
duplicate = 1 - len(set(grams)) / max(1, len(grams))
tail = words[-200:]
four = [tuple(tail[i:i + 4]) for i in range(max(0, len(tail) - 3))]
counts = Counter(four)
repetitive = bool(four) and sum(counts[g] >= 3 for g in four) / len(four) >= .5
return {'duplicate_8gram_tail': duplicate, 'repetitive_end': repetitive,
'whitespace_tail': bool(text) and not text[-1000:].strip(),
'thinking_closed': '</think>' in text}
def grade_ifeval(row, text):
import nltk
from common import ROOT
from langdetect import DetectorFactory
from instruction_following_eval import instructions_registry as registry
nltk.data.path[:] = [str(ROOT / 'data/nltk')]
DetectorFactory.seed = 0
response = text.split('</think>')[-1].strip() if '</think>' in text else ''
flags = []
for instruction_id, kwargs in zip(row['instruction_id_list'], row['kwargs']):
checker = registry.INSTRUCTION_DICT[instruction_id](instruction_id)
checker.build_description(**{k: v for k, v in (kwargs or {}).items() if v is not None})
args = checker.get_instruction_args()
if args and 'prompt' in args:
checker.build_description(prompt=row['prompt'])
flags.append(bool(response) and bool(checker.check_following(response)))
return all(flags)
|