Spaces:
Paused
Paused
Update mcq_validator.py
Browse files- mcq_validator.py +31 -6
mcq_validator.py
CHANGED
|
@@ -87,11 +87,13 @@ class MCQValidator:
|
|
| 87 |
|
| 88 |
return {}
|
| 89 |
|
| 90 |
-
def _extract_correct_answer(self, text: str) -> str | None:
|
| 91 |
"""
|
| 92 |
Extract correct_answer letter, handling formats:
|
| 93 |
- "correct_answer": "D"
|
| 94 |
- "correct_answer": "choice b"
|
|
|
|
|
|
|
| 95 |
Always returns uppercase A-D or None.
|
| 96 |
"""
|
| 97 |
letter_map = {'a': 'A', 'b': 'B', 'c': 'C', 'd': 'D'}
|
|
@@ -106,6 +108,30 @@ class MCQValidator:
|
|
| 106 |
if m:
|
| 107 |
return letter_map.get(m.group(1).lower())
|
| 108 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
return None
|
| 110 |
|
| 111 |
def parse_raw_output(self, text: str) -> dict:
|
|
@@ -125,8 +151,8 @@ class MCQValidator:
|
|
| 125 |
if len(choices) < 2:
|
| 126 |
raise ValueError(f"Could not extract choices. Found: {choices}")
|
| 127 |
|
| 128 |
-
# Extract correct answer
|
| 129 |
-
correct = self._extract_correct_answer(text)
|
| 130 |
if not correct:
|
| 131 |
raise ValueError("Could not extract 'correct_answer' from model output.")
|
| 132 |
|
|
@@ -169,9 +195,8 @@ class MCQValidator:
|
|
| 169 |
|
| 170 |
try:
|
| 171 |
parsed = json.loads(text)
|
| 172 |
-
# Ensure choices values are ints
|
| 173 |
if "choices" in parsed:
|
| 174 |
-
parsed["choices"] = {k:
|
| 175 |
return parsed
|
| 176 |
except json.JSONDecodeError:
|
| 177 |
pass
|
|
@@ -183,7 +208,7 @@ class MCQValidator:
|
|
| 183 |
if start != -1 and end > start:
|
| 184 |
parsed = json.loads(text[start:end])
|
| 185 |
if "choices" in parsed:
|
| 186 |
-
parsed["choices"] = {k:
|
| 187 |
return parsed
|
| 188 |
except Exception:
|
| 189 |
pass
|
|
|
|
| 87 |
|
| 88 |
return {}
|
| 89 |
|
| 90 |
+
def _extract_correct_answer(self, text: str, choices: dict = None) -> str | None:
|
| 91 |
"""
|
| 92 |
Extract correct_answer letter, handling formats:
|
| 93 |
- "correct_answer": "D"
|
| 94 |
- "correct_answer": "choice b"
|
| 95 |
+
- Fallback: evaluate arithmetic expression in question and match to choices
|
| 96 |
+
(handles truncated output where correct_answer field is missing)
|
| 97 |
Always returns uppercase A-D or None.
|
| 98 |
"""
|
| 99 |
letter_map = {'a': 'A', 'b': 'B', 'c': 'C', 'd': 'D'}
|
|
|
|
| 108 |
if m:
|
| 109 |
return letter_map.get(m.group(1).lower())
|
| 110 |
|
| 111 |
+
# Format 3: Fallback for truncated output — correct_answer field never appeared.
|
| 112 |
+
# Evaluate the arithmetic expression in the question and match to choices.
|
| 113 |
+
if choices:
|
| 114 |
+
q_match = re.search(r'"question":\s*"(.*?)"', text)
|
| 115 |
+
if q_match:
|
| 116 |
+
question_text = q_match.group(1)
|
| 117 |
+
expr = re.search(r'(\d+)\s*([\+\-\*\/])\s*(\d+)', question_text)
|
| 118 |
+
if expr:
|
| 119 |
+
a, op, b = expr.groups()
|
| 120 |
+
computed = None
|
| 121 |
+
if op == '+':
|
| 122 |
+
computed = int(a) + int(b)
|
| 123 |
+
elif op == '-':
|
| 124 |
+
computed = int(a) - int(b)
|
| 125 |
+
elif op == '*':
|
| 126 |
+
computed = int(a) * int(b)
|
| 127 |
+
elif op == '/' and int(b) != 0:
|
| 128 |
+
computed = round(int(a) / int(b), 4)
|
| 129 |
+
|
| 130 |
+
if computed is not None:
|
| 131 |
+
for letter, val in choices.items():
|
| 132 |
+
if round(float(val), 4) == round(float(computed), 4):
|
| 133 |
+
return letter
|
| 134 |
+
|
| 135 |
return None
|
| 136 |
|
| 137 |
def parse_raw_output(self, text: str) -> dict:
|
|
|
|
| 151 |
if len(choices) < 2:
|
| 152 |
raise ValueError(f"Could not extract choices. Found: {choices}")
|
| 153 |
|
| 154 |
+
# Extract correct answer — pass choices for arithmetic fallback
|
| 155 |
+
correct = self._extract_correct_answer(text, choices=choices)
|
| 156 |
if not correct:
|
| 157 |
raise ValueError("Could not extract 'correct_answer' from model output.")
|
| 158 |
|
|
|
|
| 195 |
|
| 196 |
try:
|
| 197 |
parsed = json.loads(text)
|
|
|
|
| 198 |
if "choices" in parsed:
|
| 199 |
+
parsed["choices"] = {k: round(float(v), 4) for k, v in parsed["choices"].items()}
|
| 200 |
return parsed
|
| 201 |
except json.JSONDecodeError:
|
| 202 |
pass
|
|
|
|
| 208 |
if start != -1 and end > start:
|
| 209 |
parsed = json.loads(text[start:end])
|
| 210 |
if "choices" in parsed:
|
| 211 |
+
parsed["choices"] = {k: round(float(v), 4) for k, v in parsed["choices"].items()}
|
| 212 |
return parsed
|
| 213 |
except Exception:
|
| 214 |
pass
|