DarkSting commited on
Commit
ccd7774
·
verified ·
1 Parent(s): 008b507

Update mcq_validator.py

Browse files
Files changed (1) hide show
  1. mcq_validator.py +31 -6
mcq_validator.py CHANGED
@@ -87,11 +87,13 @@ class MCQValidator:
87
 
88
  return {}
89
 
90
- def _extract_correct_answer(self, text: str) -> str | None:
91
  """
92
  Extract correct_answer letter, handling formats:
93
  - "correct_answer": "D"
94
  - "correct_answer": "choice b"
 
 
95
  Always returns uppercase A-D or None.
96
  """
97
  letter_map = {'a': 'A', 'b': 'B', 'c': 'C', 'd': 'D'}
@@ -106,6 +108,30 @@ class MCQValidator:
106
  if m:
107
  return letter_map.get(m.group(1).lower())
108
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
  return None
110
 
111
  def parse_raw_output(self, text: str) -> dict:
@@ -125,8 +151,8 @@ class MCQValidator:
125
  if len(choices) < 2:
126
  raise ValueError(f"Could not extract choices. Found: {choices}")
127
 
128
- # Extract correct answer using multi-strategy parser
129
- correct = self._extract_correct_answer(text)
130
  if not correct:
131
  raise ValueError("Could not extract 'correct_answer' from model output.")
132
 
@@ -169,9 +195,8 @@ class MCQValidator:
169
 
170
  try:
171
  parsed = json.loads(text)
172
- # Ensure choices values are ints
173
  if "choices" in parsed:
174
- parsed["choices"] = {k: int(float(v)) for k, v in parsed["choices"].items()}
175
  return parsed
176
  except json.JSONDecodeError:
177
  pass
@@ -183,7 +208,7 @@ class MCQValidator:
183
  if start != -1 and end > start:
184
  parsed = json.loads(text[start:end])
185
  if "choices" in parsed:
186
- parsed["choices"] = {k: int(float(v)) for k, v in parsed["choices"].items()}
187
  return parsed
188
  except Exception:
189
  pass
 
87
 
88
  return {}
89
 
90
+ def _extract_correct_answer(self, text: str, choices: dict = None) -> str | None:
91
  """
92
  Extract correct_answer letter, handling formats:
93
  - "correct_answer": "D"
94
  - "correct_answer": "choice b"
95
+ - Fallback: evaluate arithmetic expression in question and match to choices
96
+ (handles truncated output where correct_answer field is missing)
97
  Always returns uppercase A-D or None.
98
  """
99
  letter_map = {'a': 'A', 'b': 'B', 'c': 'C', 'd': 'D'}
 
108
  if m:
109
  return letter_map.get(m.group(1).lower())
110
 
111
+ # Format 3: Fallback for truncated output — correct_answer field never appeared.
112
+ # Evaluate the arithmetic expression in the question and match to choices.
113
+ if choices:
114
+ q_match = re.search(r'"question":\s*"(.*?)"', text)
115
+ if q_match:
116
+ question_text = q_match.group(1)
117
+ expr = re.search(r'(\d+)\s*([\+\-\*\/])\s*(\d+)', question_text)
118
+ if expr:
119
+ a, op, b = expr.groups()
120
+ computed = None
121
+ if op == '+':
122
+ computed = int(a) + int(b)
123
+ elif op == '-':
124
+ computed = int(a) - int(b)
125
+ elif op == '*':
126
+ computed = int(a) * int(b)
127
+ elif op == '/' and int(b) != 0:
128
+ computed = round(int(a) / int(b), 4)
129
+
130
+ if computed is not None:
131
+ for letter, val in choices.items():
132
+ if round(float(val), 4) == round(float(computed), 4):
133
+ return letter
134
+
135
  return None
136
 
137
  def parse_raw_output(self, text: str) -> dict:
 
151
  if len(choices) < 2:
152
  raise ValueError(f"Could not extract choices. Found: {choices}")
153
 
154
+ # Extract correct answer — pass choices for arithmetic fallback
155
+ correct = self._extract_correct_answer(text, choices=choices)
156
  if not correct:
157
  raise ValueError("Could not extract 'correct_answer' from model output.")
158
 
 
195
 
196
  try:
197
  parsed = json.loads(text)
 
198
  if "choices" in parsed:
199
+ parsed["choices"] = {k: round(float(v), 4) for k, v in parsed["choices"].items()}
200
  return parsed
201
  except json.JSONDecodeError:
202
  pass
 
208
  if start != -1 and end > start:
209
  parsed = json.loads(text[start:end])
210
  if "choices" in parsed:
211
+ parsed["choices"] = {k: round(float(v), 4) for k, v in parsed["choices"].items()}
212
  return parsed
213
  except Exception:
214
  pass