Buckets:
| """Deterministic answer extraction and binary task scoring.""" | |
| from __future__ import annotations | |
| import re | |
| from collections import Counter | |
| COLORS = "black|blue|brown|green|orange|purple|red|white|yellow" | |
| SHAPES = ( | |
| "capsule|cone|cube|cylinder|dodecahedron|donut|half-sphere|icosahedron|" | |
| "octahedron|pyramid|sphere|square|teardrop" | |
| ) | |
| INVENTORY_ITEM = re.compile( | |
| rf"(\d+)\s+({COLORS})\s+({SHAPES})s?\b", re.IGNORECASE | |
| ) | |
| def normalize_text(text: str) -> str: | |
| return " ".join(re.sub(r"[^a-z0-9-]+", " ", text.lower()).split()) | |
| def extract_final_answer(text: str) -> str: | |
| text = text.split("<|im_end|>", 1)[0] | |
| matches = list( | |
| re.finditer( | |
| r"final\s+answer\s*:\s*(.*?)(?=<\|im_end\|>|$)", | |
| text, | |
| flags=re.IGNORECASE | re.DOTALL, | |
| ) | |
| ) | |
| return matches[-1].group(1).strip() if matches else text.strip() | |
| def parse_inventory(text: str) -> Counter: | |
| return Counter( | |
| (color.lower(), shape.lower(), int(count)) | |
| for count, color, shape in INVENTORY_ITEM.findall(text) | |
| ) | |
| def parse_choice(text: str) -> str | None: | |
| value = text.strip() | |
| if re.fullmatch(r"(?:[A-Ha-h]|\d+)[.)]?", value): | |
| return value.rstrip(".)").lower() | |
| patterns = ( | |
| r"(?:correct\s+)?answer\s*(?:is|:)?\s*(?:option\s*)?([A-Ha-h]|\d+)", | |
| r"option\s*([A-Ha-h]|\d+)", | |
| r"^\s*([A-Ha-h])\s*[.)]\s*\S+", | |
| ) | |
| for pattern in patterns: | |
| matches = re.findall(pattern, value, flags=re.IGNORECASE) | |
| if matches: | |
| return matches[-1].lower() | |
| return None | |
| def parse_boolean(text: str) -> str | None: | |
| matches = re.findall(r"\b(yes|no|true|false)\b", text, flags=re.IGNORECASE) | |
| if not matches: | |
| return None | |
| value = matches[-1].lower() | |
| return {"true": "yes", "false": "no"}.get(value, value) | |
| def score_answer(output: str, target: str) -> dict: | |
| candidate = extract_final_answer(output) | |
| target = target.strip() | |
| exact = normalize_text(candidate) == normalize_text(target) | |
| if exact: | |
| prediction = normalize_text(candidate) | |
| correct = True | |
| method = "normalized_exact" | |
| elif re.fullmatch(r"[-+]?\d+(?:\.\d+)?", target): | |
| explicit = re.findall( | |
| r"(?:final\s+answer|answer)\s*(?:is|:)?\s*([-+]?\d+(?:\.\d+)?)", | |
| candidate, | |
| flags=re.IGNORECASE, | |
| ) | |
| numbers = re.findall(r"[-+]?\d+(?:\.\d+)?", candidate) | |
| prediction = explicit[-1] if explicit else (numbers[-1] if numbers else None) | |
| correct = prediction is not None and float(prediction) == float(target) | |
| method = "numeric" | |
| else: | |
| target_inventory = parse_inventory(target) | |
| prediction_inventory = parse_inventory(candidate) | |
| target_choice = parse_choice(target) | |
| target_boolean = parse_boolean(target) | |
| if target_inventory: | |
| prediction = sorted(prediction_inventory.elements()) | |
| correct = prediction_inventory == target_inventory | |
| method = "inventory" | |
| elif target_choice is not None: | |
| prediction = parse_choice(candidate) | |
| correct = prediction == target_choice | |
| method = "choice" | |
| elif target_boolean is not None: | |
| prediction = parse_boolean(candidate) | |
| correct = prediction == target_boolean | |
| method = "boolean" | |
| else: | |
| prediction = normalize_text(candidate) | |
| correct = False | |
| method = "normalized_exact" | |
| return { | |
| "candidate": candidate, | |
| "target": target, | |
| "prediction": prediction, | |
| "correct": correct, | |
| "exact": exact, | |
| "method": method, | |
| } | |
| __all__ = [ | |
| "extract_final_answer", | |
| "normalize_text", | |
| "parse_boolean", | |
| "parse_choice", | |
| "parse_inventory", | |
| "score_answer", | |
| ] | |
Xet Storage Details
- Size:
- 3.85 kB
- Xet hash:
- c1d36a16c35b3ff78fcc7f1dbc0345c795bfbfe3b9e42902c9ca19aaaf9f192d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.