File size: 2,174 Bytes
d5147ba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | from typing import Dict, Any, Optional
from dataclasses import fields
import importlib
import inspect
def load_model_class(identifier: str, prefix: str = "models."):
module_path, class_name = identifier.split('@')
# Import the module
module = importlib.import_module(prefix + module_path)
cls = getattr(module, class_name)
return cls
def get_model_source_path(identifier: str, prefix: str = "models."):
module_path, class_name = identifier.split('@')
module = importlib.import_module(prefix + module_path)
return inspect.getsourcefile(module)
def dict_view(obj: Any) -> Dict[str, Any]:
"""Returns a dictionary view of a dataclass, without copy. dataclasses.asdict(...) will make a deepcopy of every field."""
return {k.name: getattr(obj, k.name) for k in fields(obj)}
def last_boxed_only_string(string: str) -> Optional[str]:
idx = string.rfind("\\boxed")
if idx < 0:
idx = string.rfind("\\fbox")
if idx < 0:
return None
i = idx
left_brace_idx = None
right_brace_idx = None
num_left_braces_open = 0
while i < len(string):
if string[i] == "{":
num_left_braces_open += 1
if left_brace_idx is None:
left_brace_idx = i
elif string[i] == "}":
num_left_braces_open -= 1
if num_left_braces_open == 0:
right_brace_idx = i
break
i += 1
if left_brace_idx is None or right_brace_idx is None:
return None
return string[left_brace_idx + 1: right_brace_idx].strip()
def compute_benchmark_micro_macro_avg(stats: dict[str, dict[str, int | float]]):
results = {}
total_acc = total_invalid = 0.0
for k, v in stats.items():
results[f"n_{k}"] = v["n"]
results[f"acc_{k}"] = acc = v["correct"] / max(1, v["n"])
results[f"invalid_{k}"] = invalid = v["invalid"] / max(1, v["n"])
total_acc += acc
total_invalid += invalid
results["n"] = len(stats)
results["acc"] = total_acc / max(1, len(stats))
results["invalid"] = total_invalid / max(1, len(stats))
return results
|