File size: 593 Bytes
aad9f16 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | from __future__ import annotations
from opsd_utils.diagnostics import summarize_template_behavior
BEHAVIOR_KEYS = (
"full_cot_template",
"partial_cot_template",
"goal_without_answer",
"empty_cot_skeleton",
"malformed_answer_section",
)
def summarize_output_behavior_counts(texts: list[str]) -> dict[str, int]:
counts = {"total": len(texts), **{key: 0 for key in BEHAVIOR_KEYS}}
for text in texts:
rates = summarize_template_behavior([text])
for key in BEHAVIOR_KEYS:
counts[key] += int(rates[f"{key}_rate"] > 0.5)
return counts
|