import re def evaluate_fluency(llm_response: str) -> dict: issues = [] if len(llm_response.split()) < 5: issues.append("Response too short") if len(llm_response.split()) > 500: issues.append("Response too long") # Find the first alphabetic character, skipping list markers like # "1.", "-", "*", "•", whitespace, etc. A numbered/bulleted answer # starting with "1. The system..." is still properly capitalized — # it just isn't index [0]. first_letter_match = re.search(r'[a-zA-Z]', llm_response) if first_letter_match and not first_letter_match.group().isupper(): issues.append("Does not start with capital letter") # Strip trailing whitespace before checking the last character, and # allow a closing bullet/list item that ends in a letter followed by # nothing (common in list-formatted answers) without penalizing it # as harshly -- but still flag genuinely unpunctuated prose. stripped = llm_response.rstrip() if stripped and stripped[-1] not in [".", "?", "!", ":"]: issues.append("Does not end with proper punctuation") words = llm_response.lower().split() for i in range(len(words) - 1): if words[i] == words[i + 1]: issues.append("contains repeated consecutive words") break if len(issues) == 0: verdict = "Fluent" elif len(issues) <= 2: verdict = "Partially Fluent" else: verdict = "Not Fluent" return { "issues": issues, "verdict": verdict }