Blame for src/evaluators/fluency_evaluator.py

import re

def evaluate_fluency(llm_response: str) -> dict:
    issues = []

    if len(llm_response.split()) < 5:
        issues.append("Response too short")

    if len(llm_response.split()) > 500:
        issues.append("Response too long")

    # Find the first alphabetic character, skipping list markers like
    # "1.", "-", "*", "•", whitespace, etc. A numbered/bulleted answer
    # starting with "1. The system..." is still properly capitalized —
    # it just isn't index [0].
    first_letter_match = re.search(r'[a-zA-Z]', llm_response)
    if first_letter_match and not first_letter_match.group().isupper():
        issues.append("Does not start with capital letter")

    # Strip trailing whitespace before checking the last character, and
    # allow a closing bullet/list item that ends in a letter followed by
    # nothing (common in list-formatted answers) without penalizing it
    # as harshly -- but still flag genuinely unpunctuated prose.
    stripped = llm_response.rstrip()
    if stripped and stripped[-1] not in [".", "?", "!", ":"]:
        issues.append("Does not end with proper punctuation")

    words = llm_response.lower().split()
    for i in range(len(words) - 1):
        if words[i] == words[i + 1]:
            issues.append("contains repeated consecutive words")
            break

    if len(issues) == 0:
        verdict = "Fluent"
    elif len(issues) <= 2:
        verdict = "Partially Fluent"
    else:
        verdict = "Not Fluent"

    return {
        "issues": issues,
        "verdict": verdict
    }