tools/common.py

import re
from urllib.parse import urlparse

from config import MAX_EVIDENCE_CHARS


AUDIO_VIDEO_EXTENSIONS = {".mp3", ".wav", ".m4a", ".flac", ".mp4", ".mov", ".webm"}
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
SPREADSHEET_EXTENSIONS = {".xlsx", ".xls"}
TEXT_EXTENSIONS = {".txt", ".csv", ".json", ".md"}


def truncate_text(text: str, limit: int = MAX_EVIDENCE_CHARS) -> str:
    if len(text) <= limit:
        return text
    return text[:limit] + "\n\n[内容过长，已截断。]"


def normalize_answer(answer: str) -> str:
    """清理模型常见废话，只保留适合提交的最终答案字符串。"""
    answer = answer.strip()
    answer = re.sub(r"^```(?:text|markdown)?", "", answer, flags=re.IGNORECASE).strip()
    answer = re.sub(r"```$", "", answer).strip()

    final_markers = [
        "final answer:",
        "final:",
        "answer:",
        "submitted_answer:",
        "最终答案：",
        "答案：",
    ]
    lower_answer = answer.lower()
    for marker in final_markers:
        marker_index = lower_answer.rfind(marker)
        if marker_index != -1:
            answer = answer[marker_index + len(marker):].strip()
            break

    if "\n" in answer:
        non_empty_lines = [line.strip() for line in answer.splitlines() if line.strip()]
        if non_empty_lines:
            answer = non_empty_lines[-1]

    return answer.strip().strip('"').strip("'")


def normalize_for_compare(value: str) -> str:
    value = normalize_answer(value).lower()
    value = re.sub(r"\s*,\s*", ", ", value)
    value = re.sub(r"\s+", " ", value)
    return value.strip()


def extract_urls(text: str) -> list[str]:
    urls = re.findall(r"https?://[^\s<>\"]+", text)
    return [url.rstrip(".,);]\"'") for url in urls]


def is_youtube_url(url: str) -> bool:
    host = urlparse(url).netloc.lower()
    return "youtube.com" in host or "youtu.be" in host


def read_plain_file(file_path, limit: int = 12000) -> str:
    try:
        return truncate_text(file_path.read_text(encoding="utf-8", errors="replace"), limit)
    except Exception as exc:
        return f"读取文件失败：{exc}"