File size: 2,143 Bytes
6c511d6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import re
from urllib.parse import urlparse
from config import MAX_EVIDENCE_CHARS
AUDIO_VIDEO_EXTENSIONS = {".mp3", ".wav", ".m4a", ".flac", ".mp4", ".mov", ".webm"}
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
SPREADSHEET_EXTENSIONS = {".xlsx", ".xls"}
TEXT_EXTENSIONS = {".txt", ".csv", ".json", ".md"}
def truncate_text(text: str, limit: int = MAX_EVIDENCE_CHARS) -> str:
if len(text) <= limit:
return text
return text[:limit] + "\n\n[内容过长，已截断。]"
def normalize_answer(answer: str) -> str:
"""清理模型常见废话，只保留适合提交的最终答案字符串。"""
answer = answer.strip()
answer = re.sub(r"^```(?:text|markdown)?", "", answer, flags=re.IGNORECASE).strip()
answer = re.sub(r"```$", "", answer).strip()
final_markers = [
"final answer:",
"final:",
"answer:",
"submitted_answer:",
"最终答案：",
"答案：",
]
lower_answer = answer.lower()
for marker in final_markers:
marker_index = lower_answer.rfind(marker)
if marker_index != -1:
answer = answer[marker_index + len(marker):].strip()
break
if "\n" in answer:
non_empty_lines = [line.strip() for line in answer.splitlines() if line.strip()]
if non_empty_lines:
answer = non_empty_lines[-1]
return answer.strip().strip('"').strip("'")
def normalize_for_compare(value: str) -> str:
value = normalize_answer(value).lower()
value = re.sub(r"\s*,\s*", ", ", value)
value = re.sub(r"\s+", " ", value)
return value.strip()
def extract_urls(text: str) -> list[str]:
urls = re.findall(r"https?://[^\s<>\"]+", text)
return [url.rstrip(".,);]\"'") for url in urls]
def is_youtube_url(url: str) -> bool:
host = urlparse(url).netloc.lower()
return "youtube.com" in host or "youtu.be" in host
def read_plain_file(file_path, limit: int = 12000) -> str:
try:
return truncate_text(file_path.read_text(encoding="utf-8", errors="replace"), limit)
except Exception as exc:
return f"读取文件失败：{exc}"
|