xtc-backend / _chat_attach_selftest.py
a3216's picture
sync from GitHub c486f76: fix(chat): 带附件时附件正文只发一遍 + 修 CESU-8 导致的整段乱码与附件丢失
e09ed53 verified
Raw History Blame Contribute Delete
13.1 kB
# -*- coding: utf-8 -*-
"""附件链路自检:multipart 解析 / CESU-8 乱码还原 / 附件正文只发一遍。
复现的是 2026-10-31 的线上日志(ds2api-logs-20261031-062636.json)里的症状:
1. 带附件发消息 → **同一份附件正文被发了两遍**(prompt_tokens 6523 里有一半是重复的),
而且其中一遍整段是乱码 ``好é®é¢``;
2. 附件正文被错标成 ``[file_names]``,messages 里还夹着一段 ``["f_xxx.md"]`` 垃圾;
3. 含 emoji 的附件正文干脆丢了(```files_text`` 记成空串)。
三个根因(下面逐条断言):
* 前端把附件正文内嵌进 ``input``,同时又 multipart 上传 → 后端 normalize_chat_body
再拼一条独立 user 消息;
* 小天才平台按 **UTF-16 code unit** 拼 multipart(CESU-8),emoji 变成非法 UTF-8 字节:
- 文本字段:Starlette ``_user_safe_decode`` 只要有一个非法字节就把**整段**退回
latin-1 → 整段中文乱码;
- 文件字段:``bytes.decode("utf-8")`` 抛异常 → 被 except 记成空串 → 正文静默丢失;
* ``_parse_multipart`` 用 ``key.startswith("file_")`` 判断「文件正文」,
于是 ``file_names`` 这个普通表单字段(JSON.stringify(names))被当成了附件正文。
用法:python _chat_attach_selftest.py (在 xtc-backend-hf/ 目录下)
"""
import asyncio
import json
import sys
# 终端默认是 GBK,断言里会打印 emoji(🤔),不重设会 UnicodeEncodeError
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
from starlette.requests import Request as StarletteRequest
from app.api._common import normalize_chat_body
from app.api.xtc import _parse_multipart
from app.utils.textfix import (
decode_utf8_lenient,
looks_like_latin1_mojibake,
merge_surrogate_pairs,
repair_text,
)
fails = 0
def line(ok, label, extra=""):
global fails
if not ok:
fails += 1
print(" %s %-56s %s" % ("OK " if ok else "FAIL", label, extra))
# ===== 造「小天才平台真实会发出来的字节」 =====
EMOJI = "\U0001F914" # 🤔 U+1F914 → 代理对 D83E DD14
def cesu8(text):
"""按 UTF-16 code unit 逐个编码(CESU-8)——平台 request.upload 就是这么干的。
``🤔`` → ``ED A0 BE ED B4 94``(非法 UTF-8),而合法 UTF-8 是 ``F0 9F A4 94``。
"""
out = bytearray()
for ch in text:
cp = ord(ch)
if cp < 0x80:
out.append(cp)
elif cp < 0x800:
out.append(0xC0 | (cp >> 6))
out.append(0x80 | (cp & 0x3F))
else:
# 注意:代理项(≥0xD800)也按 3 字节写,这正是 CESU-8 的错法
out.append(0xE0 | (cp >> 12))
out.append(0x80 | ((cp >> 6) & 0x3F))
out.append(0x80 | (cp & 0x3F))
return bytes(out)
def utf16_units(text):
"""把 str 拆成 UTF-16 code unit 序列(把代理对拆成两个孤立代理项)。"""
raw = text.encode("utf-16-le")
units = []
for i in range(0, len(raw), 2):
units.append(raw[i] | (raw[i + 1] << 8))
return "".join(chr(u) for u in units)
BOUNDARY = "----XtcSelftestBoundary7d91"
def multipart(parts):
"""parts: [(name, bytes_value, filename_or_None, content_type_or_None)]"""
chunks = []
for name, value, filename, ctype in parts:
head = '--%s\r\nContent-Disposition: form-data; name="%s"' % (BOUNDARY, name)
if filename is not None:
head += '; filename="%s"' % filename
head += "\r\n"
if ctype:
head += "Content-Type: %s\r\n" % ctype
head += "\r\n"
chunks.append(head.encode("utf-8") + value + b"\r\n")
chunks.append(("--%s--\r\n" % BOUNDARY).encode("utf-8"))
return b"".join(chunks)
def make_request(body, content_type="multipart/form-data; boundary=" + BOUNDARY):
scope = {
"type": "http",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"method": "POST",
"scheme": "http",
"path": "/v1/xtc/chat/pseudo/start",
"raw_path": b"/v1/xtc/chat/pseudo/start",
"query_string": b"",
"root_path": "",
"headers": [
(b"content-type", content_type.encode("latin-1")),
(b"content-length", str(len(body)).encode("latin-1")),
],
"server": ("testserver", 80),
"client": ("testclient", 49221),
}
state = {"sent": False}
async def receive():
if state["sent"]:
return {"type": "http.disconnect"}
state["sent"] = True
return {"type": "http.request", "body": body, "more_body": False}
return StarletteRequest(scope, receive)
def parse(body):
return asyncio.run(_parse_multipart(make_request(body)))
def msg_text(msg):
"""messages[i] 的正文:content 可能是 str,也可能是 [{type:text,text:...}] 数组。"""
content = (msg or {}).get("content")
if isinstance(content, str):
return content
if isinstance(content, list):
return "\n".join(
str(p.get("text") or "") for p in content if isinstance(p, dict)
)
return ""
# ===== 1. textfix 单元 =====
print("=== textfix:CESU-8 与 UTF-8→latin-1 乱码还原 ===")
raw_ok = "好问题🤔".encode("utf-8")
raw_cesu = cesu8(utf16_units("好问题🤔"))
line(raw_ok != raw_cesu, "CESU-8 字节确实不同于合法 UTF-8", raw_cesu.hex())
line(decode_utf8_lenient(raw_ok) == "好问题🤔", "合法 UTF-8 原样解出")
line(decode_utf8_lenient(raw_cesu) == "好问题🤔", "CESU-8 也能解出同一个字符串",
repr(decode_utf8_lenient(raw_cesu)))
line("\ufffd" not in decode_utf8_lenient(raw_cesu), "CESU-8 解出后没有替换字符")
# Starlette 的兜底:整段按 latin-1 解 → 乱码
mojibake = raw_ok.decode("latin-1")
line(mojibake != "好问题🤔", "latin-1 兜底会把整段变成乱码", mojibake)
line(looks_like_latin1_mojibake(mojibake), "识别为 latin-1 乱码")
line(repair_text(mojibake) == "好问题🤔", "repair_text 还原成原文", repr(repair_text(mojibake)))
line(repair_text("好问题🤔") == "好问题🤔", "已经是好的文本 → 原样返回(幂等)")
line(repair_text("café") == "café", "真正的 latin-1 文本不会被改坏")
line(repair_text("hello world") == "hello world", "纯 ASCII 原样返回")
line(repair_text(utf16_units("a🤔b")) == merge_surrogate_pairs(utf16_units("a🤔b")),
"孤立代理项 → 合并/替换(不抛异常)")
# ===== 2. multipart:file_names 不能当附件正文,标签必须 1:1 =====
print("\n=== _parse_multipart:file_names 字段 / 标签错位 / 乱码 ===")
FILE_BODY = "白景的外貌设定…\n~~(我是不是应该放到后面性格🤔)~~\n结尾。"
USER_TEXT = "好问题,话说白城的外貌到底具体是什么样子呢?"
body = multipart([
("access_key", b"sk-test", None, None),
("model", b"deepseek-v4-flash", None, None),
# 平台把 input 字段也按 code unit 编码 → 这里的字节是 CESU-8(含 emoji 时非法)
("input", cesu8(utf16_units(USER_TEXT + "\n" + FILE_BODY + "🤔")), None, None),
# 前端发的 file_names 是普通表单字段(JSON 字符串),绝不能被当成附件正文
("file_names", json.dumps(["白景.md"], ensure_ascii=False).encode("utf-8"), None, None),
# 文本附件的字节同样是 CESU-8(含 emoji)
("files", cesu8(utf16_units(FILE_BODY)), "f_4adce40a_347773.md", "text/markdown"),
])
out = parse(body)
print(" 解析结果:input=%r" % (out.get("input", "")[:40] + "…"))
print(" files=%r" % [f[:30] + "…" for f in out.get("files", [])])
print(" file_names=%r" % (out.get("file_names"),))
line("\ufffd" not in out.get("input", ""), "input 没有替换字符")
line(out.get("input", "").startswith(USER_TEXT), "input 的乱码被还原(能读到用户那句话)")
line("外貌设定" in out.get("input", ""), "input 里的中文能正常读")
line(len(out.get("files", [])) == 1, "只有 1 个附件正文(file_names 没被当正文)",
"got=%d" % len(out.get("files", [])))
line("外貌设定" in (out.get("files") or [""])[0], "附件正文完整(含 emoji 的文件不再丢内容)")
line("\ufffd" not in (out.get("files") or [""])[0], "附件正文无替换字符")
line(out.get("file_names") == ["白景.md"], "标签用前端发来的别名,且与 files 一一对应",
repr(out.get("file_names")))
line(not any(f.strip().startswith('["') for f in out.get("files", [])),
"files 里没有混进 [\"f_xxx.md\"] 这种 JSON 垃圾")
# 两个附件:标签必须按顺序一一对应(旧实现是 names + filenames 直接 concat → 错位)
body2 = multipart([
("input", "看看这两份设定".encode("utf-8"), None, None),
("file_names", json.dumps(["白景.md", "白辰.md"], ensure_ascii=False).encode("utf-8"), None, None),
("files", "第一份正文".encode("utf-8"), "f_4adce40a.md", "text/markdown"),
("files", "第二份正文".encode("utf-8"), "f_ec8a8819.md", "text/markdown"),
])
out2 = parse(body2)
line(out2.get("file_names") == ["白景.md", "白辰.md"], "两个附件 → 标签顺序正确",
repr(out2.get("file_names")))
msgs2, _, _ = normalize_chat_body(out2)
joined2 = "\n".join(msg_text(m) for m in msgs2)
line("[白景.md]\n第一份正文" in joined2, "第一份正文本标成 白景.md")
line("[白辰.md]\n第二份正文" in joined2, "第二份正文本标成 白辰.md")
# file_names 缺失 → 退回上传文件名(不能崩、不能错位)
body3 = multipart([
("input", "只有文件".encode("utf-8"), None, None),
("files", "正文A".encode("utf-8"), "f_aaa.md", "text/markdown"),
])
out3 = parse(body3)
line(out3.get("file_names") == ["f_aaa.md"], "file_names 缺失 → 退回上传文件名",
repr(out3.get("file_names")))
# 二进制附件(PDF/ZIP)不能被容错解码成一大堆 U+FFFD 塞进 prompt——
# 老语义是「记空串 → 忽略」,这个语义必须保住(否则 5MB 的 PDF 会变成 5MB 的 �)
out4 = parse(multipart([
("input", "看看这个 pdf".encode("utf-8"), None, None),
("file_names", json.dumps(["手册.pdf"]).encode("utf-8"), None, None),
("files", b"%PDF-1.7\n\x00\x01\x02\xff\xfe binary junk", "f_pdf.pdf", "application/pdf"),
]))
line(out4.get("files") == [""], "二进制附件 → 空串(不给 prompt 灌 U+FFFD)",
repr(out4.get("files")))
msgs4, _, _ = normalize_chat_body(out4)
line(len(msgs4) == 1, "二进制附件 → 不追加附件消息", "got=%d" % len(msgs4))
# 合法 UTF-8(4 字节 emoji)的文本附件当然也要正常
out5 = parse(multipart([
("input", "看附件".encode("utf-8"), None, None),
("files", "备注:情绪🤔".encode("utf-8"), "f_ok.md", "text/markdown"),
]))
line(out5.get("files") == ["备注:情绪🤔"], "合法 UTF-8(4 字节 emoji)文本附件正常",
repr(out5.get("files")))
# ===== 3. 端到端:附件正文只发一遍 =====
print("\n=== normalize_chat_body:同一份附件只能发一遍 ===")
# 旧前端:正文同时内嵌进 input(附件内容:【文件:x】…)+ multipart 上传
legacy_input = (USER_TEXT + "\n(附件:白景.md)\n附件内容:\n【文件:白景.md】\n" + FILE_BODY)
legacy_body = {
"input": legacy_input,
"files": [FILE_BODY],
"file_names": ["白景.md"],
}
msgs, _, _ = normalize_chat_body(legacy_body)
line(len(msgs) == 1, "旧前端(内嵌+上传)→ 不再追加第二条 user 消息", "got=%d" % len(msgs))
line(msg_text(msgs[0]).count("外貌设定") == 1, "正文只出现一次",
"count=%d" % msg_text(msgs[0]).count("外貌设定"))
# 新前端:只上传,不内嵌 → 后端正常拼一条附件消息
new_body = {
"input": USER_TEXT + "\n(附件:白景.md)",
"files": [FILE_BODY],
"file_names": ["白景.md"],
}
msgs, _, _ = normalize_chat_body(new_body)
line(len(msgs) == 2, "新前端(只上传)→ 追加 1 条附件消息", "got=%d" % len(msgs))
line(msg_text(msgs[1]) == "[白景.md]\n" + FILE_BODY, "附件消息格式 = [标签]\\n正文",
repr(msg_text(msgs[1])[:24]))
joined = "\n".join(msg_text(m) for m in msgs)
line(joined.count("外貌设定") == 1, "新前端下正文也只出现一次",
"count=%d" % joined.count("外貌设定"))
# 纯聊天(无附件):行为不变
msgs, model, provider = normalize_chat_body({"input": "你好", "model": "m", "provider": "p"})
line(len(msgs) == 1 and model == "m" and provider == "p", "无附件时行为不变")
line(msg_text(msgs[0]) == "你好", "无附件 → 单条 user 消息,正文原样")
# 短内容不做去重(避免误判)
msgs, _, _ = normalize_chat_body({"input": "用 hi 打个招呼", "files": ["hi"], "file_names": ["a.md"]})
line(len(msgs) == 2, "过短的正文不走去重(防误判)", "got=%d" % len(msgs))
print("\n" + ("全部通过" if fails == 0 else "%d 处失败" % fails))
sys.exit(1 if fails else 0)