# -*- coding: utf-8 -*- """附件链路自检:multipart 解析 / CESU-8 乱码还原 / 附件正文只发一遍。 复现的是 2026-10-31 的线上日志(ds2api-logs-20261031-062636.json)里的症状: 1. 带附件发消息 → **同一份附件正文被发了两遍**(prompt_tokens 6523 里有一半是重复的), 而且其中一遍整段是乱码 ``好é®é¢``; 2. 附件正文被错标成 ``[file_names]``,messages 里还夹着一段 ``["f_xxx.md"]`` 垃圾; 3. 含 emoji 的附件正文干脆丢了(```files_text`` 记成空串)。 三个根因(下面逐条断言): * 前端把附件正文内嵌进 ``input``,同时又 multipart 上传 → 后端 normalize_chat_body 再拼一条独立 user 消息; * 小天才平台按 **UTF-16 code unit** 拼 multipart(CESU-8),emoji 变成非法 UTF-8 字节: - 文本字段:Starlette ``_user_safe_decode`` 只要有一个非法字节就把**整段**退回 latin-1 → 整段中文乱码; - 文件字段:``bytes.decode("utf-8")`` 抛异常 → 被 except 记成空串 → 正文静默丢失; * ``_parse_multipart`` 用 ``key.startswith("file_")`` 判断「文件正文」, 于是 ``file_names`` 这个普通表单字段(JSON.stringify(names))被当成了附件正文。 用法:python _chat_attach_selftest.py (在 xtc-backend-hf/ 目录下) """ import asyncio import json import sys # 终端默认是 GBK,断言里会打印 emoji(🤔),不重设会 UnicodeEncodeError if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace") from starlette.requests import Request as StarletteRequest from app.api._common import normalize_chat_body from app.api.xtc import _parse_multipart from app.utils.textfix import ( decode_utf8_lenient, looks_like_latin1_mojibake, merge_surrogate_pairs, repair_text, ) fails = 0 def line(ok, label, extra=""): global fails if not ok: fails += 1 print(" %s %-56s %s" % ("OK " if ok else "FAIL", label, extra)) # ===== 造「小天才平台真实会发出来的字节」 ===== EMOJI = "\U0001F914" # 🤔 U+1F914 → 代理对 D83E DD14 def cesu8(text): """按 UTF-16 code unit 逐个编码(CESU-8)——平台 request.upload 就是这么干的。 ``🤔`` → ``ED A0 BE ED B4 94``(非法 UTF-8),而合法 UTF-8 是 ``F0 9F A4 94``。 """ out = bytearray() for ch in text: cp = ord(ch) if cp < 0x80: out.append(cp) elif cp < 0x800: out.append(0xC0 | (cp >> 6)) out.append(0x80 | (cp & 0x3F)) else: # 注意:代理项(≥0xD800)也按 3 字节写,这正是 CESU-8 的错法 out.append(0xE0 | (cp >> 12)) out.append(0x80 | ((cp >> 6) & 0x3F)) out.append(0x80 | (cp & 0x3F)) return bytes(out) def utf16_units(text): """把 str 拆成 UTF-16 code unit 序列(把代理对拆成两个孤立代理项)。""" raw = text.encode("utf-16-le") units = [] for i in range(0, len(raw), 2): units.append(raw[i] | (raw[i + 1] << 8)) return "".join(chr(u) for u in units) BOUNDARY = "----XtcSelftestBoundary7d91" def multipart(parts): """parts: [(name, bytes_value, filename_or_None, content_type_or_None)]""" chunks = [] for name, value, filename, ctype in parts: head = '--%s\r\nContent-Disposition: form-data; name="%s"' % (BOUNDARY, name) if filename is not None: head += '; filename="%s"' % filename head += "\r\n" if ctype: head += "Content-Type: %s\r\n" % ctype head += "\r\n" chunks.append(head.encode("utf-8") + value + b"\r\n") chunks.append(("--%s--\r\n" % BOUNDARY).encode("utf-8")) return b"".join(chunks) def make_request(body, content_type="multipart/form-data; boundary=" + BOUNDARY): scope = { "type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "POST", "scheme": "http", "path": "/v1/xtc/chat/pseudo/start", "raw_path": b"/v1/xtc/chat/pseudo/start", "query_string": b"", "root_path": "", "headers": [ (b"content-type", content_type.encode("latin-1")), (b"content-length", str(len(body)).encode("latin-1")), ], "server": ("testserver", 80), "client": ("testclient", 49221), } state = {"sent": False} async def receive(): if state["sent"]: return {"type": "http.disconnect"} state["sent"] = True return {"type": "http.request", "body": body, "more_body": False} return StarletteRequest(scope, receive) def parse(body): return asyncio.run(_parse_multipart(make_request(body))) def msg_text(msg): """messages[i] 的正文:content 可能是 str,也可能是 [{type:text,text:...}] 数组。""" content = (msg or {}).get("content") if isinstance(content, str): return content if isinstance(content, list): return "\n".join( str(p.get("text") or "") for p in content if isinstance(p, dict) ) return "" # ===== 1. textfix 单元 ===== print("=== textfix:CESU-8 与 UTF-8→latin-1 乱码还原 ===") raw_ok = "好问题🤔".encode("utf-8") raw_cesu = cesu8(utf16_units("好问题🤔")) line(raw_ok != raw_cesu, "CESU-8 字节确实不同于合法 UTF-8", raw_cesu.hex()) line(decode_utf8_lenient(raw_ok) == "好问题🤔", "合法 UTF-8 原样解出") line(decode_utf8_lenient(raw_cesu) == "好问题🤔", "CESU-8 也能解出同一个字符串", repr(decode_utf8_lenient(raw_cesu))) line("\ufffd" not in decode_utf8_lenient(raw_cesu), "CESU-8 解出后没有替换字符") # Starlette 的兜底:整段按 latin-1 解 → 乱码 mojibake = raw_ok.decode("latin-1") line(mojibake != "好问题🤔", "latin-1 兜底会把整段变成乱码", mojibake) line(looks_like_latin1_mojibake(mojibake), "识别为 latin-1 乱码") line(repair_text(mojibake) == "好问题🤔", "repair_text 还原成原文", repr(repair_text(mojibake))) line(repair_text("好问题🤔") == "好问题🤔", "已经是好的文本 → 原样返回(幂等)") line(repair_text("café") == "café", "真正的 latin-1 文本不会被改坏") line(repair_text("hello world") == "hello world", "纯 ASCII 原样返回") line(repair_text(utf16_units("a🤔b")) == merge_surrogate_pairs(utf16_units("a🤔b")), "孤立代理项 → 合并/替换(不抛异常)") # ===== 2. multipart:file_names 不能当附件正文,标签必须 1:1 ===== print("\n=== _parse_multipart:file_names 字段 / 标签错位 / 乱码 ===") FILE_BODY = "白景的外貌设定…\n~~(我是不是应该放到后面性格🤔)~~\n结尾。" USER_TEXT = "好问题,话说白城的外貌到底具体是什么样子呢?" body = multipart([ ("access_key", b"sk-test", None, None), ("model", b"deepseek-v4-flash", None, None), # 平台把 input 字段也按 code unit 编码 → 这里的字节是 CESU-8(含 emoji 时非法) ("input", cesu8(utf16_units(USER_TEXT + "\n" + FILE_BODY + "🤔")), None, None), # 前端发的 file_names 是普通表单字段(JSON 字符串),绝不能被当成附件正文 ("file_names", json.dumps(["白景.md"], ensure_ascii=False).encode("utf-8"), None, None), # 文本附件的字节同样是 CESU-8(含 emoji) ("files", cesu8(utf16_units(FILE_BODY)), "f_4adce40a_347773.md", "text/markdown"), ]) out = parse(body) print(" 解析结果:input=%r" % (out.get("input", "")[:40] + "…")) print(" files=%r" % [f[:30] + "…" for f in out.get("files", [])]) print(" file_names=%r" % (out.get("file_names"),)) line("\ufffd" not in out.get("input", ""), "input 没有替换字符") line(out.get("input", "").startswith(USER_TEXT), "input 的乱码被还原(能读到用户那句话)") line("外貌设定" in out.get("input", ""), "input 里的中文能正常读") line(len(out.get("files", [])) == 1, "只有 1 个附件正文(file_names 没被当正文)", "got=%d" % len(out.get("files", []))) line("外貌设定" in (out.get("files") or [""])[0], "附件正文完整(含 emoji 的文件不再丢内容)") line("\ufffd" not in (out.get("files") or [""])[0], "附件正文无替换字符") line(out.get("file_names") == ["白景.md"], "标签用前端发来的别名,且与 files 一一对应", repr(out.get("file_names"))) line(not any(f.strip().startswith('["') for f in out.get("files", [])), "files 里没有混进 [\"f_xxx.md\"] 这种 JSON 垃圾") # 两个附件:标签必须按顺序一一对应(旧实现是 names + filenames 直接 concat → 错位) body2 = multipart([ ("input", "看看这两份设定".encode("utf-8"), None, None), ("file_names", json.dumps(["白景.md", "白辰.md"], ensure_ascii=False).encode("utf-8"), None, None), ("files", "第一份正文".encode("utf-8"), "f_4adce40a.md", "text/markdown"), ("files", "第二份正文".encode("utf-8"), "f_ec8a8819.md", "text/markdown"), ]) out2 = parse(body2) line(out2.get("file_names") == ["白景.md", "白辰.md"], "两个附件 → 标签顺序正确", repr(out2.get("file_names"))) msgs2, _, _ = normalize_chat_body(out2) joined2 = "\n".join(msg_text(m) for m in msgs2) line("[白景.md]\n第一份正文" in joined2, "第一份正文本标成 白景.md") line("[白辰.md]\n第二份正文" in joined2, "第二份正文本标成 白辰.md") # file_names 缺失 → 退回上传文件名(不能崩、不能错位) body3 = multipart([ ("input", "只有文件".encode("utf-8"), None, None), ("files", "正文A".encode("utf-8"), "f_aaa.md", "text/markdown"), ]) out3 = parse(body3) line(out3.get("file_names") == ["f_aaa.md"], "file_names 缺失 → 退回上传文件名", repr(out3.get("file_names"))) # 二进制附件(PDF/ZIP)不能被容错解码成一大堆 U+FFFD 塞进 prompt—— # 老语义是「记空串 → 忽略」,这个语义必须保住(否则 5MB 的 PDF 会变成 5MB 的 �) out4 = parse(multipart([ ("input", "看看这个 pdf".encode("utf-8"), None, None), ("file_names", json.dumps(["手册.pdf"]).encode("utf-8"), None, None), ("files", b"%PDF-1.7\n\x00\x01\x02\xff\xfe binary junk", "f_pdf.pdf", "application/pdf"), ])) line(out4.get("files") == [""], "二进制附件 → 空串(不给 prompt 灌 U+FFFD)", repr(out4.get("files"))) msgs4, _, _ = normalize_chat_body(out4) line(len(msgs4) == 1, "二进制附件 → 不追加附件消息", "got=%d" % len(msgs4)) # 合法 UTF-8(4 字节 emoji)的文本附件当然也要正常 out5 = parse(multipart([ ("input", "看附件".encode("utf-8"), None, None), ("files", "备注:情绪🤔".encode("utf-8"), "f_ok.md", "text/markdown"), ])) line(out5.get("files") == ["备注:情绪🤔"], "合法 UTF-8(4 字节 emoji)文本附件正常", repr(out5.get("files"))) # ===== 3. 端到端:附件正文只发一遍 ===== print("\n=== normalize_chat_body:同一份附件只能发一遍 ===") # 旧前端:正文同时内嵌进 input(附件内容:【文件:x】…)+ multipart 上传 legacy_input = (USER_TEXT + "\n(附件:白景.md)\n附件内容:\n【文件:白景.md】\n" + FILE_BODY) legacy_body = { "input": legacy_input, "files": [FILE_BODY], "file_names": ["白景.md"], } msgs, _, _ = normalize_chat_body(legacy_body) line(len(msgs) == 1, "旧前端(内嵌+上传)→ 不再追加第二条 user 消息", "got=%d" % len(msgs)) line(msg_text(msgs[0]).count("外貌设定") == 1, "正文只出现一次", "count=%d" % msg_text(msgs[0]).count("外貌设定")) # 新前端:只上传,不内嵌 → 后端正常拼一条附件消息 new_body = { "input": USER_TEXT + "\n(附件:白景.md)", "files": [FILE_BODY], "file_names": ["白景.md"], } msgs, _, _ = normalize_chat_body(new_body) line(len(msgs) == 2, "新前端(只上传)→ 追加 1 条附件消息", "got=%d" % len(msgs)) line(msg_text(msgs[1]) == "[白景.md]\n" + FILE_BODY, "附件消息格式 = [标签]\\n正文", repr(msg_text(msgs[1])[:24])) joined = "\n".join(msg_text(m) for m in msgs) line(joined.count("外貌设定") == 1, "新前端下正文也只出现一次", "count=%d" % joined.count("外貌设定")) # 纯聊天(无附件):行为不变 msgs, model, provider = normalize_chat_body({"input": "你好", "model": "m", "provider": "p"}) line(len(msgs) == 1 and model == "m" and provider == "p", "无附件时行为不变") line(msg_text(msgs[0]) == "你好", "无附件 → 单条 user 消息,正文原样") # 短内容不做去重(避免误判) msgs, _, _ = normalize_chat_body({"input": "用 hi 打个招呼", "files": ["hi"], "file_names": ["a.md"]}) line(len(msgs) == 2, "过短的正文不走去重(防误判)", "got=%d" % len(msgs)) print("\n" + ("全部通过" if fails == 0 else "%d 处失败" % fails)) sys.exit(1 if fails else 0)