Spaces:
Running
Running
Download _chat_attach_selftest.py from a3216/xtc-backend: direct link, hf CLI and curl.
- Browser
- Download file 13.1 kB
-
https://huggingface.co/spaces/a3216/xtc-backend/resolve/main/_chat_attach_selftest.py
- Command line
-
hf download hf://spaces/a3216/xtc-backend/_chat_attach_selftest.py
-
curl -L -o _chat_attach_selftest.py https://huggingface.co/spaces/a3216/xtc-backend/resolve/main/_chat_attach_selftest.py
13.1 kB
| # -*- coding: utf-8 -*- | |
| """附件链路自检:multipart 解析 / CESU-8 乱码还原 / 附件正文只发一遍。 | |
| 复现的是 2026-10-31 的线上日志(ds2api-logs-20261031-062636.json)里的症状: | |
| 1. 带附件发消息 → **同一份附件正文被发了两遍**(prompt_tokens 6523 里有一半是重复的), | |
| 而且其中一遍整段是乱码 ``好é®é¢``; | |
| 2. 附件正文被错标成 ``[file_names]``,messages 里还夹着一段 ``["f_xxx.md"]`` 垃圾; | |
| 3. 含 emoji 的附件正文干脆丢了(```files_text`` 记成空串)。 | |
| 三个根因(下面逐条断言): | |
| * 前端把附件正文内嵌进 ``input``,同时又 multipart 上传 → 后端 normalize_chat_body | |
| 再拼一条独立 user 消息; | |
| * 小天才平台按 **UTF-16 code unit** 拼 multipart(CESU-8),emoji 变成非法 UTF-8 字节: | |
| - 文本字段:Starlette ``_user_safe_decode`` 只要有一个非法字节就把**整段**退回 | |
| latin-1 → 整段中文乱码; | |
| - 文件字段:``bytes.decode("utf-8")`` 抛异常 → 被 except 记成空串 → 正文静默丢失; | |
| * ``_parse_multipart`` 用 ``key.startswith("file_")`` 判断「文件正文」, | |
| 于是 ``file_names`` 这个普通表单字段(JSON.stringify(names))被当成了附件正文。 | |
| 用法:python _chat_attach_selftest.py (在 xtc-backend-hf/ 目录下) | |
| """ | |
| import asyncio | |
| import json | |
| import sys | |
| # 终端默认是 GBK,断言里会打印 emoji(🤔),不重设会 UnicodeEncodeError | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace") | |
| from starlette.requests import Request as StarletteRequest | |
| from app.api._common import normalize_chat_body | |
| from app.api.xtc import _parse_multipart | |
| from app.utils.textfix import ( | |
| decode_utf8_lenient, | |
| looks_like_latin1_mojibake, | |
| merge_surrogate_pairs, | |
| repair_text, | |
| ) | |
| fails = 0 | |
| def line(ok, label, extra=""): | |
| global fails | |
| if not ok: | |
| fails += 1 | |
| print(" %s %-56s %s" % ("OK " if ok else "FAIL", label, extra)) | |
| # ===== 造「小天才平台真实会发出来的字节」 ===== | |
| EMOJI = "\U0001F914" # 🤔 U+1F914 → 代理对 D83E DD14 | |
| def cesu8(text): | |
| """按 UTF-16 code unit 逐个编码(CESU-8)——平台 request.upload 就是这么干的。 | |
| ``🤔`` → ``ED A0 BE ED B4 94``(非法 UTF-8),而合法 UTF-8 是 ``F0 9F A4 94``。 | |
| """ | |
| out = bytearray() | |
| for ch in text: | |
| cp = ord(ch) | |
| if cp < 0x80: | |
| out.append(cp) | |
| elif cp < 0x800: | |
| out.append(0xC0 | (cp >> 6)) | |
| out.append(0x80 | (cp & 0x3F)) | |
| else: | |
| # 注意:代理项(≥0xD800)也按 3 字节写,这正是 CESU-8 的错法 | |
| out.append(0xE0 | (cp >> 12)) | |
| out.append(0x80 | ((cp >> 6) & 0x3F)) | |
| out.append(0x80 | (cp & 0x3F)) | |
| return bytes(out) | |
| def utf16_units(text): | |
| """把 str 拆成 UTF-16 code unit 序列(把代理对拆成两个孤立代理项)。""" | |
| raw = text.encode("utf-16-le") | |
| units = [] | |
| for i in range(0, len(raw), 2): | |
| units.append(raw[i] | (raw[i + 1] << 8)) | |
| return "".join(chr(u) for u in units) | |
| BOUNDARY = "----XtcSelftestBoundary7d91" | |
| def multipart(parts): | |
| """parts: [(name, bytes_value, filename_or_None, content_type_or_None)]""" | |
| chunks = [] | |
| for name, value, filename, ctype in parts: | |
| head = '--%s\r\nContent-Disposition: form-data; name="%s"' % (BOUNDARY, name) | |
| if filename is not None: | |
| head += '; filename="%s"' % filename | |
| head += "\r\n" | |
| if ctype: | |
| head += "Content-Type: %s\r\n" % ctype | |
| head += "\r\n" | |
| chunks.append(head.encode("utf-8") + value + b"\r\n") | |
| chunks.append(("--%s--\r\n" % BOUNDARY).encode("utf-8")) | |
| return b"".join(chunks) | |
| def make_request(body, content_type="multipart/form-data; boundary=" + BOUNDARY): | |
| scope = { | |
| "type": "http", | |
| "asgi": {"version": "3.0"}, | |
| "http_version": "1.1", | |
| "method": "POST", | |
| "scheme": "http", | |
| "path": "/v1/xtc/chat/pseudo/start", | |
| "raw_path": b"/v1/xtc/chat/pseudo/start", | |
| "query_string": b"", | |
| "root_path": "", | |
| "headers": [ | |
| (b"content-type", content_type.encode("latin-1")), | |
| (b"content-length", str(len(body)).encode("latin-1")), | |
| ], | |
| "server": ("testserver", 80), | |
| "client": ("testclient", 49221), | |
| } | |
| state = {"sent": False} | |
| async def receive(): | |
| if state["sent"]: | |
| return {"type": "http.disconnect"} | |
| state["sent"] = True | |
| return {"type": "http.request", "body": body, "more_body": False} | |
| return StarletteRequest(scope, receive) | |
| def parse(body): | |
| return asyncio.run(_parse_multipart(make_request(body))) | |
| def msg_text(msg): | |
| """messages[i] 的正文:content 可能是 str,也可能是 [{type:text,text:...}] 数组。""" | |
| content = (msg or {}).get("content") | |
| if isinstance(content, str): | |
| return content | |
| if isinstance(content, list): | |
| return "\n".join( | |
| str(p.get("text") or "") for p in content if isinstance(p, dict) | |
| ) | |
| return "" | |
| # ===== 1. textfix 单元 ===== | |
| print("=== textfix:CESU-8 与 UTF-8→latin-1 乱码还原 ===") | |
| raw_ok = "好问题🤔".encode("utf-8") | |
| raw_cesu = cesu8(utf16_units("好问题🤔")) | |
| line(raw_ok != raw_cesu, "CESU-8 字节确实不同于合法 UTF-8", raw_cesu.hex()) | |
| line(decode_utf8_lenient(raw_ok) == "好问题🤔", "合法 UTF-8 原样解出") | |
| line(decode_utf8_lenient(raw_cesu) == "好问题🤔", "CESU-8 也能解出同一个字符串", | |
| repr(decode_utf8_lenient(raw_cesu))) | |
| line("\ufffd" not in decode_utf8_lenient(raw_cesu), "CESU-8 解出后没有替换字符") | |
| # Starlette 的兜底:整段按 latin-1 解 → 乱码 | |
| mojibake = raw_ok.decode("latin-1") | |
| line(mojibake != "好问题🤔", "latin-1 兜底会把整段变成乱码", mojibake) | |
| line(looks_like_latin1_mojibake(mojibake), "识别为 latin-1 乱码") | |
| line(repair_text(mojibake) == "好问题🤔", "repair_text 还原成原文", repr(repair_text(mojibake))) | |
| line(repair_text("好问题🤔") == "好问题🤔", "已经是好的文本 → 原样返回(幂等)") | |
| line(repair_text("café") == "café", "真正的 latin-1 文本不会被改坏") | |
| line(repair_text("hello world") == "hello world", "纯 ASCII 原样返回") | |
| line(repair_text(utf16_units("a🤔b")) == merge_surrogate_pairs(utf16_units("a🤔b")), | |
| "孤立代理项 → 合并/替换(不抛异常)") | |
| # ===== 2. multipart:file_names 不能当附件正文,标签必须 1:1 ===== | |
| print("\n=== _parse_multipart:file_names 字段 / 标签错位 / 乱码 ===") | |
| FILE_BODY = "白景的外貌设定…\n~~(我是不是应该放到后面性格🤔)~~\n结尾。" | |
| USER_TEXT = "好问题,话说白城的外貌到底具体是什么样子呢?" | |
| body = multipart([ | |
| ("access_key", b"sk-test", None, None), | |
| ("model", b"deepseek-v4-flash", None, None), | |
| # 平台把 input 字段也按 code unit 编码 → 这里的字节是 CESU-8(含 emoji 时非法) | |
| ("input", cesu8(utf16_units(USER_TEXT + "\n" + FILE_BODY + "🤔")), None, None), | |
| # 前端发的 file_names 是普通表单字段(JSON 字符串),绝不能被当成附件正文 | |
| ("file_names", json.dumps(["白景.md"], ensure_ascii=False).encode("utf-8"), None, None), | |
| # 文本附件的字节同样是 CESU-8(含 emoji) | |
| ("files", cesu8(utf16_units(FILE_BODY)), "f_4adce40a_347773.md", "text/markdown"), | |
| ]) | |
| out = parse(body) | |
| print(" 解析结果:input=%r" % (out.get("input", "")[:40] + "…")) | |
| print(" files=%r" % [f[:30] + "…" for f in out.get("files", [])]) | |
| print(" file_names=%r" % (out.get("file_names"),)) | |
| line("\ufffd" not in out.get("input", ""), "input 没有替换字符") | |
| line(out.get("input", "").startswith(USER_TEXT), "input 的乱码被还原(能读到用户那句话)") | |
| line("外貌设定" in out.get("input", ""), "input 里的中文能正常读") | |
| line(len(out.get("files", [])) == 1, "只有 1 个附件正文(file_names 没被当正文)", | |
| "got=%d" % len(out.get("files", []))) | |
| line("外貌设定" in (out.get("files") or [""])[0], "附件正文完整(含 emoji 的文件不再丢内容)") | |
| line("\ufffd" not in (out.get("files") or [""])[0], "附件正文无替换字符") | |
| line(out.get("file_names") == ["白景.md"], "标签用前端发来的别名,且与 files 一一对应", | |
| repr(out.get("file_names"))) | |
| line(not any(f.strip().startswith('["') for f in out.get("files", [])), | |
| "files 里没有混进 [\"f_xxx.md\"] 这种 JSON 垃圾") | |
| # 两个附件:标签必须按顺序一一对应(旧实现是 names + filenames 直接 concat → 错位) | |
| body2 = multipart([ | |
| ("input", "看看这两份设定".encode("utf-8"), None, None), | |
| ("file_names", json.dumps(["白景.md", "白辰.md"], ensure_ascii=False).encode("utf-8"), None, None), | |
| ("files", "第一份正文".encode("utf-8"), "f_4adce40a.md", "text/markdown"), | |
| ("files", "第二份正文".encode("utf-8"), "f_ec8a8819.md", "text/markdown"), | |
| ]) | |
| out2 = parse(body2) | |
| line(out2.get("file_names") == ["白景.md", "白辰.md"], "两个附件 → 标签顺序正确", | |
| repr(out2.get("file_names"))) | |
| msgs2, _, _ = normalize_chat_body(out2) | |
| joined2 = "\n".join(msg_text(m) for m in msgs2) | |
| line("[白景.md]\n第一份正文" in joined2, "第一份正文本标成 白景.md") | |
| line("[白辰.md]\n第二份正文" in joined2, "第二份正文本标成 白辰.md") | |
| # file_names 缺失 → 退回上传文件名(不能崩、不能错位) | |
| body3 = multipart([ | |
| ("input", "只有文件".encode("utf-8"), None, None), | |
| ("files", "正文A".encode("utf-8"), "f_aaa.md", "text/markdown"), | |
| ]) | |
| out3 = parse(body3) | |
| line(out3.get("file_names") == ["f_aaa.md"], "file_names 缺失 → 退回上传文件名", | |
| repr(out3.get("file_names"))) | |
| # 二进制附件(PDF/ZIP)不能被容错解码成一大堆 U+FFFD 塞进 prompt—— | |
| # 老语义是「记空串 → 忽略」,这个语义必须保住(否则 5MB 的 PDF 会变成 5MB 的 �) | |
| out4 = parse(multipart([ | |
| ("input", "看看这个 pdf".encode("utf-8"), None, None), | |
| ("file_names", json.dumps(["手册.pdf"]).encode("utf-8"), None, None), | |
| ("files", b"%PDF-1.7\n\x00\x01\x02\xff\xfe binary junk", "f_pdf.pdf", "application/pdf"), | |
| ])) | |
| line(out4.get("files") == [""], "二进制附件 → 空串(不给 prompt 灌 U+FFFD)", | |
| repr(out4.get("files"))) | |
| msgs4, _, _ = normalize_chat_body(out4) | |
| line(len(msgs4) == 1, "二进制附件 → 不追加附件消息", "got=%d" % len(msgs4)) | |
| # 合法 UTF-8(4 字节 emoji)的文本附件当然也要正常 | |
| out5 = parse(multipart([ | |
| ("input", "看附件".encode("utf-8"), None, None), | |
| ("files", "备注:情绪🤔".encode("utf-8"), "f_ok.md", "text/markdown"), | |
| ])) | |
| line(out5.get("files") == ["备注:情绪🤔"], "合法 UTF-8(4 字节 emoji)文本附件正常", | |
| repr(out5.get("files"))) | |
| # ===== 3. 端到端:附件正文只发一遍 ===== | |
| print("\n=== normalize_chat_body:同一份附件只能发一遍 ===") | |
| # 旧前端:正文同时内嵌进 input(附件内容:【文件:x】…)+ multipart 上传 | |
| legacy_input = (USER_TEXT + "\n(附件:白景.md)\n附件内容:\n【文件:白景.md】\n" + FILE_BODY) | |
| legacy_body = { | |
| "input": legacy_input, | |
| "files": [FILE_BODY], | |
| "file_names": ["白景.md"], | |
| } | |
| msgs, _, _ = normalize_chat_body(legacy_body) | |
| line(len(msgs) == 1, "旧前端(内嵌+上传)→ 不再追加第二条 user 消息", "got=%d" % len(msgs)) | |
| line(msg_text(msgs[0]).count("外貌设定") == 1, "正文只出现一次", | |
| "count=%d" % msg_text(msgs[0]).count("外貌设定")) | |
| # 新前端:只上传,不内嵌 → 后端正常拼一条附件消息 | |
| new_body = { | |
| "input": USER_TEXT + "\n(附件:白景.md)", | |
| "files": [FILE_BODY], | |
| "file_names": ["白景.md"], | |
| } | |
| msgs, _, _ = normalize_chat_body(new_body) | |
| line(len(msgs) == 2, "新前端(只上传)→ 追加 1 条附件消息", "got=%d" % len(msgs)) | |
| line(msg_text(msgs[1]) == "[白景.md]\n" + FILE_BODY, "附件消息格式 = [标签]\\n正文", | |
| repr(msg_text(msgs[1])[:24])) | |
| joined = "\n".join(msg_text(m) for m in msgs) | |
| line(joined.count("外貌设定") == 1, "新前端下正文也只出现一次", | |
| "count=%d" % joined.count("外貌设定")) | |
| # 纯聊天(无附件):行为不变 | |
| msgs, model, provider = normalize_chat_body({"input": "你好", "model": "m", "provider": "p"}) | |
| line(len(msgs) == 1 and model == "m" and provider == "p", "无附件时行为不变") | |
| line(msg_text(msgs[0]) == "你好", "无附件 → 单条 user 消息,正文原样") | |
| # 短内容不做去重(避免误判) | |
| msgs, _, _ = normalize_chat_body({"input": "用 hi 打个招呼", "files": ["hi"], "file_names": ["a.md"]}) | |
| line(len(msgs) == 2, "过短的正文不走去重(防误判)", "got=%d" % len(msgs)) | |
| print("\n" + ("全部通过" if fails == 0 else "%d 处失败" % fails)) | |
| sys.exit(1 if fails else 0) | |