File size: 24,929 Bytes
cd7e7a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 | """
email_thread_parser.py
Server-side port of the JS thread parser in static/js/emailLibrary.js.
Walks an email body (HTML or plain text) and returns a tree of reply turns
that the client can render directly without re-parsing.
Mirrors the rules from talon (mailgun) and email-reply-parser:
- Multilingual "On <date>, <name> wrote:" attribution lines (20+ locales)
- Outlook-style "From: ... Sent: ... Subject:" header blocks
- "----- Original Message -----" delimiters
- <blockquote> nesting (HTML)
- "> " prefix nesting (plain text)
Returns a list of dicts:
[
{"level": 0, "body_html": "...", "meta": null},
{"level": 1, "body_html": "...", "meta": "Alice <a@x> · May 5"},
{"level": 2, "body_html": "...", "meta": "Bob <b@y> · May 4"},
...
]
where level 0 is the current reply, increasing levels = deeper in the chain.
"""
from __future__ import annotations
import html as _html
import re
from typing import Any
# Bump whenever the parser's output shape or splitting rules change. The
# cache layer wraps turns as {"v": THREAD_PARSER_VERSION, "turns": [...]}
# and treats anything with a different version as stale.
THREAD_PARSER_VERSION = 6
# ── Locale tables (same as static/js/emailLibrary.js _TALON_*) ──
_WROTE = (
r"(?:wrote|écrit|escribió|scrisse|schrieb|skrev|schreef|napisał|написал|"
r"napsal|написа|έγραψε|katselivat|napisao|написав|napisała|napisali|"
r"hat geschrieben|kirjoitti|написала|escreveu)"
)
_FROM = (
r"(?:From|Från|Von|De|Da|От|Od|Van|差出人|发件人|寄件人|Lähettäjä|"
r"Avsender|Pošiljatelj|Frá)"
)
_SENT = (
r"(?:Sent|Skickat|Gesendet|Envoy[ée]|Inviato|Enviado|Verzonden|Отправлено|"
r"Wysłane|Date|送信日時|发送时间|寄件日期|Sendt|Lähetetty|Tarih|Datum|Data)"
)
_SUBJ = (
r"(?:Subject|Ämne|Betreff|Objet|Oggetto|Asunto|Onderwerp|Тема|Temat|"
r"件名|主题|主旨|Emne|Aihe|Konu)"
)
_TO = r"(?:To|Till|An|À|A|Voor|Para|Naar|Кому|Do|宛先|收件人|Komu)"
_CCBCC = r"(?:Cc|Bcc|Kopie|Skrytá kopie|Копия)"
_HDR_KEYS = rf"(?:{_FROM}|{_SENT}|{_SUBJ}|{_TO}|{_CCBCC}|Importance|Priority)"
_ORIG_RE = re.compile(
r"(?:^|\n)[\s>]*[-_=]{3,}\s*(?:Original\s+Message|Forwarded\s+message|"
r"Ursprüngliche\s+Nachricht|"
r"Mensaje\s+original|Messaggio\s+originale|Message\s+d['’]origine|"
r"Oorspronkelijk\s+bericht|Original\s+meddelande|原文|原始邮件|転送)"
r"\s*[-_=]{3,}",
re.IGNORECASE,
)
_WROTE_LINE_RE = re.compile(rf"^\s*On\s.+?\s{_WROTE}\s*:\s*$", re.IGNORECASE | re.MULTILINE)
# CJK-style attribution lines — Japanese Gmail / Yahoo Mail JP / etc.
# Examples (all valid):
# 2026年5月11日(月) 21:28 <alice@example.com>:
# 2026年5月11日 21:28 alice@example.com:
# 2026/05/11 21:28 <alice@example.com> のメッセージ:
# 2026年5月11日(月) 21:28に Alice Smith <alice@example.com> のメッセージ:
# 2026年5月11日 21:28、alice@example.com さんは書きました:
# Alice さんは 2026/05/11 21:28 に書きました:
_CJK_ATTRIB_LINE_RE = re.compile(
r"^\s*(?:"
# date(weekday) time <email>: (Gmail JP default)
r"\d{4}[年/.-]\d{1,2}[月/.-]\d{1,2}日?(?:\s*[\(\(].+?[\)\)])?"
r"\s+\d{1,2}:\d{2}(?:\s*[APAP][MM])?"
r"(?:に|、|,)?\s*(?:.+?\s+)?[<<]?[\w.+\-]+@[\w.\-]+\.[A-Za-z]{2,}[>>]?"
r"\s*(?:のメッセージ|さんは(?:書|お?書き)きました|wrote)?\s*[::]\s*$"
r"|"
# 何々さんは 2026/05/11 21:28 に書きました:
r".+?(?:さん|様)\s*(?:は|が)\s+\d{4}[年/.-]\d{1,2}[月/.-]\d{1,2}日?"
r"(?:\s*[\(\(].+?[\)\)])?\s+\d{1,2}:\d{2}\s*(?:に)?\s*(?:書|お?書き)きました\s*[::]\s*$"
r"|"
# Chinese "XXX 写道:" preceded by a date or address
r".+?\s*写道\s*[::]\s*$"
r"|"
# Korean "님이 작성:"
r".+?\s*님이\s*작성(?:한\s*내용)?\s*[::]\s*$"
r")",
re.MULTILINE,
)
_OUTLOOK_HEADER_RE = re.compile(
rf"{_FROM}\s*:\s*[^\n]+\s*\n\s*(?:.+\n)?{_SENT}\s*:\s*[^\n]+\s*\n",
re.IGNORECASE,
)
# Stop the From/Date captures at the next header key so they don't swallow
# the whole header block when whitespace has been normalised.
_FROM_STOP = rf"\s+(?:{_FROM}|{_SENT}|{_SUBJ}|{_TO}|{_CCBCC}|Importance|Priority)\s*:"
_DATE_STOP = rf"\s+(?:{_FROM}|{_SUBJ}|{_TO}|{_CCBCC}|Importance|Priority)\s*:"
_QUOTE_META_FROM = re.compile(
rf"{_FROM}\s*:\s*(.+?)(?:(?={_FROM_STOP})|$)",
re.IGNORECASE | re.DOTALL,
)
_QUOTE_META_DATE = re.compile(
rf"{_SENT}\s*:\s*(.+?)(?:(?={_DATE_STOP})|$)",
re.IGNORECASE | re.DOTALL,
)
# Greedy date capture so multi-comma dates ("Thu, May 7, 2026, 11:33 AM,")
# don't collapse to just the day. We let the comma + lazy author match
# back off to the LAST comma before "wrote:".
_GMAIL_ATTRIB = re.compile(
rf"On\s+(.+),\s+([^,]+?)\s+{_WROTE}\s*:",
re.IGNORECASE | re.DOTALL,
)
def _extract_quote_meta(text_or_html: str) -> str | None:
"""Pull a '<sender> · <date>' chip from a quoted block. Preserves
angle-bracketed email addresses (`<foo@bar.com>`) so callers can
identify the sender for chat-bubble alignment."""
if not text_or_html:
return None
plain = re.sub(r"<style[\s\S]*?</style>", " ", text_or_html, flags=re.IGNORECASE)
# Strip HTML tags, but keep <foo@bar> patterns since they carry the
# sender's address that downstream consumers (bubble renderer) need.
plain = re.sub(r"<(?![^@>\s]+@[^@>\s]+>)[^>]+>", " ", plain)
plain = re.sub(r" ", " ", plain, flags=re.IGNORECASE)
plain = plain.replace("&", "&").replace("<", "<").replace(">", ">").replace(""", '"')
plain = re.sub(r"\s+", " ", plain).strip()[:1500]
f = _QUOTE_META_FROM.search(plain)
d = _QUOTE_META_DATE.search(plain)
if f and d:
return f"{f.group(1).strip()} · {d.group(1).strip()[:80]}"
g = _GMAIL_ATTRIB.search(plain)
if g:
date, who = g.group(1).strip(), g.group(2).strip()
return f"{who} · {date}"
# CJK attribution: "YYYY年MM月DD日(曜) HH:MM <email>:"
cjk = re.search(
r"(\d{4}[年/.-]\d{1,2}[月/.-]\d{1,2}日?(?:\s*[\(\(][^\)\)]+?[\)\)])?\s+\d{1,2}:\d{2}(?:\s*[APAP][MM])?)"
r"\s*(?:に|、|,)?\s*"
r"(?:(.+?)\s+)?" # optional display name
r"[<<]?([\w.+\-]+@[\w.\-]+\.[A-Za-z]{2,})[>>]?",
plain,
)
if cjk:
date = cjk.group(1).strip()
who = (cjk.group(2) or cjk.group(3) or '').strip()
return f"{who} · {date}" if who else date
if f:
return f.group(1).strip()
if d:
return d.group(1).strip()
return None
# ── Plaintext path ──
# Outlook sometimes renders a one-line "conversation summary header" at
# the very top of a reply when the recipient's mail client copies it from
# the reading pane (whitespace gets squashed). Looks like:
# "alice@example.comThursday, May 7, 2026 3:06 PM To: housekeeping <...> Subject: ..."
# or just:
# "alice@example.comThursday, May 7, 2026 3:06 PM"
# Same info already lives in the envelope, so strip it.
_MASHED_HDR_RE = re.compile(
r"^\s*[\w.+\-]+@[\w.\-]+\.[A-Za-z]{2,}" # email address
r"\s*"
r"(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)[a-z]*,?\s+" # day name
r"\S+\s+\d+,?\s*\d{4}\s+\d{1,2}:\d{2}" # date + time
r"(?:\s*[AP]M)?" # optional AM/PM
rf"(?:\s+{_TO}\s*:\s*[^\n]+(?:\s+{_SUBJ}\s*:\s*[^\n]*)?)?" # optional To:/Subject:
r"\s*(?:\n|$)", # end of line
re.IGNORECASE,
)
def _strip_mashed_header(text: str) -> str:
if not text:
return text
m = _MASHED_HDR_RE.match(text)
if not m:
return text
rest = text[m.end():]
# Skip any blank lines that immediately follow the strip.
rest = re.sub(r"^\s*\n+", "", rest)
return rest
def _normalize_body(text: str) -> str:
"""Strip noise that mail clients (mostly Outlook) inject into the
plaintext body but that adds no signal — duplicate <mailto:> link
decorations, bracketed-URL annotations, repeated blank lines, and
the mashed conversation-header at the top."""
if not text:
return text
text = _strip_mashed_header(text)
# Outlook appends `<mailto:foo@bar>` after every email address it
# finds, and `<https://...>` after every URL. Both are duplicate
# noise — they show the same target as the visible text. Drop them.
text = re.sub(r"<mailto:[^<>\s]*>", "", text, flags=re.IGNORECASE)
text = re.sub(r"<https?://[^<>\s]*>", "", text, flags=re.IGNORECASE)
# Trim trailing whitespace (incl. NBSP / form-feed / tab) so blank
# lines that mail clients fill with non-breaking spaces still count
# as blank for the collapse step below.
text = re.sub(r"[^\S\n]+(\n|$)", r"\1", text)
# Collapse 3+ consecutive newlines (vertical-space soup) into 2.
text = re.sub(r"\n{3,}", "\n\n", text)
return text
def _outlook_header_block_end(stripped: list[str], levels: list[int], start: int) -> int:
"""If lines[start..N] form an Outlook From/Sent/To/Subject header block
at the same base level, return N (exclusive end). Otherwise return start.
Requires a From: line followed within 5 lines by a Sent:/Date: line."""
if start >= len(stripped):
return start
base = levels[start]
first = stripped[start].strip()
if not re.match(rf"^{_FROM}\s*:\s*\S", first, re.IGNORECASE):
return start
# Look ahead for the matching Sent:/Date: line at the same base level.
found_sent = False
j = start + 1
while j < len(stripped) and j < start + 6 and levels[j] == base:
nl = stripped[j].strip()
if not nl:
j += 1
continue
if re.match(rf"^{_SENT}\s*:", nl, re.IGNORECASE):
found_sent = True
break
if not re.match(rf"^{_HDR_KEYS}\s*:", nl, re.IGNORECASE):
return start # something other than a header key — abort
j += 1
if not found_sent:
return start
# Consume header-key lines until we hit a non-header line OR a blank line.
j = start + 1
while j < len(stripped) and levels[j] == base:
nl = stripped[j].strip()
if not nl:
j += 1
break
if re.match(rf"^{_HDR_KEYS}\s*:", nl, re.IGNORECASE):
j += 1
continue
break
return j
def _parse_plaintext(text: str) -> list[dict[str, Any]] | None:
"""Walk `>` quote prefix levels + inline attribution markers at any
level. Each attribution event AND each `>`-level increment counts as
one conversation step, with one important exception: an attribution
marker IMMEDIATELY followed by a deeper `>` block is the same event
as that `>` increase (the classic Gmail "On X wrote:\\n> quoted"
pattern) and contributes only one step.
Returns a flat list of {level, body_html, meta} or None when nothing
quoted is detected."""
if not text or len(text) > 200_000:
return None
text = _normalize_body(text)
lines = text.splitlines()
base_levels: list[int] = []
stripped_lines: list[str] = []
for line in lines:
m = re.match(r"^((?:>\s?)+)", line)
n = line[: m.end()].count(">") if m else 0
base_levels.append(n)
stripped_lines.append(re.sub(r"^(?:>\s?)+", "", line) if n > 0 else line)
has_quotes = any(l > 0 for l in base_levels)
has_attrib = bool(
_WROTE_LINE_RE.search(text) or _ORIG_RE.search(text)
or _OUTLOOK_HEADER_RE.search(text) or _CJK_ATTRIB_LINE_RE.search(text)
)
if not has_quotes and not has_attrib:
return None
turns: list[dict[str, Any]] = []
buf: list[str] = []
cur_level = 0
pending_meta: str | None = None
# depth_at_base[B] = the effective conversation depth recorded the
# last time we were at `>` base level B. Used to restore depth when
# the > nesting decreases (we hop back to a shallower base).
depth_at_base: dict[int, int] = {0: 0}
depth = 0
prev_base = 0
def lookahead_content_base(start_idx: int) -> int | None:
j = start_idx
while j < len(lines) and not stripped_lines[j].strip():
j += 1
return base_levels[j] if j < len(lines) else None
def flush() -> None:
# `buf` is only mutated via .clear() / .append() in the enclosing
# scope, never re-assigned, so it doesn't need `nonlocal`.
nonlocal pending_meta
if not buf:
return
body = "\n".join(buf).rstrip()
if body or cur_level > 0:
turns.append({
"level": cur_level,
"body_html": _escape_to_html(body),
"meta": pending_meta,
})
buf.clear()
pending_meta = None
i = 0
while i < len(lines):
base = base_levels[i]
stripped = stripped_lines[i]
# `>` base level change → flush current turn, then step depth.
if base > prev_base:
flush()
for b in range(prev_base + 1, base + 1):
depth += 1
depth_at_base[b] = depth
cur_level = depth
elif base < prev_base:
flush()
depth = depth_at_base.get(base, base)
for b in list(depth_at_base.keys()):
if b > base:
del depth_at_base[b]
cur_level = depth
prev_base = base
is_gmail = bool(re.match(rf"^\s*On\s.+?\s{_WROTE}\s*:\s*$", stripped, re.IGNORECASE))
is_cjk = bool(_CJK_ATTRIB_LINE_RE.match(stripped))
is_orig = bool(_ORIG_RE.search("\n" + stripped))
outlook_end = _outlook_header_block_end(stripped_lines, base_levels, i)
is_outlook = outlook_end > i
if is_gmail or is_cjk or is_orig or is_outlook:
# Collect the full attribution text for meta extraction.
attrib_end = outlook_end if is_outlook else (i + 1)
meta_text = "\n".join(stripped_lines[i:attrib_end])
# "-----Original Message-----" is almost always immediately
# followed by an Outlook From:/Sent: header — fold that into
# the SAME attribution event so we don't double-bump.
if is_orig:
j = attrib_end
while j < len(lines) and base_levels[j] == base and not stripped_lines[j].strip():
j += 1
if j < len(lines) and base_levels[j] == base:
oe2 = _outlook_header_block_end(stripped_lines, base_levels, j)
if oe2 > j:
meta_text = meta_text + "\n" + "\n".join(stripped_lines[j:oe2])
attrib_end = oe2
# If the next content line lives at a deeper > base, the
# upcoming `>` increase will be the depth step — suppress
# our own bump so we don't double up. Otherwise, this
# attribution IS the step.
next_base = lookahead_content_base(attrib_end)
flush()
if next_base is not None and next_base > base:
pending_meta = _extract_quote_meta(meta_text) or meta_text.strip().splitlines()[0]
else:
depth += 1
depth_at_base[base] = depth
cur_level = depth
pending_meta = _extract_quote_meta(meta_text) or meta_text.strip().splitlines()[0]
i = attrib_end
continue
buf.append(stripped)
i += 1
flush()
if not turns or (len(turns) == 1 and turns[0]["level"] == 0):
return None
return turns
def _escape_to_html(text: str) -> str:
"""Conservative plaintext → HTML: escape, then linkify URLs and convert
newlines to <br>."""
if not text:
return ""
out = _html.escape(text)
out = re.sub(
r"(https?://[^\s<>\"]+)",
lambda m: f'<a href="{m.group(1)}" target="_blank" rel="noopener">{m.group(1)}</a>',
out,
)
return out.replace("\n", "<br>")
# ── HTML path (BeautifulSoup) ──
def _is_quote_container(tag) -> bool:
"""Return True if a BeautifulSoup tag is a known quote-container element.
Covers Gmail (`gmail_quote`), Apple Mail (`type="cite"`), Yahoo
(`yahoo_quoted`), Outlook (`divRplyFwdMsg`, `OutlookMessageHeader`,
`gmail_attr` precedes a quote in some forwards), and the standard
`<blockquote>`."""
if tag is None:
return False
name = (getattr(tag, "name", None) or "").lower()
if name == "blockquote":
return True
cls = " ".join(tag.get("class") or []).lower() if hasattr(tag, "get") else ""
if "gmail_quote" in cls or "yahoo_quoted" in cls or "moz-cite-prefix" in cls:
return True
if "outlookmessageheader" in cls or "wordsection1" in cls:
return True
if (tag.get("id") if hasattr(tag, "get") else "") == "divRplyFwdMsg":
return True
typ = (tag.get("type") if hasattr(tag, "get") else "") or ""
if name == "div" and typ.lower() == "cite":
return True
return False
def _parse_html(html: str) -> list[dict[str, Any]] | None:
"""Walk top-level quote-container elements and recurse into nested ones.
Returns None if no quote markers are present. Recognises <blockquote>
plus the Gmail / Apple Mail / Yahoo / Outlook / Thunderbird wrappers
that don't use <blockquote>."""
if not html or len(html) > 200_000:
return None
try:
from bs4 import BeautifulSoup
except Exception:
return None # bs4 not installed → caller falls back to plaintext / client parse
try:
soup = BeautifulSoup(html, "html.parser")
except Exception:
return None
# Find all quote containers, then keep only the top-level ones (those
# whose nearest ancestor that's also a quote container is None).
all_quotes = [t for t in soup.find_all(True) if _is_quote_container(t)]
if not all_quotes:
return None
def has_quote_ancestor(t) -> bool:
p = t.parent
while p is not None:
if _is_quote_container(p):
return True
p = p.parent
return False
tops = [t for t in all_quotes if not has_quote_ancestor(t)]
if not tops:
return None
turns: list[dict[str, Any]] = []
# Collect the new-reply content from OUTSIDE the quote containers.
# Most replies are top-posted (head), but Japanese / formal emails are
# frequently bottom-posted (tail). Some users do both. We combine head
# and tail into a single level-0 turn so the new content always shows
# first, regardless of source-order position.
parent_children = list(tops[0].parent.children if tops[0].parent else [])
head_nodes = []
for sib in parent_children:
if sib is tops[0]:
break
head_nodes.append(sib)
# Tail = everything after the LAST top-level quote at this parent level
last_top = tops[-1]
tail_nodes = []
after_last = False
for sib in parent_children:
if sib is last_top:
after_last = True
continue
# Skip any other top-level quotes between (they get walked below)
if after_last and sib in tops:
continue
if after_last:
tail_nodes.append(sib)
def _strip_trailing_attribution(html_chunk: str) -> str:
text = re.sub(r"<[^>]+>", " ", html_chunk)
if not (_WROTE_LINE_RE.search(text) or _ORIG_RE.search(text) or _CJK_ATTRIB_LINE_RE.search(text)):
return html_chunk
html_chunk = re.sub(
rf"(?:<br\s*/?>|</p>|</div>|\n)?\s*On\s.+?\s{_WROTE}\s*:\s*(?:</[^>]+>)*\s*$",
"",
html_chunk,
flags=re.IGNORECASE | re.DOTALL,
)
html_chunk = re.sub(
r"(?:<br\s*/?>|</p>|</div>|\n)?\s*"
r"(?:\d{4}[年/.-]\d{1,2}[月/.-]\d{1,2}日?(?:\s*[\(\(][^\)\)]+?[\)\)])?"
r"\s+\d{1,2}:\d{2}(?:\s*[APAP][MM])?(?:に|、|,)?"
r"\s*(?:.+?\s+)?[<<]?[\w.+\-]+@[\w.\-]+\.[A-Za-z]{2,}[>>]?"
r"\s*(?:のメッセージ|さんは(?:書|お?書き)きました|wrote)?\s*[::]"
r"\s*(?:</[^>]+>)*\s*$",
"",
html_chunk,
flags=re.DOTALL,
)
return html_chunk
head_html = _strip_trailing_attribution("".join(str(n) for n in head_nodes))
tail_html = "".join(str(n) for n in tail_nodes)
# Stitch head + tail. Tail (bottom-posted reply) goes first because
# that's the most-recent / most-relevant content; head (which may just
# be empty or a forwarded preamble) follows.
parts = []
if tail_html.strip(): parts.append(tail_html.strip())
if head_html.strip(): parts.append(head_html.strip())
if parts:
turns.append({
"level": 0,
"body_html": "<br><br>".join(parts) if len(parts) > 1 else parts[0],
"meta": None,
})
def _walk(node, level: int):
meta_from_node = _extract_quote_meta(str(node))
# Recurse into nested quote containers inside this one, then strip
# them so the body of THIS turn doesn't include them.
nested = [t for t in node.find_all(True, recursive=True) if _is_quote_container(t)]
# Keep only direct-quote descendants (no other quote container between)
def has_quote_between(child, ancestor) -> bool:
p = child.parent
while p is not None and p is not ancestor:
if _is_quote_container(p):
return True
p = p.parent
return False
direct_nested = [n for n in nested if not has_quote_between(n, node)]
for n in list(direct_nested):
n.extract()
body_html = node.decode_contents()
# Collapse "wrapper-only" quote containers: if the only remaining
# content of this node (after pulling out nested quotes) is an
# attribution line, don't emit a separate turn for it. Instead,
# pass the attribution down as meta for the directly-nested child.
# Without this collapse, gmail_quote_container produces a phantom
# bubble that contains just the JP/EN attribution line.
body_text = re.sub(r"<[^>]+>", " ", body_html).strip()
body_text = _html.unescape(body_text)
body_text_collapsed = re.sub(r"\s+", " ", body_text).strip()
is_attrib_only = bool(body_text_collapsed) and (
_CJK_ATTRIB_LINE_RE.match(body_text_collapsed)
or re.match(rf"^\s*On\s.+?\s{_WROTE}\s*:\s*$", body_text_collapsed, re.IGNORECASE)
or _OUTLOOK_HEADER_RE.match(body_text_collapsed)
)
if is_attrib_only and len(direct_nested) == 1:
# Skip emitting this wrapper. Pass attribution as meta for child.
child_meta = meta_from_node or body_text_collapsed
# Recurse into child as the SAME level (replacing this wrapper)
_walk_with_meta(direct_nested[0], level, child_meta)
return
turns.append({"level": level, "body_html": body_html, "meta": meta_from_node})
for n in direct_nested:
_walk(n, level + 1)
def _walk_with_meta(node, level: int, forced_meta: str):
"""Variant that uses a passed-in meta when the node's own meta is empty."""
meta_from_node = _extract_quote_meta(str(node)) or forced_meta
nested = [t for t in node.find_all(True, recursive=True) if _is_quote_container(t)]
def has_quote_between(child, ancestor) -> bool:
p = child.parent
while p is not None and p is not ancestor:
if _is_quote_container(p):
return True
p = p.parent
return False
direct_nested = [n for n in nested if not has_quote_between(n, node)]
for n in list(direct_nested):
n.extract()
body_html = node.decode_contents()
turns.append({"level": level, "body_html": body_html, "meta": meta_from_node})
for n in direct_nested:
_walk(n, level + 1)
for bq in tops:
_walk(bq, 1)
if not turns:
return None
return turns
def parse_thread(body_html: str | None, body_text: str | None) -> list[dict[str, Any]] | None:
"""Public entry point. Prefer HTML when available, else plaintext.
Returns None if no quoted material found (caller renders flat)."""
if isinstance(body_html, str) and body_html:
out = _parse_html(body_html)
if out:
return out
if isinstance(body_text, str) and body_text:
return _parse_plaintext(body_text)
return None
|