backend/engine/engine.py

import os
import sys
import re
import jieba.posseg as pseg

from backend.config import Config

NUM_RE = re.compile(r'^[0-9一二三四五六七八九十百千万几数多半两]+$')
PUNCT_SET = {',', '.', '!', '?', ';', ':', '"', '\'', '(', ')', '[', ']', '{', '}',
             '，', '。', '！', '？', '；', '：', '“', '”', '‘', '’', '（', '）', '【', '】', '《', '》', '、', '—', '～'}


class SimpleToken:
    def __init__(self, word, tag):
        self.word = word
        self.tag = tag
        self.translated = None
        
    @property
    def flag(self):
        return self.tag
        
    @flag.setter
    def flag(self, value):
        self.tag = value

class VietphraseEngine:
    def __init__(self, config=None):
        self.config = config or {}
        self.load_dictionaries()
        
        # Check translation mode
        self.translation_mode = self.config.get("translation", {}).get("mode", "advanced")
        if self.config.get("translation", {}).get("fast_mode", False):
            self.translation_mode = "fast"
            
        # Warm-up jieba
        import jieba
        try:
            # Fix segmenter splitting overlapping words like 重生于
            jieba.add_word("重生", tag="v")
            jieba.suggest_freq(("生", "于"), True)
            jieba.suggest_freq(("着", "重"), True)
            jieba.suggest_freq(("醉", "人"), True)
            # Tag grades as nouns instead of proper names (nr)
            jieba.add_word("高一", tag="n")
            jieba.add_word("高二", tag="n")
            jieba.add_word("高三", tag="n")
        except Exception as e:
            print("Error initializing custom word splits in Jieba:", e)

        # Always initialize both tokenizers to support dynamic mode switching
        self.jieba_tokenizer = jieba.dt
        self.pseg_dict = pseg.dt.word_tag_tab
        list(self.jieba_tokenizer.cut("暖洋洋"))
        list(pseg.cut("暖洋洋"))

    def load_dictionaries(self):
        paths = self.config.get("paths", {}).get("dictionaries", {})
        vp_path = paths.get("vietphrase", "")
        if not vp_path or not os.path.isabs(vp_path):
            vp_path = os.path.join(Config.ROOT_DIR, vp_path or "dictionaries/Vietphrase.txt")
            
        dict_dir = os.path.dirname(vp_path)
        
        # Check for encrypted .bin dictionaries first, then fallback to .txt
        def load_file_content(base_name):
            bin_file = os.path.join(dict_dir, base_name + ".bin")
            txt_file = os.path.join(dict_dir, base_name + ".txt")
            
            if os.path.exists(bin_file):
                # Decrypt XOR
                with open(bin_file, "rb") as f:
                    data = f.read()
                key_bytes = "quick_translator_secret_key_2026".encode("utf-8")
                key_len = len(key_bytes)
                repeated_key = (key_bytes * (len(data) // key_len + 1))[:len(data)]
                decrypted = bytes(a ^ b for a, b in zip(data, repeated_key))
                return decrypted.decode("utf-8")
            elif os.path.exists(txt_file):
                with open(txt_file, "r", encoding="utf-8") as f:
                    return f.read()
            return ""

        print("Loading dictionaries in VietphraseEngine...")
        self.char_dict = self.parse_dict_content(load_file_content("HanViet_CharDict"))
        
        # --- ADD HÁn Nôm FALLBACK ---
        import csv
        han_csv_path = os.path.join(dict_dir, "han_all_readings.csv")
        if os.path.exists(han_csv_path):
            try:
                with open(han_csv_path, 'r', encoding='utf-8') as f:
                    reader = csv.DictReader(f)
                    for row in reader:
                        char = row.get("Ký_tự", "").strip()
                        hv = row.get("Hán_Việt", "").strip()
                        if char and hv and char not in self.char_dict:
                            self.char_dict[char] = hv.replace("~", "")
                print("Loaded han_all_readings.csv as fallback for missing Chinese characters.")
            except Exception as e:
                print("Could not load han_all_readings.csv:", e)

        self.proper_names = self.parse_dict_content(load_file_content("Aligned_HanViet"), convert_to_simplified=True)
        
        vp_content = load_file_content("Vietphrase")
        self.vietphrase = self.parse_vietphrase_content(vp_content)
        print("Dictionaries loaded successfully.")

        # Build Tries for vietphrase and hanviet modes
        from .trie import Trie
        print("Building Tries for fast translation modes...")
        self.vietphrase_trie = Trie()
        # Insert proper names (priority 1)
        for k, v in self.proper_names.items():
            self.vietphrase_trie.insert(k, v, 1)
        # Insert Vietphrase (priority 2 - higher)
        for k, v in self.vietphrase.items():
            self.vietphrase_trie.insert(k, v, 2)
            
        self.hanviet_trie = Trie()
        # Insert proper names (priority 2)
        for k, v in self.proper_names.items():
            self.hanviet_trie.insert(k, v, 2)
        print("Tries built successfully.")

        # Register proper names in Jieba dictionary for fast modes
        import jieba
        for name in self.proper_names:
            jieba.add_word(name)

    def parse_dict_content(self, content, convert_to_simplified=False):
        dictionary = {}
        if content:
            to_simplified = lambda s: s
            if convert_to_simplified:
                try:
                    from hanziconv import HanziConv
                    to_simplified = HanziConv.toSimplified
                except ImportError:
                    pass

            for line in content.splitlines():
                line = line.strip()
                if not line or "=" not in line or line.startswith('#'):
                    continue
                parts = line.split("=", 1)
                key = parts[0].strip()
                val = self.clean_annotation(parts[1].strip())
                dictionary[to_simplified(key)] = val
        return dictionary

    def parse_vietphrase_content(self, content):
        dictionary = {}
        if content:
            for line in content.splitlines():
                line = line.strip()
                if not line or "=" not in line or line.startswith('#'):
                    continue
                parts = line.split("=", 1)
                left = parts[0].strip()
                right = self.clean_annotation(parts[1].strip())
                
                if "," in left and "," in right:
                    keys = [k.strip() for k in left.split(",") if k.strip()]
                    vals = [v.strip() for v in right.split(",") if v.strip()]
                    if len(keys) == len(vals):
                        for k, v in zip(keys, vals):
                            dictionary[k] = v
                        continue
                if left:
                    dictionary[left] = right
        return dictionary

    def is_number(self, word):
        return bool(re.match(r'^[0-9一二三四五六七八九十百千万几数多半两]+$', word))

    def capitalize_phrase(self, phrase):
        chars = 'a-zA-ZàáạảãâầấậẩẫăằắặẳẵèéẹẻẽêềếệểễìíịỉĩòóọỏõôồốộổỗơờớợởỡùúụủũưừứựửữỳýỵỷỹđĐ'
        pattern = f'[{chars}]+'
        return re.sub(pattern, lambda m: m.group(0).capitalize(), phrase)

    def clean_annotation(self, text, mode='vietphrase'):
        if not text:
            return ""
        # 1. Parse curly braces {meaning:reading}
        def repl_curly(match):
            content = match.group(1)
            if ':' in content:
                parts = content.split(':', 1)
                return parts[0].strip() if mode == 'vietphrase' else parts[1].strip()
            return content.strip()
            
        text = re.sub(r'\{([^{}]+)\}', repl_curly, text)
        
        # 2. Strip (*...) annotations
        text = re.sub(r'\s*\(\*[^)]*\)', '', text)
        
        return text.strip()

    def format_translation(self, raw_value, multi_option, word=None, prefer_hanviet=False):
        if not raw_value:
            return ""
        options = [o for o in raw_value.split("/") if o.strip()]
        
        # Deduplicate options while preserving order
        seen = set()
        deduped = []
        for o in options:
            if o not in seen:
                seen.add(o)
                deduped.append(o)
                
        if not deduped:
            return ""
            
        if multi_option and len(deduped) > 1:
            return f"{deduped[0]}[{'/'.join(deduped[1:])}]"
        
        # If multi-option is False, we have a word of length >= 2, and prefer_hanviet is True, prefer Hán Việt alignment
        if prefer_hanviet and word and len(word) >= 2 and len(deduped) > 1:
            hv_sets = []
            for char in word:
                readings = set()
                if char in self.char_dict:
                    for r in self.char_dict[char].split('/'):
                        r_clean = r.strip().lower()
                        if r_clean:
                            readings.add(r_clean)
                if readings:
                    hv_sets.append(readings)
                    
            best_option = deduped[0]
            best_score = -1
            
            for opt in deduped:
                opt_syllables = [w.strip().lower() for w in opt.split() if w.strip()]
                score = 0
                for r_set in hv_sets:
                    if any(r in opt_syllables for r in r_set):
                        score += 1
                if score > best_score:
                    best_score = score
                    best_option = opt
            if best_score > 0:
                return best_option

        return deduped[0]

    def clean_punctuation_spacing(self, text):
        if not text:
            return text
        
        # 1. Ensure exactly one space after commas, semicolons, colons, periods, question marks, and exclamation marks.
        # Avoid inserting space if the next character is a closing bracket, closing quote, space, or another punctuation.
        text = re.sub(r'([,;.:!?])(?=[^\s)\]}』】”"’])', r'\1 ', text)
        
        # 2. Remove any accidental whitespace before these punctuation marks
        text = re.sub(r'\s+([,;.:!?])', r'\1', text)
        
        # 3. Clean spaces inside parentheses, brackets, and curly/double brackets (including Chinese quote styles)
        text = re.sub(r'([(\[{『【«])\s+', r'\1', text)
        text = re.sub(r'\s+([)\]}』】»])', r'\1', text)
        
        # Ensure a space exists before opening brackets and after closing brackets when they border words/digits
        text = re.sub(r'(?<=[^\s(\[{『【«])([(\[{『【«])', r' \1', text)
        text = re.sub(r'([)\]}』】»])(?=[^\s.,;:!?)\]}』】»])', r'\1 ', text)
        
        # 4. Standardize dashes/hyphens used as separators (e.g. "Artist - Song") to have one space on each side
        text = re.sub(r'\s*-\s*', ' - ', text)
        
        # 5. Clean up any duplicated/trailing whitespaces
        text = re.sub(r'\s+', ' ', text).strip()
        
        return text

    def translate_sentence(self, sentence, multi_option=False, mode=None):
        if not sentence or sentence.isspace():
            return ""
            
        # If the sentence doesn't contain any Chinese characters or symbols, preserve it as-is
        if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df\u3000-\u303f\uff00-\uffef]', sentence):
            return sentence
            
        # Segment into Chinese text blocks and non-Chinese text blocks
        # Keep Chinese characters and Chinese specific punctuations in the translation segment
        chinese_pattern = re.compile(r'([\u4e00-\u9fff\u3000-\u303f\uff00-\uffef]+)')
        parts = chinese_pattern.split(sentence)
        
        # Merge simple alphanumeric non-Chinese blocks into adjacent Chinese blocks
        i = 1
        while i < len(parts) - 1:
            non_chinese = parts[i+1]
            if re.match(r'^\s*[a-zA-Z0-9]+\s*$', non_chinese):
                parts[i] = parts[i] + non_chinese + parts[i+2]
                parts.pop(i+1)
                parts.pop(i+1)
            else:
                i += 2
                
        translated_parts = []
        capitalize_next = True
        
        for part in parts:
            if not part:
                continue
            if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df\u3000-\u303f\uff00-\uffef]', part):
                # Non-Chinese segment -> preserve exactly
                translated_parts.append(part)
                # Check if it ends with sentence terminator
                if re.search(r'[.!?]\s*$', part):
                    capitalize_next = True
                elif part.strip():
                    capitalize_next = False
            else:
                # Chinese segment -> translate
                trans = self._translate_pure_chinese_sentence(part, multi_option, mode, capitalize_first=capitalize_next)
                translated_parts.append(trans)
                # Check if it ends with sentence terminator
                if re.search(r'[.!?]\s*$', part) or re.search(r'[.!?]\s*$', trans):
                    capitalize_next = True
                else:
                    capitalize_next = False
                    
        return "".join(translated_parts)

    def _translate_pure_chinese_sentence(self, sentence, multi_option=False, mode=None, capitalize_first=True):
        if not sentence or sentence.isspace():
            return ""
            
        active_mode = mode or self.translation_mode
        
        # Tokenization & Tagging depending on mode
        if active_mode in ("advanced", "advanced_hanviet"):
            raw_tokens = [SimpleToken(t.word, t.flag) for t in pseg.cut(sentence)]
        else:
            # "fast", "vietphrase", "hanviet" modes use the fast tokenizer
            words = list(self.jieba_tokenizer.cut(sentence))
            raw_tokens = []
            for w in words:
                if w in PUNCT_SET:
                    tag = 'x'
                elif NUM_RE.match(w):
                    tag = 'm'
                else:
                    tag = self.pseg_dict.get(w, 'n')
                raw_tokens.append(SimpleToken(w, tag))
                
        if not raw_tokens:
            return ""
            
        NUM_KEYWORDS = {"重", "阶", "品", "级", "层", "剑", "星", "转", "天", "色", "关", "重天"}
        HANVIET_NUMBERS = {
            '0': 'Không', '1': 'Nhất', '2': 'Nhị', '3': 'Tam', '4': 'Tứ', '5': 'Ngũ', '6': 'Lục', '7': 'Thất', '8': 'Bát', '9': 'Cửu', '10': 'Thập',
            '一': 'Nhất', '二': 'Nhị', '三': 'Tam', '四': 'Tứ', '五': 'Ngũ', '六': 'Lục', '七': 'Thất', '八': 'Bát', '九': 'Cửu', '十': 'Thập',
            '百': 'Bách', '千': 'Thiên', '万': 'Vạn', '萬': 'Vạn', '几': 'Vài', '数': 'Số', '多': 'Đa', '半': 'Bán', '两': 'Lưỡng', '兩': 'Lưỡng'
        }
        # Helper function to translate a single token
        def translate_single_token(idx, tok, list_of_tokens):
            word = tok.word
            tag = tok.tag
            
            # Punctuation
            is_punct = (tag == 'x' or word in {',', '.', '!', '?', ';', ':', '"', '(', ')', '[', ']', '{', '}'})
            if is_punct:
                has_chinese = False
                for char in word:
                    if char in self.char_dict:
                        has_chinese = True
                        break
                if not has_chinese:
                    punct_map = {
                        '，': ',', '。': '.', '「': '"', '」': '"', '、': ',', '？': '?', '！': '!',
                        '：': ':', '；': ';', '“': '"', '”': '"', '（': '(', '）': ')'
                    }
                    tok.translated = punct_map.get(word, word)
                    return
                
            # Rule for number + 人 (e.g. 几十人, 三人)
            if len(word) > 1 and word.endswith('人') and self.is_number(word[:-1]):
                num_part = word[:-1]
                if num_part in self.vietphrase:
                    num_trans = self.format_translation(self.vietphrase[num_part], multi_option, num_part)
                else:
                    num_trans = " ".join([self.char_dict.get(c, c).split("/")[0] for c in num_part])
                tok.translated = f"{num_trans} người"
                return
                
            # Special rule for 了 (le vs liao)
            if word == 'l' or word == '了':
                is_at_end = True
                for next_tok in list_of_tokens[idx+1:]:
                    if next_tok.word in {'"', '\'', '(', ')', '[', ']', '{', '}', '“', '”', '‘', '’', '（', '）', '【', '】', '《', '》'}:
                        continue
                    if next_tok.word in {',', '.', '!', '?', ';', ':', '，', '。', '！', '？', '；', '：', '、'}:
                        is_at_end = True
                        break
                    is_at_end = False
                    break
                if is_at_end:
                    tok.translated = "rồi"
                else:
                    tok.translated = "được"
                return
                
            # Cultivation Realm (cultivation)
            if tag == 'cultivation':
                result = []
                for char in word:
                    if char in HANVIET_NUMBERS:
                        result.append(HANVIET_NUMBERS[char])
                    else:
                        cap_val = self.char_dict.get(char, char).split("/")[0].capitalize()
                        result.append(cap_val)
                tok.translated = " ".join(result)
                return
                
            # Determine if it's a noun or an adjective
            is_proper = (tag in {'nr', 'ns', 'nt'} if tag else False)
            is_noun = (tag.startswith('n') if tag else False) or tag in {'n', 'nz', 'ng'} if tag else False
            is_adj = tag in {'a', 'b', 'ad', 'an', 'z'} if tag else False
            is_noun_or_adj = is_proper or is_noun or is_adj
            
            # --- Chốt chặn cuối cùng cho Tên riêng (Proper Names Guard) ---
            if is_proper:
                if word in self.proper_names:
                    tok.translated = self.format_translation(self.proper_names[word], multi_option, word, prefer_hanviet=True)
                else:
                    if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', word):
                        tok.translated = word
                    else:
                        result = []
                        for char in word:
                            val = self.char_dict.get(char, char).split("/")[0]
                            result.append(val)
                        tok.translated = " ".join(result)
                if tok.translated:
                    tok.translated = self.capitalize_phrase(tok.translated)
            
            # --- Translate lookup strategy depending on active_mode (for non-proper names) ---
            else:
                if active_mode == 'hanviet':
                    # Mode 4: Pure Hán Việt (NO Vietphrase)
                    if word in self.proper_names:
                        tok.translated = self.format_translation(self.proper_names[word], multi_option, word, prefer_hanviet=True)
                    else:
                        if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', word):
                            tok.translated = word
                        else:
                            result = []
                            for char in word:
                                val = self.char_dict.get(char, char).split("/")[0]
                                result.append(val)
                            tok.translated = " ".join(result)

                elif active_mode == 'vietphrase':
                    # Mode 3: Prioritize Vietphrase (Traditional)
                    if word in self.vietphrase:
                        tok.translated = self.format_translation(self.vietphrase[word], multi_option, word, prefer_hanviet=False)
                    elif word in self.proper_names:
                        tok.translated = self.format_translation(self.proper_names[word], multi_option, word, prefer_hanviet=True)
                    else:
                        if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', word):
                            tok.translated = word
                        else:
                            result = []
                            for char in word:
                                val = self.char_dict.get(char, char).split("/")[0]
                                result.append(val)
                            tok.translated = " ".join(result)

                else:
                    # Modes 1, 2 & 5: 'fast', 'advanced', or 'advanced_hanviet' (POS-based noun/adjective Hán Việt override)
                    if is_noun_or_adj:
                        # Nouns/Adjectives: Bypasses vietphrase
                        if word in self.proper_names:
                            tok.translated = self.format_translation(self.proper_names[word], multi_option, word, prefer_hanviet=True)
                        else:
                            if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', word):
                                tok.translated = word
                            else:
                                result = []
                                for char in word:
                                    val = self.char_dict.get(char, char).split("/")[0]
                                    result.append(val)
                                tok.translated = " ".join(result)
                    else:
                        # Verbs and other parts of speech
                        if active_mode == 'advanced_hanviet':
                            # Prefer HanViet dictionary (proper_names) over Vietphrase
                            if word in self.proper_names:
                                tok.translated = self.format_translation(self.proper_names[word], multi_option, word, prefer_hanviet=True)
                            elif word in self.vietphrase:
                                tok.translated = self.format_translation(self.vietphrase[word], multi_option, word, prefer_hanviet=False)
                            else:
                                if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', word):
                                    tok.translated = word
                                else:
                                    result = []
                                    for char in word:
                                        val = self.char_dict.get(char, char).split("/")[0]
                                        result.append(val)
                                    tok.translated = " ".join(result)
                        else:
                            # Standard fast/advanced: vietphrase -> proper_names -> character fallback
                            if word in self.vietphrase:
                                tok.translated = self.format_translation(self.vietphrase[word], multi_option, word, prefer_hanviet=False)
                            elif word in self.proper_names:
                                tok.translated = self.format_translation(self.proper_names[word], multi_option, word, prefer_hanviet=True)
                            else:
                                if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', word):
                                    tok.translated = word
                                else:
                                    result = []
                                    for char in word:
                                        val = self.char_dict.get(char, char).split("/")[0]
                                        result.append(val)
                                    tok.translated = " ".join(result)
            
            # Strip trailing "đích" / "Đích" from modifier translations
            if tok.translated and word.endswith('的') and len(word) > 1:
                val = tok.translated
                if val.lower().endswith(' đích'):
                    tok.translated = val[:-5]
                elif val.lower().endswith('đích'):
                    tok.translated = val[:-4]

        # Step 1: Group numeral phrases and cultivation terms FIRST
        grouped = []
        i = 0
        while i < len(raw_tokens):
            tok = raw_tokens[i]
            word = tok.word
            tag = tok.tag
            
            if self.is_number(word) and i + 1 < len(raw_tokens) and raw_tokens[i+1].word in NUM_KEYWORDS:
                grouped_word = word + raw_tokens[i+1].word
                i_next = i + 2
                if i_next < len(raw_tokens) and raw_tokens[i_next].tag in {'n', 'nr', 'ns', 'nt', 'nz'}:
                    grouped_word += raw_tokens[i_next].word
                    i_next += 1
                grouped.append(SimpleToken(grouped_word, 'cultivation'))
                i = i_next
            else:
                grouped.append(SimpleToken(word, tag))
                i += 1

        # Step 2: Translate individual tokens on the cultivation-grouped tokens
        for idx, tok in enumerate(grouped):
            translate_single_token(idx, tok, grouped)

        # Step 3: Greedy merge adjacent tokens if their combination exists in dictionaries
        i = 0
        merged = []
        while i < len(grouped):
            matched = False
            for length in range(min(4, len(grouped) - i), 1, -1):
                combined_word = "".join([grouped[i+k].word for k in range(length)])
                
                # Prevent merging across '的' particle to preserve root Hán Việt translation and allow reordering
                should_skip = False
                if 'đích' in combined_word or '的' in combined_word and combined_word.find('的') > 0:
                    should_skip = True
                elif i + length < len(grouped) and grouped[i+length].word == '的':
                    # If next token is 'de' (de/的), don't merge if it would swallow a pronoun/noun/verb
                    last_tok = grouped[i+length-1]
                    if last_tok.flag in {'r', 'n', 'nr', 'ns', 'nt', 'nz', 'ng', 'v'}:
                        should_skip = True
                elif '是' in combined_word and any(p in combined_word for p in {'我', '你', 'he', 'she', 'it', '们', '您', '自己'}):
                    # Prevent merging copula + pronoun phrases (like '这是他', '那是我') to allow proper clause reordering
                    should_skip = True
                    
                # Dict check strategy depends on active_mode
                if active_mode == 'hanviet':
                    in_dicts = (combined_word in self.proper_names)
                else:
                    in_dicts = (combined_word in self.vietphrase or combined_word in self.proper_names)

                if not should_skip and in_dicts:
                    combined_tag = None
                    try:
                        cut_res = list(pseg.cut(combined_word))
                        if cut_res:
                            combined_tag = cut_res[0].flag
                    except Exception:
                        pass
                    if not combined_tag:
                        combined_tag = grouped[i].flag
                        for k in range(length):
                            if grouped[i+k].flag in {'nr', 'ns', 'nt', 'nz'}:
                                combined_tag = grouped[i+k].flag
                                break
                    new_tok = SimpleToken(combined_word, combined_tag)
                    # Translate the new merged token immediately
                    translate_single_token(0, new_tok, [new_tok])
                    merged.append(new_tok)
                    i += length
                    matched = True
                    break
            if not matched:
                merged.append(grouped[i])
                i += 1

        # Step 4: Reordering Grammar Rules
        if active_mode != 'hanviet':
            # Pass 1: Adjective + Noun reordering
            changed = True
            while changed:
                changed = False
                i = 0
            while i < len(merged) - 1:
                t_a = merged[i]
                t_n = merged[i+1]
                
                # Do not swap with prepositions/conjunctions/copulas/particles
                if t_n.word in {'跟', '和', '与', '與', '同', '在', '从', '從', '自', '由', '向', '往', '朝', '对', '對', '给', '給', '比', '是', '叫', '让', '讓', '被', '把', '使', '令', '到', '了', '的', '而', '&', '并', '並', '以', '或', '者'}:
                    i += 1
                    continue
                    
                if (t_a.tag in {'a', 'b'} or (t_a.word.endswith('的') and t_a.word != '的')) and t_n.tag in {'n', 'nr', 'ns', 'nt', 'nz', 'ng', 'v', 'vd', 'vg', 'vi', 'vn'}:
                    combined = t_n.translated + " " + t_a.translated
                    new_tok = SimpleToken(t_a.word + t_n.word, t_n.tag)
                    new_tok.translated = combined
                    merged[i:i+2] = [new_tok]
                    changed = True
                    break
                i += 1
            
        # Pass 2: "的" reordering (with multi-token noun/verb phrase lookahead)
        NOUN_PHRASE_TAGS = {'n', 'nr', 'ns', 'nt', 'nz', 'ng', 'a', 'b', 'm', 'q', 'j', 'i'}
        LOOKAHEAD_TAGS = NOUN_PHRASE_TAGS | {'v', 'vd', 'vg', 'vi', 'vn'}
        i = 1
        while i < len(merged) - 1:
            tok = merged[i]
            if tok.word in {'de', '的'}:
                t_x = merged[i-1]
                # Scan forward to collect all consecutive noun or verb phrase tokens
                k = i + 1
                has_noun = False
                while k < len(merged):
                    tok_k = merged[k]
                    # Stop collecting if we hit a locality word/orientation noun
                    if tok_k.word in {'下', '上', '中', '里', '外', '内', '內', '后', '後', '前', '旁', '侧', '側', '底', '间', '間'}:
                        break
                        
                    # If we already encountered a noun/verb in the phrase,
                    # we cannot have a subsequent adjective modifying that noun from the right.
                    if has_noun and tok_k.tag in {'a', 'b'}:
                        break
                    
                    # Do not collect a verb tag if we already have a noun/verb head
                    is_verb_tag = tok_k.tag in {'v', 'vd', 'vg', 'vi', 'vn'}
                    if has_noun and is_verb_tag:
                        break
                    
                    if tok_k.tag in LOOKAHEAD_TAGS or tok_k.word == '色':
                        if tok_k.tag in {'n', 'nr', 'ns', 'nt', 'nz', 'ng', 'v', 'vd', 'vg', 'vi', 'vn'}:
                            has_noun = True
                        k += 1
                    else:
                        break
                
                is_verb_modifier = t_x.tag in {'v', 'vd', 'vg', 'vi', 'vn'}
                
                # If we collected at least one token AND the modifier is not a verb clause
                if k > i + 1 and not is_verb_modifier:
                    y_tokens = merged[i+1:k]
                    y_translated = " ".join([t.translated for t in y_tokens if t.translated])
                    y_word = "".join([t.word for t in y_tokens])
                    
                    if t_x.tag != 'x':
                        start_idx = i - 1
                        j_back = i - 2
                        while j_back >= 0:
                            tag_back = merged[j_back].tag
                            if tag_back in {'n', 'nr', 'ns', 'nt', 'nz', 'ng', 'a', 'b', 'm', 'q', 'j', 'i', 's', 't'}:
                                start_idx = j_back
                                j_back -= 1
                            else:
                                break
                                
                        modifier_tokens = merged[start_idx:i]
                        modifier_translated = " ".join([t.translated for t in modifier_tokens if t.translated])
                        modifier_word = "".join([t.word for t in modifier_tokens])
                        
                        is_proper_or_pronoun = (
                            t_x.tag in {'nr', 'r'}
                        )
                        is_noun_modifier = is_proper_or_pronoun and not t_x.word.endswith('色')
                        if is_noun_modifier and start_idx == i - 1:
                            combined = y_translated + " của " + modifier_translated
                        else:
                            combined = y_translated + " " + modifier_translated
                        
                        new_tok = SimpleToken(modifier_word + tok.word + y_word, 'n')
                        new_tok.translated = combined
                        merged[start_idx:k] = [new_tok]
                        continue
                else:
                    # If we didn't reorder, set the '的' translation to empty string to avoid translating as 'đấy' / 'đích'
                    tok.translated = ""
            i += 1
            
        # Join words
        translated_text = " ".join([t.translated for t in merged if t.translated])
        
        # Clean spacing and punctuation
        translated_text = self.clean_punctuation_spacing(translated_text)
        
        # Capitalize sentences
        sentences = re.split(r'([.!?]\s*)', translated_text)
        start_idx = 0 if capitalize_first else 1
        for idx in range(start_idx, len(sentences)):
            s = sentences[idx]
            if s and not s.isspace() and not s[0] in {'.', '!', '?'}:
                for c_idx, char in enumerate(s):
                    if char.isalpha():
                        sentences[idx] = s[:c_idx] + char.upper() + s[c_idx+1:]
                        break
        return "".join(sentences).strip()

    def translate_paragraph(self, paragraph, multi_option=False, mode=None):
        if not paragraph or paragraph.isspace():
            return paragraph
            
        active_mode = mode or self.translation_mode
        
        if active_mode in ('vietphrase', 'hanviet'):
            # Ultra-fast Trie-based translation path (50M+ characters/minute)
            trie = self.vietphrase_trie if active_mode == 'vietphrase' else self.hanviet_trie
            prefer_hanviet = (active_mode == 'hanviet')
            
            i = 0
            text_length = len(paragraph)
            result_words = []
            
            while i < text_length:
                length, translation, priority = trie.search_longest_match(paragraph, i)
                if length > 0:
                    word = paragraph[i:i+length]
                    formatted = self.format_translation(translation, multi_option, word, prefer_hanviet=prefer_hanviet)
                    # Capitalize if it is a proper name
                    if priority == 1 or word in self.proper_names:
                        formatted = self.capitalize_phrase(formatted)
                    result_words.append(formatted)
                    i += length
                else:
                    char = paragraph[i]
                    if not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', char):
                        punct_map = {
                            '，': ',', '。': '.', '「': '"', '」': '"', '、': ',', '？': '?', '！': '!',
                            '：': ':', '；': ';', '“': '"', '”': '"', '（': '(', '）': ')',
                            '『': '"', '』': '"', '【': '[', '】': ']'
                        }
                        result_words.append(punct_map.get(char, char))
                    else:
                        val = self.char_dict.get(char, char).split("/")[0]
                        result_words.append(val)
                    i += 1
                    
            translated_text = " ".join(result_words)
            translated_text = self.clean_punctuation_spacing(translated_text)
            
            # Sentence Capitalization
            sentences = re.split(r'([.!?]\s*)', translated_text)
            for idx in range(len(sentences)):
                s = sentences[idx]
                if s and not s.isspace() and not s[0] in {'.', '!', '?'}:
                    for c_idx, char in enumerate(s):
                        if char.isalpha():
                            sentences[idx] = s[:c_idx] + char.upper() + s[c_idx+1:]
                            break
            return "".join(sentences).strip()

        # For advanced & fast modes, use normal sentence splitting & tokenization
        sentence_ends = re.compile(r'([。！？!?]+)')
        parts = sentence_ends.split(paragraph)
        
        translated_parts = []
        for part in parts:
            if not part:
                continue
            if sentence_ends.match(part):
                punct_map = {
                    '。': '.', '！': '!', '？': '?', '，': ','
                }
                translated_parts.append(punct_map.get(part, part))
            else:
                translated_parts.append(self.translate_sentence(part, multi_option, mode=mode))
                
        return self.clean_punctuation_spacing("".join(translated_parts))

    def translate_text_node(self, text, multi_option=False, mode=None):
        """
        Dich mot text node tu DOM.
        BAO TOAN HOAN TOAN cau truc: xuong dong \n, khoang trang dau/cuoi tung dong.
        """
        if not text:
            return text

        # Tach theo \n truoc -> dich tung dong doc lap -> gop lai
        lines = text.split('\n')
        translated_lines = []
        for line in lines:
            leading  = re.match(r'^\s*', line).group(0)
            trailing = re.search(r'\s*$', line).group(0)
            body = line.strip()

            if not body:
                translated_lines.append(line)  # dong rong -> giu nguyen
            elif not re.search(r'[\u3400-\u9fff\U00020000-\U0002a6df]', body):
                translated_lines.append(line)  # khong co chu Han -> giu nguyen
            else:
                translated_body = self.translate_paragraph(body, multi_option, mode=mode)
                translated_lines.append(leading + translated_body + trailing)

        return '\n'.join(translated_lines)

    def translate(self, text, multi_option=False, mode=None):
        return self.translate_text_node(text, multi_option=multi_option, mode=mode)