Update app.py

app.py

@@ -2,14 +2,21 @@ import difflib
 import re
 
 HYPHENS = set(['\u002D','\uFF0D','\u2010','\u2011','\u2013','\u2014'])
+YAKUMONO = set('。、．，〔〕《》〈〉【】［］（）()「」『』｢｣〝〟！？・…―!?〜～；：;:／/＼\\‥＋+＊*＃#＆&％%＠@＄$￥¥＾^｜|`｀′″§※〒♪♫♬♩→←↑↓↗↘↙↖◯○●◎□■△▲▽▼◇◆☆★©®™')
 BRACKET_PAIRS = [('（','）'),('(',')'),(  '〔','〕'),('《','》'),('〈','〉'),('【','】'),('［','］'),('「','」'),('『','』')]
 OPEN_BRACKETS = set(p[0] for p in BRACKET_PAIRS)
 CLOSE_BRACKETS = set(p[1] for p in BRACKET_PAIRS)
 BRACKET_MAP = {p[0]: p[1] for p in BRACKET_PAIRS}
 
 def build_active_flags(original, settings):
+    """
+    各文字が比較対象か否かを示すフラグリストを返す。
+    True=比較対象, False=除去対象
+    除去対象文字は比較からは外れるが、元テキスト上に残りハイライト対象になりうる。
+    """
     n = len(original)
     flags = [True] * n
+
     if settings.get('ignore_brackets_with_content'):
         i = 0
         while i < n:
@@ -30,51 +37,147 @@ def build_active_flags(original, settings):
                     i += 1
             else:
                 i += 1
+
+    if settings.get('ignore_yakumono'):
+        for i, ch in enumerate(original):
+            if ch in YAKUMONO: flags[i] = False
+
+    if settings.get('ignore_alpha'):
+        for i, ch in enumerate(original):
+            if re.match(r'[A-Za-zＡ-Ｚａ-ｚ]', ch): flags[i] = False
+
+    if settings.get('ignore_digit'):
+        for i, ch in enumerate(original):
+            if re.match(r'[0-9０-９]', ch): flags[i] = False
+
+    if settings.get('ignore_kana'):
+        for i, ch in enumerate(original):
+            if re.match(r'[ァ-ンｦ-ﾟ]', ch): flags[i] = False
+
     if settings.get('ignore_hyphen'):
         for i, ch in enumerate(original):
             if ch in HYPHENS: flags[i] = False
+
     if settings.get('ignore_space'):
         for i, ch in enumerate(original):
             if ch in ' \t　': flags[i] = False
+
     if settings.get('ignore_newline'):
         for i, ch in enumerate(original):
             if ch in '\r\n': flags[i] = False
+
     return flags
 
-def tokenize_with_flags(original, flags):
-    pattern = re.compile(
-        r'[a-zA-ZａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ]+|'
-        r'[0-9０-９]+|'
-        r'[ぁ-んァ-ンｦ-ﾟ\u30A0]+|'
-        r'[\u4E00-\u9FFF\u3400-\u4DBF]+|'
-        r'[ \t　]+|'
-        r'\r\n|\r|\n|'
-        r'.'
-    )
-    active_chars = [(ch, i) for i, ch in enumerate(original) if flags[i]]
-    active_text = ''.join(ch for ch, _ in active_chars)
-    tokens = []
-    for m in pattern.finditer(active_text):
-        s = m.start()
-        e = m.end()
-        orig_start = active_chars[s][1]
-        orig_end = active_chars[e-1][1] + 1
-        tokens.append((m.group(), orig_start, orig_end))
-    return tokens
+def build_active_text_and_map(original, flags):
+    """
+    有効文字のみ抽出したテキストと、
+    有効テキスト内インデックス→元テキストインデックスのマップを返す
+    """
+    active_chars = [(i, ch) for i, ch in enumerate(original) if flags[i]]
+    active_text = ''.join(ch for _, ch in active_chars)
+    index_map = [i for i, _ in active_chars]  # active_text[k] -> original[index_map[k]]
+    return active_text, index_map
+
+def compare(src, dst, settings):
+    src_flags = build_active_flags(src, settings)
+    dst_flags = build_active_flags(dst, settings)
+
+    src_active, src_map = build_active_text_and_map(src, src_flags)
+    dst_active, dst_map = build_active_text_and_map(dst, dst_flags)
+
+    matcher = difflib.SequenceMatcher(None, src_active, dst_active, autojunk=False)
+    opcodes = matcher.get_opcodes()
+
+    # 有効文字の差分範囲を元テキスト位置に変換
+    src_diff_ranges = []  # 比較元の削除箇所
+    dst_diff_ranges = []  # 比較先の追加箇所
+
+    for tag, i1, i2, j1, j2 in opcodes:
+        if tag in ('replace', 'delete') and i2 > i1:
+            orig_start = src_map[i1]
+            orig_end   = src_map[i2-1] + 1
+            src_diff_ranges.append((orig_start, orig_end))
+        if tag in ('replace', 'insert') and j2 > j1:
+            orig_start = dst_map[j1]
+            orig_end   = dst_map[j2-1] + 1
+            dst_diff_ranges.append((orig_start, orig_end))
+
+    # 除去文字のハイライト:
+    # 除去文字(flags=False)が有効文字の差分範囲に隣接している場合、
+    # その除去文字もハイライト範囲に取り込む
+    def expand_with_removed(text, flags, ranges):
+        """
+        差分範囲に隣接する除去文字を取り込んで範囲を拡張する。
+        「隣接」とは差分範囲の直前・直後に除去文字が連続している場合。
+        """
+        expanded = []
+        for s, e in ranges:
+            # 前方向: sの直前にある除去文字を取り込む
+            new_s = s
+            while new_s > 0 and not flags[new_s - 1]:
+                new_s -= 1
+            # 後方向: eの直後にある除去文字を取り込む
+            new_e = e
+            while new_e < len(text) and not flags[new_e]:
+                new_e += 1
+            expanded.append((new_s, new_e))
+        # マージ
+        merged = []
+        for s, e in sorted(expanded):
+            if s >= e: continue
+            if merged and s <= merged[-1][1]:
+                merged[-1] = (merged[-1][0], max(merged[-1][1], e))
+            else:
+                merged.append([s, e])
+        return merged
+
+    src_ranges = expand_with_removed(src, src_flags, src_diff_ranges)
+    dst_ranges = expand_with_removed(dst, dst_flags, dst_diff_ranges)
+
+    return src_ranges, dst_ranges
+
+def apply_highlight(text, ranges):
+    result = []
+    prev = 0
+    for s, e in sorted(ranges):
+        result.append(text[prev:s])
+        result.append('"' + text[s:e] + '"')
+        prev = e
+    result.append(text[prev:])
+    return ''.join(result)
 
 test_cases = [
+    ("TEST-01", "吾輩は猫である。名前はまだない。", "吾輩は猫である。名前はまだない。", {}),
+    ("TEST-02", "吾輩は猫である。", "吾輩は犬である。", {}),
+    ("TEST-03", "abcdef", "abcXYZdef", {}),
+    ("TEST-04", "abcXYZdef", "abcdef", {}),
+    ("TEST-08", "吾輩は、猫である。「名前」は（まだ）ない！", "吾輩は猫である名前はまだない",
+     {'ignore_yakumono': True}),
+    ("TEST-09", "(abc)テスト", "abcテスト",
+     {'ignore_yakumono': True}),
+    ("TEST-19", "abc-def－ghi‐jkl‑mno–pqr—stu", "abcdefghijklmnopqrstu",
+     {'ignore_hyphen': True}),
+    ("TEST-21", "abc def　ghi", "abcdefghi",
+     {'ignore_space': True}),
+    ("TEST-22", "abc\ndef\nghi", "abcdefghi",
+     {'ignore_newline': True}),
+    ("TEST-23", "吾輩は(猫)である。名前は【まだ】ない！", "吾輩はである。名前はない！",
+     {'ignore_yakumono': True, 'ignore_brackets_with_content': True}),
+    ("TEST-25",
+     "abc(def)123アイウ〔ghi〕fi⼀zero-test \nabc\n(def)123アイウ〔ghi〕fi⼀zero-test \nabc",
+     "xyz(www)789カキクケコ〔jkl〕fl⼆one+test \nxyz\n(www)789カキクケコ〔jkl〕fl⼆one+test \nxyz",
+     {'ignore_alpha': True, 'ignore_digit': True, 'ignore_kana': True, 'ignore_yakumono': True,
+      'ignore_ligature': True, 'ignore_kangxi': True, 'ignore_brackets_with_content': True,
+      'ignore_kanjinum': True, 'ignore_hyphen': True, 'ignore_kumimoji': True,
+      'ignore_space': True, 'ignore_newline': True}),
     ("TEST-30", "abc def", "abcdef", {}),
     ("TEST-35", "abcXYZdef", "abcdef", {}),
+    ("TEST-37", "abc-def－ghi‐jkl‑mno–pqr—stu", "abcdefghijklmnopqrstu",
+     {'ignore_hyphen': True}),
 ]
 
 for name, src, dst, settings in test_cases:
-    src_flags = build_active_flags(src, settings)
-    dst_flags = build_active_flags(dst, settings)
-    src_tokens = tokenize_with_flags(src, src_flags)
-    dst_tokens = tokenize_with_flags(dst, dst_flags)
-    print(f"\n=== {name} tokens ===")
-    print(f"  src_tokens: {src_tokens}")
-    print(f"  dst_tokens: {dst_tokens}")
-    matcher = difflib.SequenceMatcher(None, [t[0] for t in src_tokens], [t[0] for t in dst_tokens], autojunk=False)
-    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
-        print(f"  {tag:10} src={repr(''.join(t[0] for t in src_tokens[i1:i2]))}  dst={repr(''.join(t[0] for t in dst_tokens[j1:j2]))}")
+    src_ranges, dst_ranges = compare(src, dst, settings)
+    print(f"\n=== {name} ===")
+    print(f"  比較元: {apply_highlight(src, src_ranges)}")
+    print(f"  比較先: {apply_highlight(dst, dst_ranges)}")