"""Two-dictionary orthographic kana checks for evaluation preparation only.""" import re from importlib.metadata import version KANA = re.compile(r"[ァ-ヺー。、!?「」『』()・:,,.!?]+") def katakana(text): return "".join(chr(ord(c) + 0x60) if "ぁ" <= c <= "ゖ" else c for c in text) class DualReading: def __init__(self): import fugashi from sudachipy import Dictionary, SplitMode self.mecab = fugashi.Tagger() self.sudachi = Dictionary().create() self.mode = SplitMode.C self.versions = { n: version(n) for n in ("fugashi", "unidic-lite", "sudachipy", "sudachidict-core") } def analyse(self, text, start=0): """Read a suffix in its sentence, rejecting boundaries inside words.""" left, right, unknown = [], [], [] offset = 0 for word in self.mecab(text): surface = word.surface end = offset + len(surface) if offset < start < end: unknown.append("boundary_inside_unidic_word") if offset < start: offset = end continue offset = end if word.is_unk: unknown.append(surface) if KANA.fullmatch(katakana(surface)): left.append(katakana(surface)) elif word.feature.kana: left.append(word.feature.kana) else: left.append(surface) unknown.append(surface) for word in self.sudachi.tokenize(text, self.mode): surface = word.surface() if word.begin() < start < word.end(): unknown.append("boundary_inside_sudachi_word") if word.begin() < start: continue if KANA.fullmatch(katakana(surface)): right.append(katakana(surface)) else: right.append(word.reading_form()) if word.is_oov(): unknown.append(surface) a, b = "".join(left), "".join(right) return { "unidic": a, "sudachi": b, "agreement": a == b, "unknown": sorted(set(unknown)), "kana_only": bool(KANA.fullmatch(a) and KANA.fullmatch(b)), }