File size: 6,838 Bytes
899d656
 
8d34056
899d656
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8d34056
 
 
 
 
 
899d656
8d34056
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
899d656
8d34056
899d656
 
 
 
 
 
 
8d34056
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
"""Local Helsinki-NLP opus-mt he<->ru fallback translator (CTranslate2 int8 + sentencepiece, no torch).
Models dir: env OPUS_DIR (default ./models), subdirs opus-he-ru, opus-ru-he (ct2 int8 + source.spm/target.spm)."""
import os, re, difflib, threading, time, pathlib, logging
log = logging.getLogger("tr")
DIR = pathlib.Path(os.environ.get("OPUS_DIR", pathlib.Path(__file__).with_name("models")))
_M, _lock = {}, threading.Lock()

def available():
    return all((DIR / f"opus-{p}" / "model.bin").exists() for p in ("he-ru", "ru-he"))

def _load(pair):
    with _lock:
        if pair not in _M:
            import ctranslate2, sentencepiece as spm
            d = DIR / f"opus-{pair}"
            tr = ctranslate2.Translator(str(d), device="cpu", compute_type="int8",
                                        inter_threads=1, intra_threads=int(os.environ.get("OPUS_THREADS", "2")))
            _M[pair] = (tr, spm.SentencePieceProcessor(model_file=str(d / "source.spm")),
                        spm.SentencePieceProcessor(model_file=str(d / "target.spm")))
    return _M[pair]

def preload():
    for p in ("he-ru", "ru-he"): _load(p)

# ---------------- glossary (tested on mt_bench/glossary: 27/27 term hits, 1 of 22 sentences needs Gemini) ----------------
HE_PREF = "ובלמהשכ"
def is_he(t): return bool(re.search(r"[\u0590-\u05FF]", t))
def parse(gl):
    g=[]
    for line in (gl or "").splitlines():
        if "=" in line:
            a,b=(x.strip() for x in line.split("=",1))
            if a and b: g.append((a,b))
    return g
def ru_stem(w):
    w=w.lower(); return re.sub(r"[аеиоуыэюяйь]+$","",w) if len(w)>3 else w
def src_pattern(term):
    """Regex for a glossary term in source text: Hebrew allows 0-2 prefix letters; Russian allows inflection endings."""
    words=re.split(r"([\s\-]+)",term)
    if is_he(term):
        body="".join(re.escape(w) if i%2==0 else r"[\s\-]+" for i,w in enumerate(words))
        return re.compile(rf"(?<![\w\u0590-\u05FF])([{HE_PREF}]{{0,2}})({body})(?![\w\u0590-\u05FF])")
    parts=[]
    for i,w in enumerate(words):
        if i%2: parts.append(r"[\s\-]+")
        else:
            st=ru_stem(w); parts.append(re.escape(st)+r"[а-яё]{0,3}")
    return re.compile(r"(?<![\w])()("+"".join(parts)+r")(?![\w])", re.I)
def find_terms(text, gl):
    """-> list of (src_term, tgt_term) whose source side occurs in text, longest first, both directions."""
    pairs=parse(gl); pairs=pairs+[(b,a) for a,b in pairs]
    src_he=is_he(text); out=[]
    for a,b in sorted(pairs,key=lambda x:-len(x[0])):
        if is_he(a)!=src_he: continue
        if src_pattern(a).search(text): out.append((a,b))
    return out
# --- skeletons for fuzzy matching
HE_MAP=str.maketrans({"ק":"כ","ט":"ת","ס":"ש","ך":"כ","ם":"מ","ן":"נ","ף":"פ","ץ":"צ","ב":"ב","'":"","־":"","-":""," ":""})
RU_MAP=str.maketrans({"х":"к","ё":"е","й":"и","ъ":"","ь":"","-":""," ":""})
def skel(t):
    if is_he(t):
        t=t.translate(HE_MAP); return re.sub("[ויאהע]","",t) or t
    t=t.lower().translate(RU_MAP); t=re.sub(r"(.)\1",r"\1",t)  # кк->к
    return re.sub("[аеиоуыэюя]","",t) or t
def sim(a,b): return difflib.SequenceMatcher(None,skel(a),skel(b)).ratio()
def fuzzy_sub(out, tgt, thr=0.6):
    """Replace the output span most similar to glossary target `tgt` by `tgt` (keeps Hebrew prefix / Russian ending)."""
    if tgt.lower() in out.lower(): return out, True
    toks=list(re.finditer(r"[\w\u0590-\u05FF'׳]+(?:-[\w\u0590-\u05FF'׳]+)*", out))
    if len(skel(tgt))<=3: thr=max(thr,0.75)
    n=len(tgt.split()); best=(0,None)
    for i in range(len(toks)):
        for k in range(max(1,n-1),n+2):
            if i+k>len(toks): break
            s,e=toks[i].start(),toks[i+k-1].end(); span=out[s:e]
            core=span; pre=""; suf=""
            if is_he(tgt):
                m=re.match(f"^([{HE_PREF}]{{0,2}})(.+)$",span)
                cands=[(m.group(1)[:j],span[j:]) for j in range(len(m.group(1))+1)]
            else:
                cands=[("",span)]
            for p,c in cands:
                if not is_he(tgt):
                    # allow inflected ending on the last word
                    for cut in range(0,4):
                        cc=c[:len(c)-cut] if cut else c
                        if len(cc)<2: break
                        r=sim(cc,tgt)
                        if r>best[0]: best=(r,(s,e,p,c[len(c)-cut:] if cut else ""))
                else:
                    r=sim(c,tgt)
                    if r>best[0]: best=(r,(s,e,p,""))
    if best[1] and best[0]>=thr:
        s,e,p,suf=best[1]
        if not is_he(tgt) and suf and not re.fullmatch(r"[аеиоуыюяйьём]{1,3}",suf): suf=""
        return out[:s]+p+tgt+suf+out[e:], True
    return out, False

def _raw(text, target):
    pair = "ru-he" if target == "he" else "he-ru"
    tr, ss, ts = _load(pair)
    out = []
    for sent in [s for s in re.split(r"(?<=[.!?…])\s+", text.strip()) if s] or [text]:
        toks = ss.encode(sent, out_type=str) + ["</s>"]
        r = tr.translate_batch([toks], beam_size=4, max_decoding_length=200, repetition_penalty=1.1)
        out.append(ts.decode(r[0].hypotheses[0]))
    return re.sub(r"(\d{1,2}): (\d{2})\b", r"\1:\2", " ".join(out))  # '15: 30' -> '15:30'

def term_ok(out, tgt):
    if is_he(tgt): return tgt in out
    o = out.lower(); return all(ru_stem(w) in o for w in re.split(r"[\s\-]+", tgt.lower()))

PH_BAD = re.compile(r"ZQX|קיו|אקס|ЗКХ|зкх", re.I)
def translate_ex(text, target, glossary=""):
    """-> (translation, missing glossary targets, method). Strategy per glossary hit in the source:
    1) plain translation + fuzzy post-substitution of the model's transliteration (most fluent);
    2) source term -> placeholder 'ZQX<n>' (opus-mt copies it), then put the target term back.
    The caller may fall back to an LLM when terms are still missing."""
    terms = find_terms(text, glossary)
    out = _raw(text, target)
    if not terms: return out, [], "plain"
    best = None
    for a, b in terms: out, _ = fuzzy_sub(out, b)
    miss = [b for a, b in terms if not term_ok(out, b)]
    if not miss: return out, [], "fuzzy"
    best = (out, miss, "fuzzy")
    src, ph = text, {}
    for i, (a, b) in enumerate(terms):
        key = f"ZQX{i + 1}"; src = src_pattern(a).sub(lambda m: m.group(1) + key, src); ph[key] = b
    out2 = _raw(src, target)
    for k, v in ph.items(): out2 = out2.replace(k, v)
    miss2 = [b for a, b in terms if not term_ok(out2, b)]
    if not PH_BAD.search(out2) and len(miss2) < len(miss): best = (out2, miss2, "placeholder")
    return best

def translate(text, target, glossary=""):
    """Translate Hebrew->Russian (target='ru') or Russian->Hebrew (target='he'). Blocking; call in a thread."""
    return translate_ex(text, target, glossary)[0]