Spaces:
Sleeping
Sleeping
File size: 6,838 Bytes
899d656 8d34056 899d656 8d34056 899d656 8d34056 899d656 8d34056 899d656 8d34056 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """Local Helsinki-NLP opus-mt he<->ru fallback translator (CTranslate2 int8 + sentencepiece, no torch).
Models dir: env OPUS_DIR (default ./models), subdirs opus-he-ru, opus-ru-he (ct2 int8 + source.spm/target.spm)."""
import os, re, difflib, threading, time, pathlib, logging
log = logging.getLogger("tr")
DIR = pathlib.Path(os.environ.get("OPUS_DIR", pathlib.Path(__file__).with_name("models")))
_M, _lock = {}, threading.Lock()
def available():
return all((DIR / f"opus-{p}" / "model.bin").exists() for p in ("he-ru", "ru-he"))
def _load(pair):
with _lock:
if pair not in _M:
import ctranslate2, sentencepiece as spm
d = DIR / f"opus-{pair}"
tr = ctranslate2.Translator(str(d), device="cpu", compute_type="int8",
inter_threads=1, intra_threads=int(os.environ.get("OPUS_THREADS", "2")))
_M[pair] = (tr, spm.SentencePieceProcessor(model_file=str(d / "source.spm")),
spm.SentencePieceProcessor(model_file=str(d / "target.spm")))
return _M[pair]
def preload():
for p in ("he-ru", "ru-he"): _load(p)
# ---------------- glossary (tested on mt_bench/glossary: 27/27 term hits, 1 of 22 sentences needs Gemini) ----------------
HE_PREF = "ובלמהשכ"
def is_he(t): return bool(re.search(r"[\u0590-\u05FF]", t))
def parse(gl):
g=[]
for line in (gl or "").splitlines():
if "=" in line:
a,b=(x.strip() for x in line.split("=",1))
if a and b: g.append((a,b))
return g
def ru_stem(w):
w=w.lower(); return re.sub(r"[аеиоуыэюяйь]+$","",w) if len(w)>3 else w
def src_pattern(term):
"""Regex for a glossary term in source text: Hebrew allows 0-2 prefix letters; Russian allows inflection endings."""
words=re.split(r"([\s\-]+)",term)
if is_he(term):
body="".join(re.escape(w) if i%2==0 else r"[\s\-]+" for i,w in enumerate(words))
return re.compile(rf"(?<![\w\u0590-\u05FF])([{HE_PREF}]{{0,2}})({body})(?![\w\u0590-\u05FF])")
parts=[]
for i,w in enumerate(words):
if i%2: parts.append(r"[\s\-]+")
else:
st=ru_stem(w); parts.append(re.escape(st)+r"[а-яё]{0,3}")
return re.compile(r"(?<![\w])()("+"".join(parts)+r")(?![\w])", re.I)
def find_terms(text, gl):
"""-> list of (src_term, tgt_term) whose source side occurs in text, longest first, both directions."""
pairs=parse(gl); pairs=pairs+[(b,a) for a,b in pairs]
src_he=is_he(text); out=[]
for a,b in sorted(pairs,key=lambda x:-len(x[0])):
if is_he(a)!=src_he: continue
if src_pattern(a).search(text): out.append((a,b))
return out
# --- skeletons for fuzzy matching
HE_MAP=str.maketrans({"ק":"כ","ט":"ת","ס":"ש","ך":"כ","ם":"מ","ן":"נ","ף":"פ","ץ":"צ","ב":"ב","'":"","־":"","-":""," ":""})
RU_MAP=str.maketrans({"х":"к","ё":"е","й":"и","ъ":"","ь":"","-":""," ":""})
def skel(t):
if is_he(t):
t=t.translate(HE_MAP); return re.sub("[ויאהע]","",t) or t
t=t.lower().translate(RU_MAP); t=re.sub(r"(.)\1",r"\1",t) # кк->к
return re.sub("[аеиоуыэюя]","",t) or t
def sim(a,b): return difflib.SequenceMatcher(None,skel(a),skel(b)).ratio()
def fuzzy_sub(out, tgt, thr=0.6):
"""Replace the output span most similar to glossary target `tgt` by `tgt` (keeps Hebrew prefix / Russian ending)."""
if tgt.lower() in out.lower(): return out, True
toks=list(re.finditer(r"[\w\u0590-\u05FF'׳]+(?:-[\w\u0590-\u05FF'׳]+)*", out))
if len(skel(tgt))<=3: thr=max(thr,0.75)
n=len(tgt.split()); best=(0,None)
for i in range(len(toks)):
for k in range(max(1,n-1),n+2):
if i+k>len(toks): break
s,e=toks[i].start(),toks[i+k-1].end(); span=out[s:e]
core=span; pre=""; suf=""
if is_he(tgt):
m=re.match(f"^([{HE_PREF}]{{0,2}})(.+)$",span)
cands=[(m.group(1)[:j],span[j:]) for j in range(len(m.group(1))+1)]
else:
cands=[("",span)]
for p,c in cands:
if not is_he(tgt):
# allow inflected ending on the last word
for cut in range(0,4):
cc=c[:len(c)-cut] if cut else c
if len(cc)<2: break
r=sim(cc,tgt)
if r>best[0]: best=(r,(s,e,p,c[len(c)-cut:] if cut else ""))
else:
r=sim(c,tgt)
if r>best[0]: best=(r,(s,e,p,""))
if best[1] and best[0]>=thr:
s,e,p,suf=best[1]
if not is_he(tgt) and suf and not re.fullmatch(r"[аеиоуыюяйьём]{1,3}",suf): suf=""
return out[:s]+p+tgt+suf+out[e:], True
return out, False
def _raw(text, target):
pair = "ru-he" if target == "he" else "he-ru"
tr, ss, ts = _load(pair)
out = []
for sent in [s for s in re.split(r"(?<=[.!?…])\s+", text.strip()) if s] or [text]:
toks = ss.encode(sent, out_type=str) + ["</s>"]
r = tr.translate_batch([toks], beam_size=4, max_decoding_length=200, repetition_penalty=1.1)
out.append(ts.decode(r[0].hypotheses[0]))
return re.sub(r"(\d{1,2}): (\d{2})\b", r"\1:\2", " ".join(out)) # '15: 30' -> '15:30'
def term_ok(out, tgt):
if is_he(tgt): return tgt in out
o = out.lower(); return all(ru_stem(w) in o for w in re.split(r"[\s\-]+", tgt.lower()))
PH_BAD = re.compile(r"ZQX|קיו|אקס|ЗКХ|зкх", re.I)
def translate_ex(text, target, glossary=""):
"""-> (translation, missing glossary targets, method). Strategy per glossary hit in the source:
1) plain translation + fuzzy post-substitution of the model's transliteration (most fluent);
2) source term -> placeholder 'ZQX<n>' (opus-mt copies it), then put the target term back.
The caller may fall back to an LLM when terms are still missing."""
terms = find_terms(text, glossary)
out = _raw(text, target)
if not terms: return out, [], "plain"
best = None
for a, b in terms: out, _ = fuzzy_sub(out, b)
miss = [b for a, b in terms if not term_ok(out, b)]
if not miss: return out, [], "fuzzy"
best = (out, miss, "fuzzy")
src, ph = text, {}
for i, (a, b) in enumerate(terms):
key = f"ZQX{i + 1}"; src = src_pattern(a).sub(lambda m: m.group(1) + key, src); ph[key] = b
out2 = _raw(src, target)
for k, v in ph.items(): out2 = out2.replace(k, v)
miss2 = [b for a, b in terms if not term_ok(out2, b)]
if not PH_BAD.search(out2) and len(miss2) < len(miss): best = (out2, miss2, "placeholder")
return best
def translate(text, target, glossary=""):
"""Translate Hebrew->Russian (target='ru') or Russian->Hebrew (target='he'). Blocking; call in a thread."""
return translate_ex(text, target, glossary)[0]
|