Update app.py
Browse files
app.py
CHANGED
|
@@ -1,11 +1,24 @@
|
|
| 1 |
-
|
| 2 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import gradio as gr
|
| 4 |
|
| 5 |
# ------------ Archivos esperados ------------
|
| 6 |
-
LEX_MINI = "lexicon_minimax.json"
|
| 7 |
-
LEX_KOMI = "lexicon_komin.json"
|
| 8 |
-
LEX_MASTER = "lexicon_master.json"
|
| 9 |
|
| 10 |
# ------------ Normalización ------------
|
| 11 |
WORD_RE = re.compile(r"[A-Za-zÁÉÍÓÚÜÑáéíóúüñ]+", re.UNICODE)
|
|
@@ -61,7 +74,7 @@ def load_lexicons():
|
|
| 61 |
EN2MINI, EN2KOMI, MINI2EN, KOMI2EN,
|
| 62 |
ES2EN_LEMMA, EN2ES_LEMMA) = load_lexicons()
|
| 63 |
|
| 64 |
-
# ------------ OOV reversible ------------
|
| 65 |
ALPHA_MINI64 = "@ptkmnslraeiouy0123456789><=:/!?.+-_*#bcdfghjvqwxzACEGHIJKLMNOPRS"[:64]
|
| 66 |
CJK_BASE = (
|
| 67 |
"天地人日月山川雨風星火水木土金石光影花草鳥犬猫魚"
|
|
@@ -105,7 +118,7 @@ def is_oov_minimax(code: str) -> bool:
|
|
| 105 |
def is_oov_komin(code: str) -> bool:
|
| 106 |
return len(code) >= 2 and code.startswith("「") and code.endswith("」")
|
| 107 |
|
| 108 |
-
# ------------ spaCy ------------
|
| 109 |
USE_SPACY = False
|
| 110 |
try:
|
| 111 |
import spacy
|
|
@@ -124,13 +137,8 @@ def lemma_of(tok, src_lang: str) -> str:
|
|
| 124 |
else:
|
| 125 |
return norm_en(tok.lemma_ if tok.lemma_ else tok.text)
|
| 126 |
|
| 127 |
-
# ------------
|
| 128 |
def pick_predicative_sentence(doc):
|
| 129 |
-
"""
|
| 130 |
-
Devuelve la oración que contiene un verbo ROOT (VERB/AUX).
|
| 131 |
-
Si hay varias: prioriza la que tiene interrogación o sujeto.
|
| 132 |
-
Si no hay ninguna, devuelve el doc completo.
|
| 133 |
-
"""
|
| 134 |
sents = list(doc.sents) if doc.has_annotation("SENT_START") else [doc]
|
| 135 |
candidates = []
|
| 136 |
for s in sents:
|
|
@@ -146,19 +154,16 @@ def pick_predicative_sentence(doc):
|
|
| 146 |
return doc
|
| 147 |
return sorted(candidates, key=lambda x: x[0], reverse=True)[0][1].as_doc()
|
| 148 |
|
| 149 |
-
# ------------ Filtro de contenido (idioma-agnóstico) ------------
|
| 150 |
def is_content_token(t) -> bool:
|
| 151 |
-
# fuera: puntuación/símbolos/partículas/determinantes/preposiciones/conjunciones
|
| 152 |
if t.pos_ in ("INTJ", "DET", "ADP", "SCONJ", "CCONJ", "PART", "SYM", "PUNCT"):
|
| 153 |
return False
|
| 154 |
-
# fuera: discurso/vocativos/etc.
|
| 155 |
if t.dep_ in ("discourse", "intj", "vocative", "dep"):
|
| 156 |
return False
|
| 157 |
-
# WH genéricas por tag (EN) y lower forms típicas (ES/EN)
|
| 158 |
low = t.lower_.strip("¿?¡!.,;:()[]{}\"'").lower()
|
| 159 |
if t.tag_.startswith("W") or low in {
|
| 160 |
"como","cómo","que","qué","quien","quién","donde","dónde","cuando","cuándo",
|
| 161 |
-
"porqué","por","por qué","cuanto","cuánto",
|
|
|
|
| 162 |
"hola","hello","hi","hey","ey","eh","oye","adios","adiós","ciao"
|
| 163 |
}:
|
| 164 |
return False
|
|
@@ -181,7 +186,7 @@ def code_en(lemma: str, target: str) -> str:
|
|
| 181 |
if EN2KOMI: return EN2KOMI.get(lemma) or enc_oov_komin(lemma)
|
| 182 |
return enc_oov_komin(lemma)
|
| 183 |
|
| 184 |
-
# ------------
|
| 185 |
TAM_MINI = {"Pres":"P", "Past":"T", "Fut":"F", "UNK":"P"}
|
| 186 |
TAM_KOMI = {"Pres":"Ⓟ", "Past":"Ⓣ", "Fut":"Ⓕ", "UNK":"Ⓟ"}
|
| 187 |
|
|
@@ -203,7 +208,8 @@ def detect_tense(root) -> str:
|
|
| 203 |
if c.pos_ == "AUX":
|
| 204 |
cm = c.morph
|
| 205 |
if "Tense=Past" in cm: return "Past"
|
| 206 |
-
|
|
|
|
| 207 |
return "Pres"
|
| 208 |
|
| 209 |
def extract_core(doc):
|
|
@@ -222,80 +228,83 @@ def extract_core(doc):
|
|
| 222 |
obls.sort(key=lambda x: x.i); advs.sort(key=lambda x: x.i)
|
| 223 |
return root, subs, objs, obls, advs
|
| 224 |
|
| 225 |
-
|
| 226 |
-
def realize_minimax(doc, src_lang: str, drop_articles=True, zero_copula=True):
|
| 227 |
root, subs, objs, obls, advs = extract_core(doc)
|
| 228 |
tense, is_q, is_neg = detect_tense(root), detect_polarity(doc), detect_neg(doc)
|
| 229 |
|
| 230 |
-
vlem = lemma_of(root, src_lang)
|
| 231 |
vcode = code_es(vlem, "Minimax-ASCII") if src_lang=="Español" else code_en(vlem, "Minimax-ASCII")
|
| 232 |
|
| 233 |
tail = TAM_MINI.get(tense, "P")
|
|
|
|
|
|
|
| 234 |
if is_neg: tail += "N"
|
| 235 |
if is_q: tail += "Q"
|
| 236 |
if tail:
|
| 237 |
-
vcode = f"{vcode}·{tail}"
|
| 238 |
|
| 239 |
def realize_np(tokens):
|
| 240 |
outs=[]
|
| 241 |
for t in tokens:
|
| 242 |
-
if not is_content_token(t):
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
|
|
|
| 247 |
return outs
|
| 248 |
|
| 249 |
S = realize_np(subs)
|
| 250 |
O = realize_np(objs) + realize_np(obls)
|
| 251 |
ADV=[]
|
| 252 |
for a in advs:
|
| 253 |
-
if not is_content_token(a):
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
ADV.append(code_es(lem, "Minimax-ASCII") if src_lang=="Español" else code_en(lem, "Minimax-ASCII"))
|
| 257 |
|
| 258 |
-
|
| 259 |
-
if zero_copula and vlem in ("ser","estar","be") and tense=="Pres" and not is_neg and not is_q:
|
| 260 |
parts = S + O + ADV
|
| 261 |
else:
|
| 262 |
parts = [vcode] + S + O + ADV
|
| 263 |
return " ".join(p for p in parts if p)
|
| 264 |
|
| 265 |
-
|
| 266 |
-
def realize_komin(doc, src_lang: str, drop_articles=True, zero_copula=True):
|
| 267 |
root, subs, objs, obls, advs = extract_core(doc)
|
| 268 |
tense, is_q, is_neg = detect_tense(root), detect_polarity(doc), detect_neg(doc)
|
| 269 |
|
| 270 |
-
vlem = lemma_of(root, src_lang)
|
| 271 |
vcode = code_es(vlem, "Kōmín-CJK") if src_lang=="Español" else code_en(vlem, "Kōmín-CJK")
|
| 272 |
|
| 273 |
P_SUBJ, P_OBJ = "ᵖ", "ᵒ"
|
| 274 |
NEG_M, Q_FIN = "̆", "?"
|
| 275 |
TAM = TAM_KOMI.get(tense, "Ⓟ")
|
| 276 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
def realize_np(tokens, particle):
|
| 278 |
outs=[]
|
| 279 |
for t in tokens:
|
| 280 |
-
if not is_content_token(t):
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
|
|
|
| 285 |
return outs
|
| 286 |
|
| 287 |
S = realize_np(subs, P_SUBJ)
|
| 288 |
O = realize_np(objs + obls, P_OBJ)
|
| 289 |
ADV=[]
|
| 290 |
for a in advs:
|
| 291 |
-
if not is_content_token(a):
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
ADV.append(code_es(lem, "Kōmín-CJK") if src_lang=="Español" else code_en(lem, "Kōmín-CJK"))
|
| 295 |
|
| 296 |
v_form = vcode + TAM + (NEG_M if is_neg else "")
|
| 297 |
|
| 298 |
-
if zero_copula and vlem in ("ser","estar","be") and tense=="Pres" and not is_neg and not is_q:
|
| 299 |
parts = S + O + ADV
|
| 300 |
else:
|
| 301 |
parts = S + O + ADV + [v_form]
|
|
@@ -303,18 +312,84 @@ def realize_komin(doc, src_lang: str, drop_articles=True, zero_copula=True):
|
|
| 303 |
if is_q: out += " " + Q_FIN
|
| 304 |
return out
|
| 305 |
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
def encode_simple(text: str, src_lang: str, target: str) -> str:
|
| 319 |
if not text.strip(): return ""
|
| 320 |
def repl_es(m):
|
|
@@ -330,16 +405,15 @@ def encode_simple(text: str, src_lang: str, target: str) -> str:
|
|
| 330 |
repl = repl_es if src_lang=="Español" else repl_en
|
| 331 |
return WORD_RE.sub(repl, text)
|
| 332 |
|
| 333 |
-
# --- DECODER bidireccional (pela marcadores/partículas) ---
|
| 334 |
def decode_simple(text: str, source: str, tgt_lang: str) -> str:
|
| 335 |
if not text.strip():
|
| 336 |
return ""
|
| 337 |
code2es = MINI2ES if source=="Minimax-ASCII" else KOMI2ES
|
| 338 |
code2en = MINI2EN if source=="Minimax-ASCII" else KOMI2EN
|
| 339 |
|
| 340 |
-
mini_tail_re = re.compile(r"^(?P<stem>.+?)·(?P<tail>[
|
| 341 |
komi_np_re = re.compile(r"(?P<stem>.+?)(?:[ᵖᵒ])$")
|
| 342 |
-
komi_tam_re = re.compile(r"^(?P<stem>.+?)[ⓅⓉⒻ]
|
| 343 |
|
| 344 |
tokens = re.split(r"(\s+|[^\w\s]+)", text)
|
| 345 |
out=[]
|
|
@@ -356,6 +430,8 @@ def decode_simple(text: str, source: str, tgt_lang: str) -> str:
|
|
| 356 |
look = part
|
| 357 |
|
| 358 |
if source=="Minimax-ASCII":
|
|
|
|
|
|
|
| 359 |
w_es = code2es.get(look)
|
| 360 |
w_en = code2en.get(look) if code2en else None
|
| 361 |
if tgt_lang=="Español" and w_es: out.append(w_es); continue
|
|
@@ -372,8 +448,8 @@ def decode_simple(text: str, source: str, tgt_lang: str) -> str:
|
|
| 372 |
out.append(look); continue
|
| 373 |
|
| 374 |
else: # Kōmín-CJK
|
| 375 |
-
if look.endswith("?"):
|
| 376 |
-
|
| 377 |
|
| 378 |
w_es = code2es.get(look)
|
| 379 |
w_en = code2en.get(look) if code2en else None
|
|
@@ -404,48 +480,135 @@ def decode_simple(text: str, source: str, tgt_lang: str) -> str:
|
|
| 404 |
|
| 405 |
return "".join(out)
|
| 406 |
|
| 407 |
-
# ------------
|
| 408 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
if not text.strip(): return ""
|
| 410 |
-
if not USE_SPACY: return "[Error] spaCy no está cargado (revisa requirements.txt)."
|
| 411 |
-
nlp = nlp_es if src_lang=="Español" else nlp_en
|
| 412 |
-
doc = nlp(text)
|
| 413 |
-
out=[]
|
| 414 |
-
for t in doc:
|
| 415 |
-
if not t.is_alpha:
|
| 416 |
-
out.append(t.text); continue
|
| 417 |
-
lem = lemma_of(t, src_lang)
|
| 418 |
-
if src_lang=="Español":
|
| 419 |
-
tr = ES2EN_LEMMA.get(lem)
|
| 420 |
-
out.append(tr if tr else lem)
|
| 421 |
-
else:
|
| 422 |
-
tr = EN2ES_LEMMA.get(lem)
|
| 423 |
-
out.append(tr if tr else lem)
|
| 424 |
-
return " ".join(out)
|
| 425 |
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
return "", ""
|
| 430 |
-
conlang_out = build_sentence(text, src_lang, target, drop_articles, zero_copula)
|
| 431 |
-
back_out = decode_simple(conlang_out, target, src_lang)
|
| 432 |
-
return conlang_out, back_out
|
| 433 |
|
| 434 |
-
|
| 435 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
|
| 437 |
def universal_translate(text: str, src: str, tgt: str,
|
| 438 |
-
drop_articles: bool, zero_copula: bool
|
|
|
|
| 439 |
if not text.strip(): return ""
|
| 440 |
if src == tgt: return text
|
| 441 |
|
| 442 |
# Natural → Conlang
|
| 443 |
if src in ("Español","English") and tgt in ("Minimax-ASCII","Kōmín-CJK"):
|
| 444 |
-
return build_sentence(text, src, tgt, drop_articles, zero_copula)
|
| 445 |
|
| 446 |
-
# Conlang → Natural
|
| 447 |
if src in ("Minimax-ASCII","Kōmín-CJK") and tgt in ("Español","English"):
|
| 448 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
|
| 450 |
# Natural ↔ Natural (lemas)
|
| 451 |
if src in ("Español","English") and tgt in ("Español","English"):
|
|
@@ -453,8 +616,11 @@ def universal_translate(text: str, src: str, tgt: str,
|
|
| 453 |
|
| 454 |
# Conlang ↔ Conlang
|
| 455 |
if src in ("Minimax-ASCII","Kōmín-CJK") and tgt in ("Minimax-ASCII","Kōmín-CJK"):
|
| 456 |
-
#
|
| 457 |
-
|
|
|
|
|
|
|
|
|
|
| 458 |
words = re.findall(r"\w+|[^\w\s]+", es_lemmas)
|
| 459 |
out=[]
|
| 460 |
for w in words:
|
|
@@ -465,34 +631,65 @@ def universal_translate(text: str, src: str, tgt: str,
|
|
| 465 |
out.append(code)
|
| 466 |
else:
|
| 467 |
out.append(w)
|
| 468 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 469 |
|
| 470 |
return "[No soportado]"
|
| 471 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
# ------------ UI y explicaciones ------------
|
| 473 |
EXPLAIN_ES = """
|
| 474 |
-
**
|
| 475 |
-
- **
|
| 476 |
-
- **
|
| 477 |
-
- **
|
| 478 |
-
- **
|
| 479 |
-
|
|
|
|
|
|
|
|
|
|
| 480 |
"""
|
|
|
|
| 481 |
EXPLAIN_EN = """
|
| 482 |
-
**
|
| 483 |
-
- **
|
| 484 |
-
- **
|
| 485 |
-
- **
|
| 486 |
-
- **
|
| 487 |
-
- We process **only the predicative sentence** (verbal ROOT) to ignore greetings/fillers.
|
| 488 |
"""
|
| 489 |
|
| 490 |
-
|
|
|
|
|
|
|
| 491 |
gr.Markdown("# Universal Conlang Translator · Minimax-ASCII / Kōmín-CJK")
|
| 492 |
with gr.Row():
|
| 493 |
gr.Markdown(EXPLAIN_ES)
|
| 494 |
gr.Markdown(EXPLAIN_EN)
|
| 495 |
|
|
|
|
|
|
|
| 496 |
# --- Traducir (universal) ---
|
| 497 |
with gr.Tab("Traducir (universal)"):
|
| 498 |
with gr.Row():
|
|
@@ -502,14 +699,15 @@ with gr.Blocks(title="Universal Conlang Translator (Reversible)") as demo:
|
|
| 502 |
with gr.Row():
|
| 503 |
uni_drop = gr.Checkbox(value=True, label="Omitir artículos (ES/EN→conlang)")
|
| 504 |
uni_zero = gr.Checkbox(value=True, label="Cópula cero (presente afirm.) (ES/EN→conlang)")
|
|
|
|
| 505 |
uni_out = gr.Textbox(lines=6, label="Traducción")
|
| 506 |
gr.Button("Traducir").click(
|
| 507 |
-
universal_translate,
|
| 508 |
-
[uni_text, uni_src, uni_tgt, uni_drop, uni_zero
|
| 509 |
[uni_out]
|
| 510 |
)
|
| 511 |
|
| 512 |
-
# --- Construir frase (
|
| 513 |
with gr.Tab("Construir frase (ES/EN → Conlang)"):
|
| 514 |
with gr.Row():
|
| 515 |
src_lang = gr.Dropdown(["Español","English"], value="Español", label="Fuente")
|
|
@@ -518,40 +716,54 @@ with gr.Blocks(title="Universal Conlang Translator (Reversible)") as demo:
|
|
| 518 |
with gr.Row():
|
| 519 |
drop_articles = gr.Checkbox(value=True, label="Omitir artículos")
|
| 520 |
zero_copula = gr.Checkbox(value=True, label="Cópula cero (presente afirm.)")
|
|
|
|
| 521 |
out = gr.Textbox(lines=6, label="Salida")
|
| 522 |
-
gr.Button("Construir").click(
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
src_lang2 = gr.Dropdown(["Español","English"], value="Español", label="Fuente")
|
| 528 |
-
target2 = gr.Dropdown(["Minimax-ASCII","Kōmín-CJK"], value="Minimax-ASCII", label="Conlang")
|
| 529 |
-
text_in2 = gr.Textbox(lines=3, label="Texto")
|
| 530 |
-
out2 = gr.Textbox(lines=6, label="Salida")
|
| 531 |
-
gr.Button("Codificar").click(encode_simple, [text_in2, src_lang2, target2], [out2])
|
| 532 |
|
|
|
|
| 533 |
with gr.Tab("Decodificar (Conlang → ES/EN)"):
|
| 534 |
with gr.Row():
|
| 535 |
src_code = gr.Dropdown(["Minimax-ASCII","Kōmín-CJK"], value="Minimax-ASCII", label="Fuente")
|
| 536 |
tgt_lang = gr.Dropdown(["Español","English"], value="Español", label="Destino")
|
| 537 |
-
code_in = gr.Textbox(lines=3, label="Texto en conlang")
|
| 538 |
out3 = gr.Textbox(lines=6, label="Salida")
|
| 539 |
-
gr.Button("Decodificar").click(decode_simple, [code_in, src_code, tgt_lang], [out3])
|
| 540 |
|
| 541 |
-
|
| 542 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 543 |
with gr.Row():
|
| 544 |
-
rt_src = gr.Dropdown(["Español","English"], value="Español", label="Fuente
|
| 545 |
rt_tgt = gr.Dropdown(["Minimax-ASCII","Kōmín-CJK"], value="Minimax-ASCII", label="Conlang")
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
rt_drop = gr.Checkbox(value=True, label="Omitir artículos")
|
| 549 |
-
rt_zero = gr.Checkbox(value=True, label="Cópula cero (presente afirm.)")
|
| 550 |
rt_out_conlang = gr.Textbox(lines=3, label="Conlang (ida)")
|
| 551 |
-
rt_out_back = gr.Textbox(lines=3, label="Vuelta
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 555 |
[rt_out_conlang, rt_out_back]
|
| 556 |
)
|
| 557 |
|
|
@@ -565,4 +777,3 @@ if __name__ == "__main__":
|
|
| 565 |
|
| 566 |
|
| 567 |
|
| 568 |
-
|
|
|
|
| 1 |
+
# app.py — Universal Conlang Translator (Compacto / Semi-lossless / Lossless / Referencia)
|
| 2 |
+
# Archivos necesarios en la raíz:
|
| 3 |
+
# - lexicon_minimax.json
|
| 4 |
+
# - lexicon_komin.json
|
| 5 |
+
# - lexicon_master.json
|
| 6 |
+
#
|
| 7 |
+
# requirements.txt (mínimo):
|
| 8 |
+
# gradio>=4.36.0
|
| 9 |
+
# (Opcional para análisis mejorado)
|
| 10 |
+
# spacy>=3.7.4
|
| 11 |
+
# https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.7.0/es_core_news_sm-3.7.0-py3-none-any.whl
|
| 12 |
+
# https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
|
| 13 |
+
|
| 14 |
+
import os, re, json, base64, zlib, hashlib
|
| 15 |
+
from typing import Dict, Tuple, Optional
|
| 16 |
import gradio as gr
|
| 17 |
|
| 18 |
# ------------ Archivos esperados ------------
|
| 19 |
+
LEX_MINI = "lexicon_minimax.json"
|
| 20 |
+
LEX_KOMI = "lexicon_komin.json"
|
| 21 |
+
LEX_MASTER = "lexicon_master.json"
|
| 22 |
|
| 23 |
# ------------ Normalización ------------
|
| 24 |
WORD_RE = re.compile(r"[A-Za-zÁÉÍÓÚÜÑáéíóúüñ]+", re.UNICODE)
|
|
|
|
| 74 |
EN2MINI, EN2KOMI, MINI2EN, KOMI2EN,
|
| 75 |
ES2EN_LEMMA, EN2ES_LEMMA) = load_lexicons()
|
| 76 |
|
| 77 |
+
# ------------ OOV reversible (modo Compacto/Semi-lossless) ------------
|
| 78 |
ALPHA_MINI64 = "@ptkmnslraeiouy0123456789><=:/!?.+-_*#bcdfghjvqwxzACEGHIJKLMNOPRS"[:64]
|
| 79 |
CJK_BASE = (
|
| 80 |
"天地人日月山川雨風星火水木土金石光影花草鳥犬猫魚"
|
|
|
|
| 118 |
def is_oov_komin(code: str) -> bool:
|
| 119 |
return len(code) >= 2 and code.startswith("「") and code.endswith("」")
|
| 120 |
|
| 121 |
+
# ------------ spaCy opcional ------------
|
| 122 |
USE_SPACY = False
|
| 123 |
try:
|
| 124 |
import spacy
|
|
|
|
| 137 |
else:
|
| 138 |
return norm_en(tok.lemma_ if tok.lemma_ else tok.text)
|
| 139 |
|
| 140 |
+
# ------------ Selección de oración predicativa ------------
|
| 141 |
def pick_predicative_sentence(doc):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
sents = list(doc.sents) if doc.has_annotation("SENT_START") else [doc]
|
| 143 |
candidates = []
|
| 144 |
for s in sents:
|
|
|
|
| 154 |
return doc
|
| 155 |
return sorted(candidates, key=lambda x: x[0], reverse=True)[0][1].as_doc()
|
| 156 |
|
|
|
|
| 157 |
def is_content_token(t) -> bool:
|
|
|
|
| 158 |
if t.pos_ in ("INTJ", "DET", "ADP", "SCONJ", "CCONJ", "PART", "SYM", "PUNCT"):
|
| 159 |
return False
|
|
|
|
| 160 |
if t.dep_ in ("discourse", "intj", "vocative", "dep"):
|
| 161 |
return False
|
|
|
|
| 162 |
low = t.lower_.strip("¿?¡!.,;:()[]{}\"'").lower()
|
| 163 |
if t.tag_.startswith("W") or low in {
|
| 164 |
"como","cómo","que","qué","quien","quién","donde","dónde","cuando","cuándo",
|
| 165 |
+
"porqué","por","por qué","cuanto","cuánto",
|
| 166 |
+
"which","what","who","where","when","why","how",
|
| 167 |
"hola","hello","hi","hey","ey","eh","oye","adios","adiós","ciao"
|
| 168 |
}:
|
| 169 |
return False
|
|
|
|
| 186 |
if EN2KOMI: return EN2KOMI.get(lemma) or enc_oov_komin(lemma)
|
| 187 |
return enc_oov_komin(lemma)
|
| 188 |
|
| 189 |
+
# ------------ Fraseador compacto ------------
|
| 190 |
TAM_MINI = {"Pres":"P", "Past":"T", "Fut":"F", "UNK":"P"}
|
| 191 |
TAM_KOMI = {"Pres":"Ⓟ", "Past":"Ⓣ", "Fut":"Ⓕ", "UNK":"Ⓟ"}
|
| 192 |
|
|
|
|
| 208 |
if c.pos_ == "AUX":
|
| 209 |
cm = c.morph
|
| 210 |
if "Tense=Past" in cm: return "Past"
|
| 211 |
+
# inglés "will"
|
| 212 |
+
if getattr(c, "tag_", "") == "MD" and c.lower_ == "will": return "Fut"
|
| 213 |
return "Pres"
|
| 214 |
|
| 215 |
def extract_core(doc):
|
|
|
|
| 228 |
obls.sort(key=lambda x: x.i); advs.sort(key=lambda x: x.i)
|
| 229 |
return root, subs, objs, obls, advs
|
| 230 |
|
| 231 |
+
def realize_minimax(doc, src_lang: str, drop_articles=True, zero_copula=True, semi_lossless=False, person_hint="2s"):
|
|
|
|
| 232 |
root, subs, objs, obls, advs = extract_core(doc)
|
| 233 |
tense, is_q, is_neg = detect_tense(root), detect_polarity(doc), detect_neg(doc)
|
| 234 |
|
| 235 |
+
vlem = lemma_of(root, src_lang) if USE_SPACY else ("ser" if "?" in doc.text else "estar")
|
| 236 |
vcode = code_es(vlem, "Minimax-ASCII") if src_lang=="Español" else code_en(vlem, "Minimax-ASCII")
|
| 237 |
|
| 238 |
tail = TAM_MINI.get(tense, "P")
|
| 239 |
+
if semi_lossless:
|
| 240 |
+
tail += _person_of_doc(doc, src_lang) or person_hint # añade 1s,2s,... si es posible
|
| 241 |
if is_neg: tail += "N"
|
| 242 |
if is_q: tail += "Q"
|
| 243 |
if tail:
|
| 244 |
+
vcode = f"{vcode}·{tail}"
|
| 245 |
|
| 246 |
def realize_np(tokens):
|
| 247 |
outs=[]
|
| 248 |
for t in tokens:
|
| 249 |
+
if not USE_SPACY or is_content_token(t):
|
| 250 |
+
lem = lemma_of(t, src_lang) if USE_SPACY else (t.text)
|
| 251 |
+
code = code_es(lem, "Minimax-ASCII") if src_lang=="Español" else code_en(lem, "Minimax-ASCII")
|
| 252 |
+
if semi_lossless and USE_SPACY and t.tag_ in ("NNS","NNPS") or ("Number=Plur" in t.morph):
|
| 253 |
+
code = f"{code}[PL]"
|
| 254 |
+
outs.append(code)
|
| 255 |
return outs
|
| 256 |
|
| 257 |
S = realize_np(subs)
|
| 258 |
O = realize_np(objs) + realize_np(obls)
|
| 259 |
ADV=[]
|
| 260 |
for a in advs:
|
| 261 |
+
if not USE_SPACY or is_content_token(a):
|
| 262 |
+
lem = lemma_of(a, src_lang) if USE_SPACY else a.text
|
| 263 |
+
ADV.append(code_es(lem, "Minimax-ASCII") if src_lang=="Español" else code_en(lem, "Minimax-ASCII"))
|
|
|
|
| 264 |
|
| 265 |
+
if zero_copula and vlem in ("ser","estar","be") and tense=="Pres" and not is_neg and not is_q and not semi_lossless:
|
|
|
|
| 266 |
parts = S + O + ADV
|
| 267 |
else:
|
| 268 |
parts = [vcode] + S + O + ADV
|
| 269 |
return " ".join(p for p in parts if p)
|
| 270 |
|
| 271 |
+
def realize_komin(doc, src_lang: str, drop_articles=True, zero_copula=True, semi_lossless=False, person_hint="2s"):
|
|
|
|
| 272 |
root, subs, objs, obls, advs = extract_core(doc)
|
| 273 |
tense, is_q, is_neg = detect_tense(root), detect_polarity(doc), detect_neg(doc)
|
| 274 |
|
| 275 |
+
vlem = lemma_of(root, src_lang) if USE_SPACY else ("ser" if "?" in doc.text else "estar")
|
| 276 |
vcode = code_es(vlem, "Kōmín-CJK") if src_lang=="Español" else code_en(vlem, "Kōmín-CJK")
|
| 277 |
|
| 278 |
P_SUBJ, P_OBJ = "ᵖ", "ᵒ"
|
| 279 |
NEG_M, Q_FIN = "̆", "?"
|
| 280 |
TAM = TAM_KOMI.get(tense, "Ⓟ")
|
| 281 |
|
| 282 |
+
if semi_lossless:
|
| 283 |
+
pi = _person_of_doc(doc, src_lang) or person_hint
|
| 284 |
+
TAM = TAM + f"[{pi}]"
|
| 285 |
+
|
| 286 |
def realize_np(tokens, particle):
|
| 287 |
outs=[]
|
| 288 |
for t in tokens:
|
| 289 |
+
if not USE_SPACY or is_content_token(t):
|
| 290 |
+
lem = lemma_of(t, src_lang) if USE_SPACY else t.text
|
| 291 |
+
code = code_es(lem, "Kōmín-CJK") if src_lang=="Español" else code_en(lem, "Kōmín-CJK")
|
| 292 |
+
if semi_lossless and USE_SPACY and (t.tag_ in ("NNS","NNPS") or "Number=Plur" in t.morph):
|
| 293 |
+
code = f"{code}[PL]"
|
| 294 |
+
outs.append(code + particle)
|
| 295 |
return outs
|
| 296 |
|
| 297 |
S = realize_np(subs, P_SUBJ)
|
| 298 |
O = realize_np(objs + obls, P_OBJ)
|
| 299 |
ADV=[]
|
| 300 |
for a in advs:
|
| 301 |
+
if not USE_SPACY or is_content_token(a):
|
| 302 |
+
lem = lemma_of(a, src_lang) if USE_SPACY else a.text
|
| 303 |
+
ADV.append(code_es(lem, "Kōmín-CJK") if src_lang=="Español" else code_en(lem, "Kōmín-CJK"))
|
|
|
|
| 304 |
|
| 305 |
v_form = vcode + TAM + (NEG_M if is_neg else "")
|
| 306 |
|
| 307 |
+
if zero_copula and vlem in ("ser","estar","be") and tense=="Pres" and not is_neg and not is_q and not semi_lossless:
|
| 308 |
parts = S + O + ADV
|
| 309 |
else:
|
| 310 |
parts = S + O + ADV + [v_form]
|
|
|
|
| 312 |
if is_q: out += " " + Q_FIN
|
| 313 |
return out
|
| 314 |
|
| 315 |
+
def _person_of_doc(doc, src_lang: str) -> Optional[str]:
|
| 316 |
+
# Heurística simple de persona/número del sujeto
|
| 317 |
+
try:
|
| 318 |
+
root = next((t for t in doc if t.dep_=="ROOT"), doc[0])
|
| 319 |
+
subj = next((t for t in root.children if t.dep_.startswith("nsubj")), None)
|
| 320 |
+
if subj is None: return None
|
| 321 |
+
# singular/plural
|
| 322 |
+
plur = ("Number=Plur" in subj.morph) if src_lang=="Español" else (subj.tag_ in ("NNS","NNPS"))
|
| 323 |
+
# persona (heurística): pronombres personales
|
| 324 |
+
low = subj.lower_
|
| 325 |
+
if src_lang=="Español":
|
| 326 |
+
if low in ("yo",): return "1p" if plur else "1s"
|
| 327 |
+
if low in ("tú","vos"): return "2p" if plur else "2s"
|
| 328 |
+
if low in ("usted","él","ella"): return "3p" if plur else "3s"
|
| 329 |
+
# lematiza pronombres
|
| 330 |
+
lem = lemma_of(subj, "Español")
|
| 331 |
+
if lem in ("yo","nosotros"): return "1p" if plur else "1s"
|
| 332 |
+
if lem in ("tú","vosotros"): return "2p" if plur else "2s"
|
| 333 |
+
return "3p" if plur else "3s"
|
| 334 |
+
else:
|
| 335 |
+
if low in ("i",): return "1p" if plur else "1s"
|
| 336 |
+
if low in ("you",): return "2p" if plur else "2s"
|
| 337 |
+
if low in ("he","she","it"): return "3p" if plur else "3s"
|
| 338 |
+
return "3p" if plur else "3s"
|
| 339 |
+
except Exception:
|
| 340 |
+
return None
|
| 341 |
+
|
| 342 |
+
# ------------ Lossless (Base85 comprimido) ------------
|
| 343 |
+
SIDECAR_B85_RE = re.compile(r"\s?§\((?P<b85>[A-Za-z0-9!#$%&()*+\-;<=>?@^_`{|}~]+)\)$")
|
| 344 |
+
|
| 345 |
+
def b85_enc_raw(s: str) -> str:
|
| 346 |
+
comp = zlib.compress(s.encode("utf-8"), 9)
|
| 347 |
+
return base64.a85encode(comp, adobe=False).decode("ascii")
|
| 348 |
+
|
| 349 |
+
def b85_dec_raw(b85s: str) -> str:
|
| 350 |
+
comp = base64.a85decode(b85s.encode("ascii"), adobe=False)
|
| 351 |
+
return zlib.decompress(comp).decode("utf-8")
|
| 352 |
+
|
| 353 |
+
def attach_sidecar_b85(conlang_text: str, original_text: str) -> str:
|
| 354 |
+
blob = b85_enc_raw(original_text)
|
| 355 |
+
return f"{conlang_text} §({blob})"
|
| 356 |
+
|
| 357 |
+
def extract_sidecar_b85(text: str) -> Optional[str]:
|
| 358 |
+
m = SIDECAR_B85_RE.search(text)
|
| 359 |
+
if not m: return None
|
| 360 |
+
try:
|
| 361 |
+
return b85_dec_raw(m.group("b85"))
|
| 362 |
+
except Exception:
|
| 363 |
+
return None
|
| 364 |
|
| 365 |
+
def strip_sidecar(text: str) -> str:
|
| 366 |
+
return SIDECAR_B85_RE.sub("", text).rstrip()
|
| 367 |
+
|
| 368 |
+
# ------------ Referencia (hash→original en memoria) ------------
|
| 369 |
+
REFCACHE: Dict[str,str] = {}
|
| 370 |
+
SIDECAR_REF_RE = re.compile(r"\s?§\{(?P<ref>[A-Z2-7]+)\}$") # base32
|
| 371 |
+
|
| 372 |
+
def make_ref_id(text: str, length: int = 16) -> str:
|
| 373 |
+
# BLAKE2s → base32 (A-Z2-7), truncado
|
| 374 |
+
digest = hashlib.blake2s(text.encode("utf-8")).digest()
|
| 375 |
+
b32 = base64.b32encode(digest).decode("ascii").rstrip("=")
|
| 376 |
+
return b32[:length]
|
| 377 |
+
|
| 378 |
+
def attach_sidecar_ref(conlang_text: str, original_text: str) -> str:
|
| 379 |
+
rid = make_ref_id(original_text)
|
| 380 |
+
REFCACHE[rid] = original_text
|
| 381 |
+
return f"{conlang_text} §{{{rid}}}"
|
| 382 |
+
|
| 383 |
+
def extract_sidecar_ref(text: str) -> Optional[str]:
|
| 384 |
+
m = SIDECAR_REF_RE.search(text)
|
| 385 |
+
if not m: return None
|
| 386 |
+
rid = m.group("ref")
|
| 387 |
+
return REFCACHE.get(rid)
|
| 388 |
+
|
| 389 |
+
def strip_sidecar_ref(text: str) -> str:
|
| 390 |
+
return SIDECAR_REF_RE.sub("", text).rstrip()
|
| 391 |
+
|
| 392 |
+
# ------------ Codificar / Decodificar léxico puro ------------
|
| 393 |
def encode_simple(text: str, src_lang: str, target: str) -> str:
|
| 394 |
if not text.strip(): return ""
|
| 395 |
def repl_es(m):
|
|
|
|
| 405 |
repl = repl_es if src_lang=="Español" else repl_en
|
| 406 |
return WORD_RE.sub(repl, text)
|
| 407 |
|
|
|
|
| 408 |
def decode_simple(text: str, source: str, tgt_lang: str) -> str:
|
| 409 |
if not text.strip():
|
| 410 |
return ""
|
| 411 |
code2es = MINI2ES if source=="Minimax-ASCII" else KOMI2ES
|
| 412 |
code2en = MINI2EN if source=="Minimax-ASCII" else KOMI2EN
|
| 413 |
|
| 414 |
+
mini_tail_re = re.compile(r"^(?P<stem>.+?)·(?P<tail>[PTFNQ12sp]+)$")
|
| 415 |
komi_np_re = re.compile(r"(?P<stem>.+?)(?:[ᵖᵒ])$")
|
| 416 |
+
komi_tam_re = re.compile(r"^(?P<stem>.+?)[ⓅⓉⒻ](?:\[[123]s|[123]p\])?(?:\u0306)?$")
|
| 417 |
|
| 418 |
tokens = re.split(r"(\s+|[^\w\s]+)", text)
|
| 419 |
out=[]
|
|
|
|
| 430 |
look = part
|
| 431 |
|
| 432 |
if source=="Minimax-ASCII":
|
| 433 |
+
# quitar marcas semilossless de plural en nombres
|
| 434 |
+
look = look.replace("[PL]", "")
|
| 435 |
w_es = code2es.get(look)
|
| 436 |
w_en = code2en.get(look) if code2en else None
|
| 437 |
if tgt_lang=="Español" and w_es: out.append(w_es); continue
|
|
|
|
| 448 |
out.append(look); continue
|
| 449 |
|
| 450 |
else: # Kōmín-CJK
|
| 451 |
+
if look.endswith("?"): look = look[:-1]
|
| 452 |
+
look = re.sub(r"\[PL\]$", "", look) # semilossless plural
|
| 453 |
|
| 454 |
w_es = code2es.get(look)
|
| 455 |
w_en = code2en.get(look) if code2en else None
|
|
|
|
| 480 |
|
| 481 |
return "".join(out)
|
| 482 |
|
| 483 |
+
# ------------ Conjugadores mínimos para "naturalizar" (semi-lossless) ------------
|
| 484 |
+
_ES_SUBJ = {"1s":"yo","2s":"tú","3s":"él/ella","1p":"nosotros","2p":"vosotros","3p":"ellos"}
|
| 485 |
+
_EN_SUBJ = {"1s":"I","2s":"you","3s":"he","1p":"we","2p":"you","3p":"they"}
|
| 486 |
+
|
| 487 |
+
def _es_conj_regular(lemma, tense, person):
|
| 488 |
+
if not lemma.endswith(("ar","er","ir")): return lemma
|
| 489 |
+
stem = lemma[:-2]; vtype = lemma[-2:]
|
| 490 |
+
pres = {
|
| 491 |
+
"ar": {"1s":"o","2s":"as","3s":"a","1p":"amos","2p":"áis","3p":"an"},
|
| 492 |
+
"er": {"1s":"o","2s":"es","3s":"e","1p":"emos","2p":"éis","3p":"en"},
|
| 493 |
+
"ir": {"1s":"o","2s":"es","3s":"e","1p":"imos","2p":"ís","3p":"en"},
|
| 494 |
+
}
|
| 495 |
+
pret = {
|
| 496 |
+
"ar": {"1s":"é","2s":"aste","3s":"ó","1p":"amos","2p":"asteis","3p":"aron"},
|
| 497 |
+
"er": {"1s":"í","2s":"iste","3s":"ió","1p":"imos","2p":"isteis","3p":"ieron"},
|
| 498 |
+
"ir": {"1s":"í","2s":"iste","3s":"ió","1p":"imos","2p":"isteis","3p":"ieron"},
|
| 499 |
+
}
|
| 500 |
+
fut = {"1s":"é","2s":"ás","3s":"á","1p":"emos","2p":"éis","3p":"án"}
|
| 501 |
+
if tense == "Pres": return stem + pres[vtype].get(person, pres[vtype]["3s"])
|
| 502 |
+
if tense == "Past": return stem + pret[vtype].get(person, pret[vtype]["3s"])
|
| 503 |
+
return lemma + fut.get(person, fut["3s"])
|
| 504 |
+
|
| 505 |
+
def _es_conj(lemma, tense, person):
|
| 506 |
+
if lemma == "ser":
|
| 507 |
+
tab = {
|
| 508 |
+
"Pres":{"1s":"soy","2s":"eres","3s":"es","1p":"somos","2p":"sois","3p":"son"},
|
| 509 |
+
"Past":{"1s":"fui","2s":"fuiste","3s":"fue","1p":"fuimos","2p":"fuisteis","3p":"fueron"},
|
| 510 |
+
"Fut":{"1s":"seré","2s":"serás","3s":"será","1p":"seremos","2p":"seréis","3p":"serán"},
|
| 511 |
+
}; return tab[tense][person]
|
| 512 |
+
if lemma == "estar":
|
| 513 |
+
tab = {
|
| 514 |
+
"Pres":{"1s":"estoy","2s":"estás","3s":"está","1p":"estamos","2p":"estáis","3p":"están"},
|
| 515 |
+
"Past":{"1s":"estuve","2s":"estuviste","3s":"estuvo","1p":"estuvimos","2p":"estuvisteis","3p":"estuvieron"},
|
| 516 |
+
"Fut":{"1s":"estaré","2s":"estarás","3s":"estará","1p":"estaremos","2p":"estaréis","3p":"estarán"},
|
| 517 |
+
}; return tab[tense][person]
|
| 518 |
+
if lemma == "ir":
|
| 519 |
+
tab = {
|
| 520 |
+
"Pres":{"1s":"voy","2s":"vas","3s":"va","1p":"vamos","2p":"vais","3p":"van"},
|
| 521 |
+
"Past":{"1s":"fui","2s":"fuiste","3s":"fue","1p":"fuimos","2p":"fuisteis","3p":"fueron"},
|
| 522 |
+
"Fut":{"1s":"iré","2s":"irás","3s":"irá","1p":"iremos","2p":"iréis","3p":"irán"},
|
| 523 |
+
}; return tab[tense][person]
|
| 524 |
+
return _es_conj_regular(lemma, tense, person)
|
| 525 |
+
|
| 526 |
+
def _en_conj(lemma, tense, person):
|
| 527 |
+
if lemma == "be":
|
| 528 |
+
if tense == "Pres":
|
| 529 |
+
return {"1s":"am","2s":"are","3s":"is","1p":"are","2p":"are","3p":"are"}[person]
|
| 530 |
+
if tense == "Past":
|
| 531 |
+
return {"1s":"was","2s":"were","3s":"was","1p":"were","2p":"were","3p":"were"}[person]
|
| 532 |
+
return "be"
|
| 533 |
+
if lemma == "have":
|
| 534 |
+
if tense == "Pres": return "has" if person=="3s" else "have"
|
| 535 |
+
if tense == "Past": return "had"
|
| 536 |
+
return "have"
|
| 537 |
+
if lemma == "go":
|
| 538 |
+
if tense == "Past": return "went"
|
| 539 |
+
return "does" if (tense=="Pres" and person=="3s") else "go"
|
| 540 |
+
if lemma == "do":
|
| 541 |
+
if tense == "Past": return "did"
|
| 542 |
+
return "does" if (tense=="Pres" and person=="3s") else "do"
|
| 543 |
+
|
| 544 |
+
if tense == "Pres":
|
| 545 |
+
if person == "3s":
|
| 546 |
+
if lemma.endswith("y") and (len(lemma)<2 or lemma[-2] not in "aeiou"):
|
| 547 |
+
return lemma[:-1] + "ies"
|
| 548 |
+
if lemma.endswith(("s","sh","ch","x","z","o")):
|
| 549 |
+
return lemma + "es"
|
| 550 |
+
return lemma + "s"
|
| 551 |
+
return lemma
|
| 552 |
+
elif tense == "Past":
|
| 553 |
+
if lemma.endswith("e"): return lemma + "d"
|
| 554 |
+
if lemma.endswith("y") and (len(lemma)<2 or lemma[-2] not in "aeiou"): return lemma[:-1] + "ied"
|
| 555 |
+
return lemma + "ed"
|
| 556 |
+
else:
|
| 557 |
+
return lemma # future -> will + base
|
| 558 |
+
|
| 559 |
+
# ------------ Lossless / Semi-lossless / Compacto (rutas) ------------
|
| 560 |
+
MODES = ["Compacto", "Semi-lossless", "Lossless (b85)", "Referencia (hash)"]
|
| 561 |
+
|
| 562 |
+
def build_sentence(text: str, src_lang: str, target: str,
|
| 563 |
+
drop_articles: bool, zero_copula: bool, mode: str) -> str:
|
| 564 |
if not text.strip(): return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 565 |
|
| 566 |
+
if mode == "Lossless (b85)":
|
| 567 |
+
core = encode_simple(text, src_lang, target) if not USE_SPACY else _build_with_spacy(text, src_lang, target, drop_articles, zero_copula, semi_lossless=False)
|
| 568 |
+
return attach_sidecar_b85(core, text)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 569 |
|
| 570 |
+
if mode == "Referencia (hash)":
|
| 571 |
+
core = encode_simple(text, src_lang, target) if not USE_SPACY else _build_with_spacy(text, src_lang, target, drop_articles, zero_copula, semi_lossless=False)
|
| 572 |
+
return attach_sidecar_ref(core, text)
|
| 573 |
+
|
| 574 |
+
# Compacto / Semi-lossless
|
| 575 |
+
semi = (mode == "Semi-lossless")
|
| 576 |
+
if USE_SPACY:
|
| 577 |
+
return _build_with_spacy(text, src_lang, target, drop_articles, zero_copula, semi_lossless=semi)
|
| 578 |
+
else:
|
| 579 |
+
# sin spaCy -> léxico puro (semi-lossless no aplica)
|
| 580 |
+
return encode_simple(text, src_lang, target)
|
| 581 |
+
|
| 582 |
+
def _build_with_spacy(text: str, src_lang: str, target: str,
|
| 583 |
+
drop_articles: bool, zero_copula: bool, semi_lossless: bool) -> str:
|
| 584 |
+
nlp = nlp_es if src_lang=="Español" else nlp_en
|
| 585 |
+
doc_full = nlp(text)
|
| 586 |
+
doc = pick_predicative_sentence(doc_full)
|
| 587 |
+
if target == "Minimax-ASCII":
|
| 588 |
+
return realize_minimax(doc, src_lang, drop_articles, zero_copula, semi_lossless=semi_lossless)
|
| 589 |
+
else:
|
| 590 |
+
return realize_komin(doc, src_lang, drop_articles, zero_copula, semi_lossless=semi_lossless)
|
| 591 |
|
| 592 |
def universal_translate(text: str, src: str, tgt: str,
|
| 593 |
+
drop_articles: bool, zero_copula: bool,
|
| 594 |
+
mode: str) -> str:
|
| 595 |
if not text.strip(): return ""
|
| 596 |
if src == tgt: return text
|
| 597 |
|
| 598 |
# Natural → Conlang
|
| 599 |
if src in ("Español","English") and tgt in ("Minimax-ASCII","Kōmín-CJK"):
|
| 600 |
+
return build_sentence(text, src, tgt, drop_articles, zero_copula, mode)
|
| 601 |
|
| 602 |
+
# Conlang → Natural (considera sidecars)
|
| 603 |
if src in ("Minimax-ASCII","Kōmín-CJK") and tgt in ("Español","English"):
|
| 604 |
+
# Referencia (hash) tiene prioridad
|
| 605 |
+
ref = extract_sidecar_ref(text)
|
| 606 |
+
if ref is not None: return ref
|
| 607 |
+
# Lossless b85
|
| 608 |
+
orig = extract_sidecar_b85(text)
|
| 609 |
+
if orig is not None: return orig
|
| 610 |
+
# Compacto / Semi-lossless -> decodificación léxica
|
| 611 |
+
return decode_simple(strip_sidecar(strip_sidecar_ref(text)), src, tgt)
|
| 612 |
|
| 613 |
# Natural ↔ Natural (lemas)
|
| 614 |
if src in ("Español","English") and tgt in ("Español","English"):
|
|
|
|
| 616 |
|
| 617 |
# Conlang ↔ Conlang
|
| 618 |
if src in ("Minimax-ASCII","Kōmín-CJK") and tgt in ("Minimax-ASCII","Kōmín-CJK"):
|
| 619 |
+
# preserva sidecars
|
| 620 |
+
orig_ref = extract_sidecar_ref(text)
|
| 621 |
+
orig_b85 = extract_sidecar_b85(text)
|
| 622 |
+
core = strip_sidecar(strip_sidecar_ref(text))
|
| 623 |
+
es_lemmas = decode_simple(core, src, "Español")
|
| 624 |
words = re.findall(r"\w+|[^\w\s]+", es_lemmas)
|
| 625 |
out=[]
|
| 626 |
for w in words:
|
|
|
|
| 631 |
out.append(code)
|
| 632 |
else:
|
| 633 |
out.append(w)
|
| 634 |
+
out_text = " ".join(out)
|
| 635 |
+
if orig_ref is not None:
|
| 636 |
+
return attach_sidecar_ref(out_text, orig_ref)
|
| 637 |
+
if orig_b85 is not None:
|
| 638 |
+
return attach_sidecar_b85(out_text, orig_b85)
|
| 639 |
+
return out_text
|
| 640 |
|
| 641 |
return "[No soportado]"
|
| 642 |
|
| 643 |
+
# ------------ Natural ↔ Natural (lemas ES↔EN) ------------
|
| 644 |
+
def translate_natural(text: str, src_lang: str, tgt_lang: str) -> str:
|
| 645 |
+
if not text.strip(): return ""
|
| 646 |
+
if not USE_SPACY: return text
|
| 647 |
+
nlp = nlp_es if src_lang=="Español" else nlp_en
|
| 648 |
+
doc = nlp(text)
|
| 649 |
+
out=[]
|
| 650 |
+
for t in doc:
|
| 651 |
+
if not t.is_alpha:
|
| 652 |
+
out.append(t.text); continue
|
| 653 |
+
lem = lemma_of(t, src_lang)
|
| 654 |
+
if src_lang=="Español":
|
| 655 |
+
tr = ES2EN_LEMMA.get(lem)
|
| 656 |
+
out.append(tr if tr else lem)
|
| 657 |
+
else:
|
| 658 |
+
tr = EN2ES_LEMMA.get(lem)
|
| 659 |
+
out.append(tr if tr else lem)
|
| 660 |
+
return " ".join(out)
|
| 661 |
+
|
| 662 |
# ------------ UI y explicaciones ------------
|
| 663 |
EXPLAIN_ES = """
|
| 664 |
+
**Modos**
|
| 665 |
+
- **Compacto**: conlang corto (reversible a *lemas*).
|
| 666 |
+
- **Semi-lossless**: añade persona/tiempo y plural mínimo → vuelve **natural** sin pedir parámetros.
|
| 667 |
+
- **Lossless (b85)**: adjunta `§(…)` comprimido → **inversa exacta**.
|
| 668 |
+
- **Referencia (hash)**: adjunta `§{ID}` y guarda el original en memoria → **exacta** mientras dure la sesión.
|
| 669 |
+
**Conlangs**
|
| 670 |
+
- **Minimax (VSO)**: `verbo·TAM` + `N` (neg) + `Q` (pregunta). Ej.: `estar·PQ`, en semi-lossless `estar·P2sQ`.
|
| 671 |
+
- **Kōmín (SOV)**: ᵖ/ᵒ, TAM `Ⓟ/Ⓣ/Ⓕ`, negación breve `̆`, `?`. En semi-lossless: `Ⓟ[2s]`.
|
| 672 |
+
**OOV** (compacto/semi): `~…` / `「…」` reversibles.
|
| 673 |
"""
|
| 674 |
+
|
| 675 |
EXPLAIN_EN = """
|
| 676 |
+
**Modes**
|
| 677 |
+
- **Compact**: short conlang (reversible to *lemmas*).
|
| 678 |
+
- **Semi-lossless**: adds person/tense and minimal number → **natural** back without parameters.
|
| 679 |
+
- **Lossless (b85)**: appends `§(…)` compressed → **exact inverse**.
|
| 680 |
+
- **Reference (hash)**: appends `§{ID}` and keeps original in memory → **exact** while the session lives.
|
|
|
|
| 681 |
"""
|
| 682 |
|
| 683 |
+
ALL_LANGS = ["Español","English","Minimax-ASCII","Kōmín-CJK"]
|
| 684 |
+
|
| 685 |
+
with gr.Blocks(title="Universal Conlang Translator") as demo:
|
| 686 |
gr.Markdown("# Universal Conlang Translator · Minimax-ASCII / Kōmín-CJK")
|
| 687 |
with gr.Row():
|
| 688 |
gr.Markdown(EXPLAIN_ES)
|
| 689 |
gr.Markdown(EXPLAIN_EN)
|
| 690 |
|
| 691 |
+
modes_dd = ["Compacto", "Semi-lossless", "Lossless (b85)", "Referencia (hash)"]
|
| 692 |
+
|
| 693 |
# --- Traducir (universal) ---
|
| 694 |
with gr.Tab("Traducir (universal)"):
|
| 695 |
with gr.Row():
|
|
|
|
| 699 |
with gr.Row():
|
| 700 |
uni_drop = gr.Checkbox(value=True, label="Omitir artículos (ES/EN→conlang)")
|
| 701 |
uni_zero = gr.Checkbox(value=True, label="Cópula cero (presente afirm.) (ES/EN→conlang)")
|
| 702 |
+
uni_mode = gr.Dropdown(modes_dd, value="Lossless (b85)", label="Modo")
|
| 703 |
uni_out = gr.Textbox(lines=6, label="Traducción")
|
| 704 |
gr.Button("Traducir").click(
|
| 705 |
+
universal_translate,
|
| 706 |
+
[uni_text, uni_src, uni_tgt, uni_drop, uni_zero, uni_mode],
|
| 707 |
[uni_out]
|
| 708 |
)
|
| 709 |
|
| 710 |
+
# --- Construir frase (ES/EN → Conlang) ---
|
| 711 |
with gr.Tab("Construir frase (ES/EN → Conlang)"):
|
| 712 |
with gr.Row():
|
| 713 |
src_lang = gr.Dropdown(["Español","English"], value="Español", label="Fuente")
|
|
|
|
| 716 |
with gr.Row():
|
| 717 |
drop_articles = gr.Checkbox(value=True, label="Omitir artículos")
|
| 718 |
zero_copula = gr.Checkbox(value=True, label="Cópula cero (presente afirm.)")
|
| 719 |
+
mode = gr.Dropdown(modes_dd, value="Lossless (b85)", label="Modo")
|
| 720 |
out = gr.Textbox(lines=6, label="Salida")
|
| 721 |
+
gr.Button("Construir").click(
|
| 722 |
+
build_sentence,
|
| 723 |
+
[text_in, src_lang, target, drop_articles, zero_copula, mode],
|
| 724 |
+
[out]
|
| 725 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 726 |
|
| 727 |
+
# --- Decodificar (Conlang → ES/EN) ---
|
| 728 |
with gr.Tab("Decodificar (Conlang → ES/EN)"):
|
| 729 |
with gr.Row():
|
| 730 |
src_code = gr.Dropdown(["Minimax-ASCII","Kōmín-CJK"], value="Minimax-ASCII", label="Fuente")
|
| 731 |
tgt_lang = gr.Dropdown(["Español","English"], value="Español", label="Destino")
|
| 732 |
+
code_in = gr.Textbox(lines=3, label="Texto en conlang (incluye §(...) o §{ID} si procede)")
|
| 733 |
out3 = gr.Textbox(lines=6, label="Salida")
|
|
|
|
| 734 |
|
| 735 |
+
def decode_lossless_aware(text, src, tgt):
|
| 736 |
+
# Referencia (hash)
|
| 737 |
+
ref = extract_sidecar_ref(text)
|
| 738 |
+
if ref is not None:
|
| 739 |
+
return ref
|
| 740 |
+
# Lossless b85
|
| 741 |
+
orig = extract_sidecar_b85(text)
|
| 742 |
+
if orig is not None:
|
| 743 |
+
return orig
|
| 744 |
+
# Normal (compacto/semi)
|
| 745 |
+
return decode_simple(strip_sidecar(strip_sidecar_ref(text)), src, tgt)
|
| 746 |
+
|
| 747 |
+
gr.Button("Decodificar").click(
|
| 748 |
+
decode_lossless_aware, [code_in, src_code, tgt_lang], [out3]
|
| 749 |
+
)
|
| 750 |
+
|
| 751 |
+
# --- Round-trip ---
|
| 752 |
+
with gr.Tab("Prueba ida→vuelta"):
|
| 753 |
with gr.Row():
|
| 754 |
+
rt_src = gr.Dropdown(["Español","English"], value="Español", label="Fuente")
|
| 755 |
rt_tgt = gr.Dropdown(["Minimax-ASCII","Kōmín-CJK"], value="Minimax-ASCII", label="Conlang")
|
| 756 |
+
rt_mode = gr.Dropdown(modes_dd, value="Lossless (b85)", label="Modo")
|
| 757 |
+
rt_text = gr.Textbox(lines=3, label="Frase", value="Hola, ¿cómo estás?")
|
|
|
|
|
|
|
| 758 |
rt_out_conlang = gr.Textbox(lines=3, label="Conlang (ida)")
|
| 759 |
+
rt_out_back = gr.Textbox(lines=3, label="Vuelta")
|
| 760 |
+
def round_trip(text, src, tgt, mode):
|
| 761 |
+
conlang = universal_translate(text, src, tgt, True, True, mode)
|
| 762 |
+
back = universal_translate(conlang, tgt, src, True, True, mode)
|
| 763 |
+
return conlang, back
|
| 764 |
+
gr.Button("Probar").click(
|
| 765 |
+
round_trip,
|
| 766 |
+
[rt_text, rt_src, rt_tgt, rt_mode],
|
| 767 |
[rt_out_conlang, rt_out_back]
|
| 768 |
)
|
| 769 |
|
|
|
|
| 777 |
|
| 778 |
|
| 779 |
|
|
|