File size: 3,778 Bytes
d7ec002 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """
Phase C — training-signal generator: paraphrase augmentation.
Word-level synonyms (EN + AR), word dropout, typos. This is the 'exposure'
that teaches the encoder that start≈begin, عاصمة≈العاصمة, etc.
(Honest note: with 243 entries this lexicon is the training signal;
at Phase-D scale the same InfoNCE learns distributional semantics from data.)
"""
import re
SYN = {
# English
"start": ["begin", "commence", "launch"],
"started": ["began", "commenced", "launched"],
"operating": ["flying", "functioning", "working", "operations"],
"survive": ["live", "endure"],
"get": ["find", "obtain", "retrieve"],
"total": ["sum"],
"routine": ["function"],
"carrier": ["airline"],
"make": ["write", "create"],
"write": ["create", "make"],
"generate": ["produce", "create"],
"operate": ["run", "work", "function"],
"services": ["flights", "operations", "service"],
"commenced": ["began", "started"],
"capital": ["capital city"],
"what": ["which"],
"when": ["at what time"],
"how": ["in what way"],
"many": ["number of"],
"people": ["population", "inhabitants"],
"live": ["reside", "living"],
"difference": ["distinction", "gap"],
"created": ["built", "made", "developed"],
"built": ["made", "created", "developed"],
"developed": ["created", "built", "made"],
"create": ["make", "write", "build"],
"calculate": ["compute", "sum", "evaluate"],
"function": ["method", "program"],
"sequence": ["list", "series"],
"integers": ["numbers", "ints"],
"brain": ["mind"],
"human": ["person"],
"works": ["functions", "operates"],
"working": ["functioning", "operating"],
"artificial": ["synthetic"],
"learning": ["training"],
"machine": ["computer"],
"intelligence": ["ai"],
"who": ["whom"],
"your": ["ur"],
# Arabic
"ما": ["ايه"],
"هي": ["هى"],
"من": ["مين"],
"كيف": ["ازاي"],
"كم": ["قد ايه"],
"عاصمة": ["العاصمة", "العاصمة"],
"يبلغ": ["يصل", "يقدر"],
"يعمل": ["يشتغل", "يقوم"],
"عملك": ["طورك", "صنعك", "تطويرك"],
"تطويرك": ["عملك", "صنعك"],
"أنت": ["انت"],
"دماغ": ["عقل"],
"الإنسان": ["البشري", "الانسان"],
"بين": ["وبين"],
"ماهي": ["ايه هي"],
"الفرق": ["الاختلاف"],
"عدد": ["عدد"],
"ساكنين": ["سكان", "السكان"],
"الساحل": ["الشاطئ"],
}
_DROP = ["the", "of", "a", "an", "to", "is", "are", "was", "في", "من"]
def augment(text, rng):
"""One noisy paraphrase view of `text`."""
words = re.findall(r"\w+", text, flags=re.UNICODE)
if len(words) == 0:
return text
out = []
for w in words:
lw = w.lower()
if rng.rand() < 0.6 and lw in SYN: # aggressive synonymy
out.append(SYN[lw][rng.randint(len(SYN[lw]))])
else:
out.append(w)
# word dropout (keep at least 2 words)
kept = [w for w in out if rng.rand() > 0.25 or w.lower() not in _DROP]
if len(kept) >= 2:
out = kept
# typos (char delete/swap/insert) — char 3-grams make this robust
final = []
for w in out:
if rng.rand() < 0.12 and len(w) > 3:
i = rng.randint(1, len(w) - 1)
kind = rng.randint(3)
if kind == 0:
w = w[:i] + w[i + 1:] # delete
elif kind == 1:
w = w[:i] + w[i + 1] + w[i] + w[i + 2:] # swap
else:
w = w[:i] + w[i] + w[i:] # insert
final.append(w)
return " ".join(final)
|