File size: 4,071 Bytes
8fa1c5a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | import json, os, random, re, sys, time
import torch
TASH = '/kaggle/working/tashkil'
MDIR = TASH + '/model'
NSAMPLE = int(sys.argv[1]) if len(sys.argv) > 1 else 10
PERBOOK = int(sys.argv[2]) if len(sys.argv) > 2 else 30000
torch.set_num_threads(4)
sys.path.insert(0, MDIR)
import model as mm
ckpt = torch.load(MDIR + '/mishkala.pt', map_location='cpu', weights_only=False)
model = mm.ArabicDiacritizerModel(**ckpt['config'])
model.load_state_dict(ckpt['model_state_dict'], strict=False)
model.eval()
c2i = json.load(open(MDIR + '/tokenizer.json'))['char_to_id']
UNK, PAD, BOS, EOS = 1, 0, 2, 3
CLASSES = mm.DIACRITIC_CLASSES
DM = mm.DIACRITIC_MAP
DIAC_SET = mm.DIACRITICS_SET
PARSE_DIACS = set(chr(c) for c in [0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650,
0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x06D6,
0x06D7, 0x0610, 0x0611, 0x0612, 0x0613, 0x0616, 0x0656])
def stripd(t):
return ''.join(c for c in t if c not in DIAC_SET)
def split_long(text, maxc=450):
if len(text) <= maxc:
return [text] if text else []
parts = re.split(r'([.،؟!؟؛:;,\n])', text)
chunks, cur = [], ''
for p in parts:
if not p:
continue
if len(cur) + len(p) > maxc and cur:
chunks.append(cur)
cur = p
else:
cur += p
if cur:
chunks.append(cur)
return chunks
def parse_diacs(text):
chars, diacs = [], []
i = 0
while i < len(text):
ch = text[i]
if ch in PARSE_DIACS:
i += 1
continue
i += 1
chars.append(ch)
j = i
while j < len(text) and text[j] in PARSE_DIACS:
j += 1
diacs.append(text[i:j])
i = j
return chars, diacs
def run_batch(texts):
items = []
for t in texts:
chars = list(stripd(t))
ids = [BOS] + [c2i.get(c, UNK) for c in chars] + [EOS]
items.append((chars, ids))
maxlen = max(len(x[1]) for x in items)
n = len(items)
pad = torch.full((n, maxlen), PAD, dtype=torch.long)
mask = torch.zeros((n, maxlen), dtype=torch.long)
for i, (_, ids) in enumerate(items):
pad[i, :len(ids)] = torch.tensor(ids)
mask[i, :len(ids)] = 1
with torch.no_grad():
o = model(pad, mask)
preds = o['predictions']
res = []
for (chars, ids), p in zip(items, preds):
buf = []
for i, ch in enumerate(chars):
buf.append(ch)
d = DM.get(CLASSES[p[i + 1]], '')
if d:
buf.append(d)
res.append(''.join(buf))
return res
def batched_run(texts, budget=40000, maxb=48):
res = [''] * len(texts)
b0 = 0
while b0 < len(texts):
b1 = b0 + 1
acc = len(texts[b0])
while b1 < len(texts) and acc + len(texts[b1]) <= budget and (b1 - b0) < maxb:
acc += len(texts[b1])
b1 += 1
res[b0:b1] = run_batch(texts[b0:b1])
b0 = b1
return res
root = TASH + '/books_raw'
books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int)
random.seed(42)
sample = random.sample(books, NSAMPLE)
total = gold_n = match = pred_n = 0
t0 = time.time()
for b in sample:
text = open(os.path.join(root, b + '.txt'), encoding='utf-8').read()[:PERBOOK]
lines = text.split('\n')
units = []
for ln in lines:
units.extend(split_long(ln))
res = batched_run(units)
for u, r in zip(units, res):
uc, ug = parse_diacs(u)
rc, rp = parse_diacs(r)
assert uc == rc, f'misalign in book {b}: {len(uc)} vs {len(rc)}'
for a, b_ in zip(ug, rp):
total += 1
if b_:
pred_n += 1
if a:
gold_n += 1
if a == b_:
match += 1
el = time.time() - t0
print(f'books={len(sample)} chars_eval={total} gold_diac_letters={gold_n} '
f'match={match} gold_match_rate={match/gold_n:.4f} pred_diac_rate={pred_n/total:.3f} {el:.0f}s', flush=True)
|