tashkil-scripts / eval_gate.py
flokymind's picture
Upload eval_gate.py with huggingface_hub
8fa1c5a verified
Raw History Blame Contribute Delete
4.07 kB
import json, os, random, re, sys, time
import torch
TASH = '/kaggle/working/tashkil'
MDIR = TASH + '/model'
NSAMPLE = int(sys.argv[1]) if len(sys.argv) > 1 else 10
PERBOOK = int(sys.argv[2]) if len(sys.argv) > 2 else 30000
torch.set_num_threads(4)
sys.path.insert(0, MDIR)
import model as mm
ckpt = torch.load(MDIR + '/mishkala.pt', map_location='cpu', weights_only=False)
model = mm.ArabicDiacritizerModel(**ckpt['config'])
model.load_state_dict(ckpt['model_state_dict'], strict=False)
model.eval()
c2i = json.load(open(MDIR + '/tokenizer.json'))['char_to_id']
UNK, PAD, BOS, EOS = 1, 0, 2, 3
CLASSES = mm.DIACRITIC_CLASSES
DM = mm.DIACRITIC_MAP
DIAC_SET = mm.DIACRITICS_SET
PARSE_DIACS = set(chr(c) for c in [0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650,
0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x06D6,
0x06D7, 0x0610, 0x0611, 0x0612, 0x0613, 0x0616, 0x0656])
def stripd(t):
return ''.join(c for c in t if c not in DIAC_SET)
def split_long(text, maxc=450):
if len(text) <= maxc:
return [text] if text else []
parts = re.split(r'([.،؟!؟؛:;,\n])', text)
chunks, cur = [], ''
for p in parts:
if not p:
continue
if len(cur) + len(p) > maxc and cur:
chunks.append(cur)
cur = p
else:
cur += p
if cur:
chunks.append(cur)
return chunks
def parse_diacs(text):
chars, diacs = [], []
i = 0
while i < len(text):
ch = text[i]
if ch in PARSE_DIACS:
i += 1
continue
i += 1
chars.append(ch)
j = i
while j < len(text) and text[j] in PARSE_DIACS:
j += 1
diacs.append(text[i:j])
i = j
return chars, diacs
def run_batch(texts):
items = []
for t in texts:
chars = list(stripd(t))
ids = [BOS] + [c2i.get(c, UNK) for c in chars] + [EOS]
items.append((chars, ids))
maxlen = max(len(x[1]) for x in items)
n = len(items)
pad = torch.full((n, maxlen), PAD, dtype=torch.long)
mask = torch.zeros((n, maxlen), dtype=torch.long)
for i, (_, ids) in enumerate(items):
pad[i, :len(ids)] = torch.tensor(ids)
mask[i, :len(ids)] = 1
with torch.no_grad():
o = model(pad, mask)
preds = o['predictions']
res = []
for (chars, ids), p in zip(items, preds):
buf = []
for i, ch in enumerate(chars):
buf.append(ch)
d = DM.get(CLASSES[p[i + 1]], '')
if d:
buf.append(d)
res.append(''.join(buf))
return res
def batched_run(texts, budget=40000, maxb=48):
res = [''] * len(texts)
b0 = 0
while b0 < len(texts):
b1 = b0 + 1
acc = len(texts[b0])
while b1 < len(texts) and acc + len(texts[b1]) <= budget and (b1 - b0) < maxb:
acc += len(texts[b1])
b1 += 1
res[b0:b1] = run_batch(texts[b0:b1])
b0 = b1
return res
root = TASH + '/books_raw'
books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int)
random.seed(42)
sample = random.sample(books, NSAMPLE)
total = gold_n = match = pred_n = 0
t0 = time.time()
for b in sample:
text = open(os.path.join(root, b + '.txt'), encoding='utf-8').read()[:PERBOOK]
lines = text.split('\n')
units = []
for ln in lines:
units.extend(split_long(ln))
res = batched_run(units)
for u, r in zip(units, res):
uc, ug = parse_diacs(u)
rc, rp = parse_diacs(r)
assert uc == rc, f'misalign in book {b}: {len(uc)} vs {len(rc)}'
for a, b_ in zip(ug, rp):
total += 1
if b_:
pred_n += 1
if a:
gold_n += 1
if a == b_:
match += 1
el = time.time() - t0
print(f'books={len(sample)} chars_eval={total} gold_diac_letters={gold_n} '
f'match={match} gold_match_rate={match/gold_n:.4f} pred_diac_rate={pred_n/total:.3f} {el:.0f}s', flush=True)