import json, os, random, re, sys, time import torch TASH = '/kaggle/working/tashkil' MDIR = TASH + '/model' NSAMPLE = int(sys.argv[1]) if len(sys.argv) > 1 else 10 PERBOOK = int(sys.argv[2]) if len(sys.argv) > 2 else 30000 torch.set_num_threads(4) sys.path.insert(0, MDIR) import model as mm ckpt = torch.load(MDIR + '/mishkala.pt', map_location='cpu', weights_only=False) model = mm.ArabicDiacritizerModel(**ckpt['config']) model.load_state_dict(ckpt['model_state_dict'], strict=False) model.eval() c2i = json.load(open(MDIR + '/tokenizer.json'))['char_to_id'] UNK, PAD, BOS, EOS = 1, 0, 2, 3 CLASSES = mm.DIACRITIC_CLASSES DM = mm.DIACRITIC_MAP DIAC_SET = mm.DIACRITICS_SET PARSE_DIACS = set(chr(c) for c in [0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650, 0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x06D6, 0x06D7, 0x0610, 0x0611, 0x0612, 0x0613, 0x0616, 0x0656]) def stripd(t): return ''.join(c for c in t if c not in DIAC_SET) def split_long(text, maxc=450): if len(text) <= maxc: return [text] if text else [] parts = re.split(r'([.،؟!؟؛:;,\n])', text) chunks, cur = [], '' for p in parts: if not p: continue if len(cur) + len(p) > maxc and cur: chunks.append(cur) cur = p else: cur += p if cur: chunks.append(cur) return chunks def parse_diacs(text): chars, diacs = [], [] i = 0 while i < len(text): ch = text[i] if ch in PARSE_DIACS: i += 1 continue i += 1 chars.append(ch) j = i while j < len(text) and text[j] in PARSE_DIACS: j += 1 diacs.append(text[i:j]) i = j return chars, diacs def run_batch(texts): items = [] for t in texts: chars = list(stripd(t)) ids = [BOS] + [c2i.get(c, UNK) for c in chars] + [EOS] items.append((chars, ids)) maxlen = max(len(x[1]) for x in items) n = len(items) pad = torch.full((n, maxlen), PAD, dtype=torch.long) mask = torch.zeros((n, maxlen), dtype=torch.long) for i, (_, ids) in enumerate(items): pad[i, :len(ids)] = torch.tensor(ids) mask[i, :len(ids)] = 1 with torch.no_grad(): o = model(pad, mask) preds = o['predictions'] res = [] for (chars, ids), p in zip(items, preds): buf = [] for i, ch in enumerate(chars): buf.append(ch) d = DM.get(CLASSES[p[i + 1]], '') if d: buf.append(d) res.append(''.join(buf)) return res def batched_run(texts, budget=40000, maxb=48): res = [''] * len(texts) b0 = 0 while b0 < len(texts): b1 = b0 + 1 acc = len(texts[b0]) while b1 < len(texts) and acc + len(texts[b1]) <= budget and (b1 - b0) < maxb: acc += len(texts[b1]) b1 += 1 res[b0:b1] = run_batch(texts[b0:b1]) b0 = b1 return res root = TASH + '/books_raw' books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int) random.seed(42) sample = random.sample(books, NSAMPLE) total = gold_n = match = pred_n = 0 t0 = time.time() for b in sample: text = open(os.path.join(root, b + '.txt'), encoding='utf-8').read()[:PERBOOK] lines = text.split('\n') units = [] for ln in lines: units.extend(split_long(ln)) res = batched_run(units) for u, r in zip(units, res): uc, ug = parse_diacs(u) rc, rp = parse_diacs(r) assert uc == rc, f'misalign in book {b}: {len(uc)} vs {len(rc)}' for a, b_ in zip(ug, rp): total += 1 if b_: pred_n += 1 if a: gold_n += 1 if a == b_: match += 1 el = time.time() - t0 print(f'books={len(sample)} chars_eval={total} gold_diac_letters={gold_n} ' f'match={match} gold_match_rate={match/gold_n:.4f} pred_diac_rate={pred_n/total:.3f} {el:.0f}s', flush=True)