Download eval_gate.py from flokymind/tashkil-scripts: direct link, hf CLI and curl.
- Browser
- Download file 4.07 kB
-
https://huggingface.co/flokymind/tashkil-scripts/resolve/main/eval_gate.py
- Command line
-
hf download hf://flokymind/tashkil-scripts/eval_gate.py
-
curl -L -o eval_gate.py https://huggingface.co/flokymind/tashkil-scripts/resolve/main/eval_gate.py
4.07 kB
| import json, os, random, re, sys, time | |
| import torch | |
| TASH = '/kaggle/working/tashkil' | |
| MDIR = TASH + '/model' | |
| NSAMPLE = int(sys.argv[1]) if len(sys.argv) > 1 else 10 | |
| PERBOOK = int(sys.argv[2]) if len(sys.argv) > 2 else 30000 | |
| torch.set_num_threads(4) | |
| sys.path.insert(0, MDIR) | |
| import model as mm | |
| ckpt = torch.load(MDIR + '/mishkala.pt', map_location='cpu', weights_only=False) | |
| model = mm.ArabicDiacritizerModel(**ckpt['config']) | |
| model.load_state_dict(ckpt['model_state_dict'], strict=False) | |
| model.eval() | |
| c2i = json.load(open(MDIR + '/tokenizer.json'))['char_to_id'] | |
| UNK, PAD, BOS, EOS = 1, 0, 2, 3 | |
| CLASSES = mm.DIACRITIC_CLASSES | |
| DM = mm.DIACRITIC_MAP | |
| DIAC_SET = mm.DIACRITICS_SET | |
| PARSE_DIACS = set(chr(c) for c in [0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650, | |
| 0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x06D6, | |
| 0x06D7, 0x0610, 0x0611, 0x0612, 0x0613, 0x0616, 0x0656]) | |
| def stripd(t): | |
| return ''.join(c for c in t if c not in DIAC_SET) | |
| def split_long(text, maxc=450): | |
| if len(text) <= maxc: | |
| return [text] if text else [] | |
| parts = re.split(r'([.،؟!؟؛:;,\n])', text) | |
| chunks, cur = [], '' | |
| for p in parts: | |
| if not p: | |
| continue | |
| if len(cur) + len(p) > maxc and cur: | |
| chunks.append(cur) | |
| cur = p | |
| else: | |
| cur += p | |
| if cur: | |
| chunks.append(cur) | |
| return chunks | |
| def parse_diacs(text): | |
| chars, diacs = [], [] | |
| i = 0 | |
| while i < len(text): | |
| ch = text[i] | |
| if ch in PARSE_DIACS: | |
| i += 1 | |
| continue | |
| i += 1 | |
| chars.append(ch) | |
| j = i | |
| while j < len(text) and text[j] in PARSE_DIACS: | |
| j += 1 | |
| diacs.append(text[i:j]) | |
| i = j | |
| return chars, diacs | |
| def run_batch(texts): | |
| items = [] | |
| for t in texts: | |
| chars = list(stripd(t)) | |
| ids = [BOS] + [c2i.get(c, UNK) for c in chars] + [EOS] | |
| items.append((chars, ids)) | |
| maxlen = max(len(x[1]) for x in items) | |
| n = len(items) | |
| pad = torch.full((n, maxlen), PAD, dtype=torch.long) | |
| mask = torch.zeros((n, maxlen), dtype=torch.long) | |
| for i, (_, ids) in enumerate(items): | |
| pad[i, :len(ids)] = torch.tensor(ids) | |
| mask[i, :len(ids)] = 1 | |
| with torch.no_grad(): | |
| o = model(pad, mask) | |
| preds = o['predictions'] | |
| res = [] | |
| for (chars, ids), p in zip(items, preds): | |
| buf = [] | |
| for i, ch in enumerate(chars): | |
| buf.append(ch) | |
| d = DM.get(CLASSES[p[i + 1]], '') | |
| if d: | |
| buf.append(d) | |
| res.append(''.join(buf)) | |
| return res | |
| def batched_run(texts, budget=40000, maxb=48): | |
| res = [''] * len(texts) | |
| b0 = 0 | |
| while b0 < len(texts): | |
| b1 = b0 + 1 | |
| acc = len(texts[b0]) | |
| while b1 < len(texts) and acc + len(texts[b1]) <= budget and (b1 - b0) < maxb: | |
| acc += len(texts[b1]) | |
| b1 += 1 | |
| res[b0:b1] = run_batch(texts[b0:b1]) | |
| b0 = b1 | |
| return res | |
| root = TASH + '/books_raw' | |
| books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int) | |
| random.seed(42) | |
| sample = random.sample(books, NSAMPLE) | |
| total = gold_n = match = pred_n = 0 | |
| t0 = time.time() | |
| for b in sample: | |
| text = open(os.path.join(root, b + '.txt'), encoding='utf-8').read()[:PERBOOK] | |
| lines = text.split('\n') | |
| units = [] | |
| for ln in lines: | |
| units.extend(split_long(ln)) | |
| res = batched_run(units) | |
| for u, r in zip(units, res): | |
| uc, ug = parse_diacs(u) | |
| rc, rp = parse_diacs(r) | |
| assert uc == rc, f'misalign in book {b}: {len(uc)} vs {len(rc)}' | |
| for a, b_ in zip(ug, rp): | |
| total += 1 | |
| if b_: | |
| pred_n += 1 | |
| if a: | |
| gold_n += 1 | |
| if a == b_: | |
| match += 1 | |
| el = time.time() - t0 | |
| print(f'books={len(sample)} chars_eval={total} gold_diac_letters={gold_n} ' | |
| f'match={match} gold_match_rate={match/gold_n:.4f} pred_diac_rate={pred_n/total:.3f} {el:.0f}s', flush=True) | |