File size: 4,071 Bytes
8fa1c5a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
import json, os, random, re, sys, time
import torch

TASH = '/kaggle/working/tashkil'
MDIR = TASH + '/model'
NSAMPLE = int(sys.argv[1]) if len(sys.argv) > 1 else 10
PERBOOK = int(sys.argv[2]) if len(sys.argv) > 2 else 30000

torch.set_num_threads(4)
sys.path.insert(0, MDIR)
import model as mm

ckpt = torch.load(MDIR + '/mishkala.pt', map_location='cpu', weights_only=False)
model = mm.ArabicDiacritizerModel(**ckpt['config'])
model.load_state_dict(ckpt['model_state_dict'], strict=False)
model.eval()
c2i = json.load(open(MDIR + '/tokenizer.json'))['char_to_id']
UNK, PAD, BOS, EOS = 1, 0, 2, 3
CLASSES = mm.DIACRITIC_CLASSES
DM = mm.DIACRITIC_MAP
DIAC_SET = mm.DIACRITICS_SET
PARSE_DIACS = set(chr(c) for c in [0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650,
                                   0x0651, 0x0652, 0x0653, 0x0654, 0x0655, 0x06D6,
                                   0x06D7, 0x0610, 0x0611, 0x0612, 0x0613, 0x0616, 0x0656])

def stripd(t):
    return ''.join(c for c in t if c not in DIAC_SET)

def split_long(text, maxc=450):
    if len(text) <= maxc:
        return [text] if text else []
    parts = re.split(r'([.،؟!؟؛:;,\n])', text)
    chunks, cur = [], ''
    for p in parts:
        if not p:
            continue
        if len(cur) + len(p) > maxc and cur:
            chunks.append(cur)
            cur = p
        else:
            cur += p
    if cur:
        chunks.append(cur)
    return chunks

def parse_diacs(text):
    chars, diacs = [], []
    i = 0
    while i < len(text):
        ch = text[i]
        if ch in PARSE_DIACS:
            i += 1
            continue
        i += 1
        chars.append(ch)
        j = i
        while j < len(text) and text[j] in PARSE_DIACS:
            j += 1
        diacs.append(text[i:j])
        i = j
    return chars, diacs

def run_batch(texts):
    items = []
    for t in texts:
        chars = list(stripd(t))
        ids = [BOS] + [c2i.get(c, UNK) for c in chars] + [EOS]
        items.append((chars, ids))
    maxlen = max(len(x[1]) for x in items)
    n = len(items)
    pad = torch.full((n, maxlen), PAD, dtype=torch.long)
    mask = torch.zeros((n, maxlen), dtype=torch.long)
    for i, (_, ids) in enumerate(items):
        pad[i, :len(ids)] = torch.tensor(ids)
        mask[i, :len(ids)] = 1
    with torch.no_grad():
        o = model(pad, mask)
    preds = o['predictions']
    res = []
    for (chars, ids), p in zip(items, preds):
        buf = []
        for i, ch in enumerate(chars):
            buf.append(ch)
            d = DM.get(CLASSES[p[i + 1]], '')
            if d:
                buf.append(d)
        res.append(''.join(buf))
    return res

def batched_run(texts, budget=40000, maxb=48):
    res = [''] * len(texts)
    b0 = 0
    while b0 < len(texts):
        b1 = b0 + 1
        acc = len(texts[b0])
        while b1 < len(texts) and acc + len(texts[b1]) <= budget and (b1 - b0) < maxb:
            acc += len(texts[b1])
            b1 += 1
        res[b0:b1] = run_batch(texts[b0:b1])
        b0 = b1
    return res

root = TASH + '/books_raw'
books = sorted((f[:-4] for f in os.listdir(root) if f.endswith('.txt')), key=int)
random.seed(42)
sample = random.sample(books, NSAMPLE)

total = gold_n = match = pred_n = 0
t0 = time.time()
for b in sample:
    text = open(os.path.join(root, b + '.txt'), encoding='utf-8').read()[:PERBOOK]
    lines = text.split('\n')
    units = []
    for ln in lines:
        units.extend(split_long(ln))
    res = batched_run(units)
    for u, r in zip(units, res):
        uc, ug = parse_diacs(u)
        rc, rp = parse_diacs(r)
        assert uc == rc, f'misalign in book {b}: {len(uc)} vs {len(rc)}'
        for a, b_ in zip(ug, rp):
            total += 1
            if b_:
                pred_n += 1
            if a:
                gold_n += 1
                if a == b_:
                    match += 1
el = time.time() - t0
print(f'books={len(sample)} chars_eval={total} gold_diac_letters={gold_n} '
      f'match={match} gold_match_rate={match/gold_n:.4f} pred_diac_rate={pred_n/total:.3f} {el:.0f}s', flush=True)