| """PC tool: false positives of the additive matcher over the fit corpus's words. usage: fpcheck.py [--matcher add] |
| Prints words (not in the vocabulary) whose value sum falls in a class window, with their counts.""" |
| import re, sys |
| from collections import Counter |
| from pathlib import Path |
| import numpy as np |
| import compile as c |
| vocab = c.pick_vocab(None) |
| c.solve_additive(vocab) |
| C = Path("corpus_cache") |
| data = np.memmap(C / "data.bin", dtype=np.uint8, mode="r") |
| gold = np.memmap(C / "gold.bin", dtype=np.uint8, mode="r") |
| wre = re.compile(rb"[A-Za-z_\x80-\xff][A-Za-z_0-9\x80-\xff]*") |
| cnt = Counter(); nbytes = Counter() |
| for line in open(C / "index.tsv"): |
| sp, pkg, rel, off, n = line.rstrip("\n").split("\t") |
| if sp != "fit": |
| continue |
| off, n = int(off), int(n) |
| d = data[off:off + n].tobytes(); g = gold[off:off + n] |
| for m in wre.finditer(d): |
| if g[m.start()] == 0: |
| w = m.group().decode("latin1") |
| cnt[w] += 1; nbytes[w] += m.end() - m.start() |
| tot = fp = fpb = 0 |
| hits = Counter(); hitb = Counter() |
| for w, n in cnt.items(): |
| if w in vocab: |
| continue |
| tot += n |
| v = sum(c.ADD_TABLES.get(f, {}).get(code, 0) * k for (f, code), k in c.word_feats(w).items()) |
| for t in (1, 2, 3, 4, 5): |
| if abs(v - t * c.ADD_T) <= c.ADD_EPS: |
| hits[w] += n; hitb[w] += nbytes[w]; fp += n; fpb += nbytes[w] |
| print(f"non-vocabulary text words: {tot} occurrences, {len(cnt)} distinct; false positives: {fp} occurrences, {fpb} bytes ({fpb / 74787051:.6f} of corpus)") |
| for w, n in hits.most_common(40): |
| print(f" {w:30} {n:7} occ {hitb[w]:8} B") |
|
|
