| """Check bench/scanparity (native Swift scanner) against citenet.text on a text file.""" |
| import subprocess, sys, os |
| import numpy as np |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| from citenet import text, spec |
|
|
| path = sys.argv[1] |
| s = open(path, encoding="utf-8", newline="").read() |
| found = text.extract(s) |
| py_cites = [c for _, _, c in found] |
| py_codes = text.encode(py_cites).reshape(len(py_cites), spec.SEQ).astype(np.float16) |
| out = subprocess.run([os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "bench", "scanparity"), |
| path, "/tmp/scanparity_out"], capture_output=True, text=True) |
| print(out.stdout.strip(), out.stderr.strip()[:500]) |
| sw_codes = np.fromfile("/tmp/scanparity_out.f16", dtype=np.float16).reshape(-1, spec.SEQ) |
| sw_cites = [l.replace("\\r", "\r").replace("\\n", "\n") for l in open("/tmp/scanparity_out.txt", encoding="utf-8", newline="").read().split("\n")] if len(sw_codes) else [] |
| print(f"python {len(py_cites):,} citations, swift {len(sw_codes):,}") |
| n = min(len(py_cites), len(sw_cites)) |
| cite_mis = [i for i in range(n) if py_cites[i] != sw_cites[i]] |
| print(f"first {n:,}: cite text mismatches {len(cite_mis)}") |
| for i in cite_mis[:5]: |
| print(" py:", repr(py_cites[i]), "\n sw:", repr(sw_cites[i])) |
| if not cite_mis and len(py_cites) == len(sw_cites): |
| bad = np.where((py_codes != sw_codes).any(1))[0] |
| print(f"code rows identical: {len(py_cites) - len(bad):,}/{len(py_cites):,}") |
| inv = {v: k for k, v in spec.PUNCT.items()} |
| dec = lambda r: "".join(chr(48 + int(c) - 3) if 3 <= c <= 12 else chr(97 + int(c) - 13) if 13 <= c <= 38 else inv.get(int(c), "?") if c > 2 else "" for c in r) |
| for i in bad[:6]: |
| print(" ", repr(py_cites[i]), "\n py:", dec(py_codes[i]), "\n sw:", dec(sw_codes[i])) |
|
|
