| """OCR-like corruption of normalised transliterations. |
| |
| Substitutions follow confusions between graphically or phonetically close runes and their |
| transliterations (k/g, t/þ/d, b/p, i/e, u/o/y, R/r, m/n, a/æ/ą, s/z); deletions and |
| insertions are drawn uniformly. `confusions` can be replaced by a matrix estimated from a |
| recogniser's own errors (see estimate_confusions). |
| """ |
| import random |
| from collections import Counter, defaultdict |
|
|
| DEFAULT_CONFUSIONS = { |
| "k": "g", "g": "k", "t": "þd", "þ": "td", "d": "tþ", "b": "p", "p": "b", |
| "i": "e", "e": "i", "u": "oy", "o": "u", "y": "u", "R": "r", "r": "R", |
| "m": "n", "n": "m", "a": "æą", "æ": "a", "ą": "a", "s": "z", "z": "s", |
| "l": "i", "f": "u", "h": "n", |
| } |
| LETTERS = "abdefghiklmnorstuþR" |
|
|
|
|
| def corrupt(text, cer, rng, confusions=None): |
| """Return text with approximately `cer` character edits per reference character.""" |
| if cer <= 0 or not text: |
| return text |
| conf = confusions or DEFAULT_CONFUSIONS |
| out = [] |
| for ch in text: |
| if ch == " ": |
| |
| if rng.random() < cer * 0.3: |
| continue |
| out.append(ch) |
| continue |
| r = rng.random() |
| if r < cer: |
| op = rng.random() |
| if op < 0.6: |
| cands = conf.get(ch) |
| if isinstance(cands, dict): |
| keys, w = zip(*cands.items()) |
| out.append(rng.choices(keys, weights=w)[0]) |
| else: |
| out.append(rng.choice(cands) if cands else rng.choice(LETTERS)) |
| elif op < 0.85: |
| continue |
| else: |
| out.append(ch) |
| out.append(rng.choice(LETTERS)) |
| else: |
| out.append(ch) |
| return "".join(out) |
|
|
|
|
| def estimate_confusions(pairs): |
| """pairs: iterable of (reference, hypothesis). Returns {ref_char: {hyp_char: count}} from |
| Levenshtein alignments (substitutions only).""" |
| from rapidfuzz.distance import Levenshtein |
|
|
| subs = defaultdict(Counter) |
| for ref, hyp in pairs: |
| for op, i, j in Levenshtein.editops(ref, hyp): |
| if op == "replace" and ref[i] != " " and hyp[j] != " ": |
| subs[ref[i]][hyp[j]] += 1 |
| return {k: dict(v) for k, v in subs.items()} |
|
|
|
|
| if __name__ == "__main__": |
| rng = random.Random(0) |
| s = "haraltr kunukR baþ kaurua kubl þausi aft kurm faþur sin" |
| for c in (0.1, 0.3, 0.6): |
| print(c, corrupt(s, c, rng)) |
|
|
