Download copilot/matcher.py from Wardline/cloak-api: direct link, hf CLI and curl.
- Browser
- Download file 10.3 kB
-
https://huggingface.co/spaces/Wardline/cloak-api/resolve/main/copilot/matcher.py
- Command line
-
hf download hf://spaces/Wardline/cloak-api/copilot/matcher.py
-
curl -L -o matcher.py https://huggingface.co/spaces/Wardline/cloak-api/resolve/main/copilot/matcher.py
10.3 kB
| """Finds vocabulary phrases, tenant names and literal values (domains, numbers ...) in the tokens. | |
| Output is one Item per position: a matched phrase (with every sense it can have), a literal, | |
| an unknown word or a punctuation mark. Deciding which sense is meant is interpret.py's job. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from dataclasses import dataclass, field | |
| from .lexicon import Lexicon, Sense, load | |
| from .platform.context import PlatformContext | |
| from .text import Token, stem, is_serial | |
| FUZZY_MIN = 5 | |
| EXT_ID = re.compile(r'^[a-p]{32}$') # a Chrome / Edge extension id | |
| class Item: | |
| i: int # token range [i, j) | |
| j: int | |
| start: int # character range in the normalised text | |
| end: int | |
| text: str | |
| senses: list[Sense] = field(default_factory=list) | |
| method: str = 'none' # exact | stem | fuzzy | context | literal | none | punct | |
| score: float = 1.0 | |
| typed: str = '' # what the user actually typed, when it was corrected (fuzzy) | |
| def has(self, *slots: str) -> bool: | |
| return any(s.slot in slots for s in self.senses) | |
| def get(self, slot: str) -> Sense | None: | |
| return next((s for s in self.senses if s.slot == slot), None) | |
| def values(self, slot: str) -> list: | |
| return [s.value for s in self.senses if s.slot == slot] | |
| def is_punct(self) -> bool: | |
| return self.method == 'punct' | |
| def _osa(a: str, b: str, limit: int) -> int: | |
| """Optimal string alignment distance (Levenshtein + adjacent transposition), early exit.""" | |
| if abs(len(a) - len(b)) > limit: | |
| return limit + 1 | |
| prev2: list[int] | None = None | |
| prev = list(range(len(b) + 1)) | |
| for i in range(1, len(a) + 1): | |
| cur = [i] + [0] * len(b) | |
| best = cur[0] | |
| for j in range(1, len(b) + 1): | |
| cost = 0 if a[i - 1] == b[j - 1] else 1 | |
| v = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost) | |
| if prev2 is not None and i > 1 and j > 1 and a[i - 1] == b[j - 2] and a[i - 2] == b[j - 1]: | |
| v = min(v, prev2[j - 2] + 1) | |
| cur[j] = v | |
| best = min(best, v) | |
| if best > limit: | |
| return limit + 1 | |
| prev2, prev = prev, cur | |
| return prev[-1] | |
| class _Overlay: | |
| """Tenant names (groups, custom rules and lists, connectors) indexed like vocabulary.""" | |
| def __init__(self, ctx: PlatformContext, lx: Lexicon): | |
| from .text import tokenize | |
| self.exact: dict[tuple[str, ...], list[Sense]] = {} | |
| self.stemmed: dict[tuple[str, ...], list[Sense]] = {} | |
| self.fuzzy: dict[str, list[Sense]] = {} | |
| self.max_len = 1 | |
| def add(name: str, sense: Sense) -> None: | |
| toks = tuple(t.text for t in tokenize(name) if t.kind != 'punct' or t.text in '&/') | |
| if not toks: | |
| return | |
| for d, key in ((self.exact, toks), (self.stemmed, tuple(stem(t) for t in toks))): | |
| lst = d.setdefault(key, []) | |
| if sense not in lst: | |
| lst.append(sense) | |
| self.max_len = max(self.max_len, len(toks)) | |
| if len(toks) == 1 and len(toks[0]) >= FUZZY_MIN and toks[0].isalpha() and toks[0] not in lx.no_fuzzy: | |
| self.fuzzy.setdefault(toks[0], []).append(sense) | |
| for g in ctx.groups: | |
| sense = Sense('group', g.id, g.name) | |
| add(g.name, sense) | |
| # "Finance Team" is also "finance"; "Sales-Ops" is also "sales ops". | |
| words = g.name.lower().replace('-', ' ').replace('_', ' ').split() | |
| if len(words) > 1 and words[-1] in ('team', 'group', 'dept', 'department', 'users', 'staff'): | |
| add(' '.join(words[:-1]), sense) | |
| if '-' in g.name or '_' in g.name: | |
| add(' '.join(words), sense) | |
| for r in ctx.rules: | |
| if not r.builtin: | |
| add(r.name, Sense('rule_ref', r.id, r.name)) | |
| for b in ctx.blockLists: | |
| if not b.builtin: | |
| add(b.name, Sense('blist', b.id, b.name)) | |
| for w in ctx.webCategories: | |
| add(w.name, Sense('wcat', w.name, w.name)) # the tenant's own category name, as stored | |
| for ch, conns in ctx.connectors.items(): | |
| for c in conns: | |
| add(c.name, Sense('connector', (ch, c.id), c.name)) | |
| # The tenant's regex detectors: "employee code" -> EMPLOYEE_CODE, by its name and by its label. | |
| for x in ctx.detectors: | |
| sense = Sense('entity', x.label, x.label) | |
| add(x.name, sense) | |
| add(x.label.replace('_', ' '), sense) | |
| def match(tokens: list[Token], ctx: PlatformContext, lx: Lexicon | None = None) -> list[Item]: | |
| lx = lx or load() | |
| ov = _Overlay(ctx, lx) | |
| tokens = _split_unknown_compounds(tokens, lx) | |
| texts = [t.text for t in tokens] | |
| items: list[Item] = [] | |
| i = 0 | |
| n = len(tokens) | |
| while i < n: | |
| tok = tokens[i] | |
| if tok.kind == 'punct' and tok.text not in '/&:': | |
| items.append(Item(i, i + 1, tok.start, tok.end, tok.text, method='punct')) | |
| i += 1 | |
| continue | |
| found = _longest(texts, i, lx, ov, tokens) | |
| if found: | |
| L, senses, method = found | |
| j = i + L | |
| items.append(Item(i, j, tok.start, tokens[j - 1].end, ' '.join(texts[i:j]), senses, method)) | |
| i = j | |
| continue | |
| if tok.kind == 'punct': | |
| items.append(Item(i, i + 1, tok.start, tok.end, tok.text, method='punct')) | |
| i += 1 | |
| continue | |
| lit = _literal(tok) | |
| if lit: | |
| items.append(Item(i, i + 1, tok.start, tok.end, tok.text, [lit], 'literal')) | |
| i += 1 | |
| continue | |
| fz = _fuzzy(tok, lx, ov) | |
| if fz: | |
| senses, target, score = fz | |
| items.append(Item(i, i + 1, tok.start, tok.end, target, senses, 'fuzzy', score, typed=tok.text)) | |
| i += 1 | |
| continue | |
| items.append(Item(i, i + 1, tok.start, tok.end, tok.text)) | |
| i += 1 | |
| return items | |
| def _longest(texts: list[str], i: int, lx: Lexicon, ov: _Overlay, tokens: list[Token]): | |
| n = len(texts) | |
| for L in range(min(max(lx.max_len, ov.max_len), n - i), 0, -1): | |
| # A phrase never spans a sentence break. | |
| if any(tokens[k].kind == 'punct' and tokens[k].text not in '/&:' for k in range(i, i + L)): | |
| continue | |
| key = tuple(texts[i:i + L]) | |
| senses = _tenant_first(ov.exact.get(key, []), lx.exact.get(key, [])) | |
| if senses: | |
| return L, _dedupe(senses), 'context' if key in ov.exact else 'exact' | |
| skey = tuple(stem(t) for t in key) | |
| senses = _tenant_first(ov.stemmed.get(skey, []), lx.stemmed.get(skey, [])) | |
| if senses: | |
| return L, _dedupe(senses), 'context' if skey in ov.stemmed else 'stem' | |
| return None | |
| def _tenant_first(tenant: list[Sense], vocab: list[Sense]) -> list[Sense]: | |
| """A tenant's own detector named exactly like a vocabulary phrase is what the admin means: | |
| "employee code" is their EMPLOYEE_CODE detector, not also the built-in EMPLOYEE_ID.""" | |
| if any(s.slot == 'entity' for s in tenant): | |
| vocab = [s for s in vocab if s.slot != 'entity'] | |
| return list(tenant) + list(vocab) | |
| def _dedupe(senses: list[Sense]) -> list[Sense]: | |
| out: list[Sense] = [] | |
| for s in senses: | |
| if s not in out: | |
| out.append(s) | |
| return out | |
| def _literal(tok: Token) -> Sense | None: | |
| if tok.kind == 'domain': | |
| return Sense('domain', tok.value) | |
| if tok.kind == 'email': | |
| return Sense('email_addr', tok.value) | |
| if tok.kind == 'number' and tok.value is not None: | |
| return Sense('number', tok.value) | |
| if tok.kind == 'time' and tok.value is not None: | |
| return Sense('time', tok.value) | |
| if tok.kind == 'quoted': | |
| return Sense('quoted', tok.value) | |
| if tok.kind == 'path': | |
| return Sense('path', tok.value) | |
| if tok.kind == 'word' and EXT_ID.match(tok.text): | |
| return Sense('ext_id', tok.text) | |
| if tok.kind == 'word' and is_serial(tok.text): | |
| return Sense('serial', tok.text.upper()) | |
| return None | |
| def _fuzzy(tok: Token, lx: Lexicon, ov: _Overlay): | |
| w = tok.text | |
| if tok.kind != 'word' or len(w) < FUZZY_MIN or not w.isalpha() or w in lx.no_fuzzy or w in lx.words: | |
| return None | |
| # Group names in another word form: "engineers" -> Engineering, "marketers" -> Marketing. | |
| forms = [] | |
| for target, senses in ov.fuzzy.items(): | |
| if not any(s.slot == 'group' for s in senses): | |
| continue | |
| cp = len(_common_prefix(w, target)) | |
| if cp >= 6 and cp >= 0.75 * min(len(w), len(target)): | |
| forms.append((target, senses)) | |
| if len(forms) == 1: | |
| target, senses = forms[0] | |
| return _dedupe(senses), target, 0.85 | |
| limit = 1 if len(w) <= 7 else 2 | |
| best: list[tuple[int, str, list[Sense]]] = [] | |
| for pool in (ov.fuzzy, lx.fuzzy_targets): | |
| for target, senses in pool.items(): | |
| if target[0] != w[0] or abs(len(target) - len(w)) > limit: | |
| continue | |
| d = _osa(w, target, limit) | |
| if d <= limit: | |
| best.append((d, target, senses)) | |
| if not best: | |
| return None | |
| best.sort(key=lambda x: x[0]) | |
| top = [b for b in best if b[0] == best[0][0]] | |
| # Two different words equally close: guessing would be a coin toss, so do not. | |
| if len({b[1] for b in top}) > 1: | |
| return None | |
| d, target, senses = top[0] | |
| return _dedupe(senses), target, round(1 - d / max(len(w), 1), 2) | |
| def _common_prefix(a: str, b: str) -> str: | |
| n = 0 | |
| while n < min(len(a), len(b)) and a[n] == b[n]: | |
| n += 1 | |
| return a[:n] | |
| def _split_unknown_compounds(tokens: list[Token], lx: Lexicon) -> list[Token]: | |
| """'pen-drive' is not a vocabulary word, but 'pen drive' is: split hyphenated unknowns.""" | |
| out: list[Token] = [] | |
| for t in tokens: | |
| if t.kind == 'word' and '-' in t.text and t.text not in lx.words and not is_serial(t.text): | |
| pos = t.start | |
| for part in t.text.split('-'): | |
| if part: | |
| out.append(Token(part, pos, pos + len(part), 'word')) | |
| pos += len(part) + 1 | |
| else: | |
| out.append(t) | |
| return out | |