Blame for copilot/text.py

"""Text normalisation and tokenisation.

Tokens keep their character offsets in the normalised text, so every value the copilot
fills can point back at the words that produced it (the UI highlights them).
"""
from __future__ import annotations

import re
import unicodedata
from dataclasses import dataclass

MAX_CHARS = 2000

_QUOTES = {'‘': "'", '’': "'", '‚': "'", '‛': "'", '′': "'",
           '“': '"', '”': '"', '„': '"', '″': '"', '«': '"', '»': '"',
           '–': '-', '—': '-', '−': '-', ' ': ' '}

CONTRACTIONS = {
    "don't": 'do not', 'dont': 'do not', "doesn't": 'does not', 'doesnt': 'does not', "didn't": 'did not',
    "can't": 'can not', 'cant': 'can not', 'cannot': 'can not', "won't": 'will not', 'wont': 'will not',
    "shouldn't": 'should not', 'shouldnt': 'should not', "mustn't": 'must not', 'mustnt': 'must not',
    "isn't": 'is not', "aren't": 'are not', "wasn't": 'was not', "weren't": 'were not',
    "wouldn't": 'would not', "couldn't": 'could not', "haven't": 'have not', "hasn't": 'has not',
    "shan't": 'shall not', "needn't": 'need not', "mayn't": 'may not', "ain't": 'is not',
    "let's": 'let us', "it's": 'it is', "i'm": 'i am', "they're": 'they are', "we're": 'we are',
    "you're": 'you are', "that's": 'that is', "what's": 'what is', "there's": 'there is',
    "i'd": 'i would', "we'd": 'we would', "i'll": 'i will', "we'll": 'we will', "i've": 'i have', "we've": 'we have',
}

NUMBER_WORDS = {
    'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8,
    'nine': 9, 'ten': 10, 'eleven': 11, 'twelve': 12, 'thirteen': 13, 'fourteen': 14, 'fifteen': 15,
    'sixteen': 16, 'seventeen': 17, 'eighteen': 18, 'nineteen': 19, 'twenty': 20, 'thirty': 30,
    'forty': 40, 'fifty': 50, 'sixty': 60, 'seventy': 70, 'eighty': 80, 'ninety': 90,
    'hundred': 100, 'thousand': 1000, 'dozen': 12,
}

# Order matters: the first alternative that matches at a position wins.
_TOKEN_RE = re.compile(r"""
    (?P<quoted>"[^"\n]{1,80}"|(?<![\w'])'[^'\n]{1,60}'(?![\w']))
  | (?P<email>[a-z0-9._%+-]+@[a-z0-9-]+(?:\.[a-z0-9-]+)+)
  | (?P<url>https?://[^\s,;"'<>()]+)
  | (?P<path>(?:\*\*/|\*|~)?[./\\]?[a-z0-9_*-]*\.(?:env|pem|key|pfx|p12|kdbx|ssh|aws|npmrc|pypirc|netrc|git-credentials)\b|\*\*?/[^\s,;]+|\*\.[a-z0-9]{1,8}\b)
  | (?P<time>\d{1,2}(?::\d{2})?\s?(?:am|pm)\b|\d{1,2}:\d{2}\b)
  | (?P<number>\d[\d,]*(?:\.\d+)?(?=(?:kb|mb|gb|tb)\b|[^a-z0-9.,]|[.,](?![0-9])|$))   # whole numbers only: "1234abcd" is a serial, "50mb" is 50 + mb
  | (?P<word>[a-z0-9][a-z0-9_]*(?:['&.+-][a-z0-9_]+)*(?:'s)?)
  | (?P<punct>[,.;:!?/()&|\n-])
""", re.VERBOSE)

_TLD = re.compile(r'\.[a-z]{2,24}$')
# "CLAUDE.md", "config.json", "readme.txt": file names, not websites (even where the suffix is a real TLD)
FILE_EXTS = {'md', 'json', 'txt', 'env', 'yaml', 'yml', 'py', 'js', 'ts', 'pem', 'key', 'csv', 'docx', 'xlsx', 'pdf',
             'sh', 'toml', 'ini', 'cfg', 'log', 'xml', 'doc', 'xls', 'ppt', 'pptx', 'zip', 'exe', 'dll', 'jar', 'sql',
             'bak', 'conf', 'pfx', 'p12', 'kdbx', 'rules', 'lock'}
_DOMAIN = re.compile(r'^(?:[a-z0-9-]+\.)+[a-z]{2,24}$')
_SERIAL = re.compile(r'^(?=.*\d)(?=.*[a-z])[a-z0-9]+(?:-[a-z0-9]+)+$|^(?=.*\d)[a-z]{2,}\d{4,}[a-z0-9]*$'
                     r'|^(?=.*\d)(?=.*[a-z])[a-z0-9]{8,}$')   # 1234ABCD, 4C530001231120115142


@dataclass
class Token:
    text: str          # normalised form used for matching
    start: int         # offsets into the normalised text
    end: int
    kind: str          # word | number | quoted | email | domain | url | path | time | punct
    value: object = None   # number value, quoted content, cleaned domain ...

    @property
    def is_word(self) -> bool:
        return self.kind in ('word', 'domain')


_TAGS = re.compile(r'<\s*(script|style)\b.*?<\s*/\s*\1\s*>|<[^<>]{1,200}>', re.I | re.S)
_PLUS = re.compile(r'(?<=[a-z0-9])\s*\+\s*(?=[a-z])', re.I)


def clean(raw: object) -> tuple[str, list[str]]:
    """NFKC, straight quotes, no control characters, length cap. Returns (text, notes)."""
    notes: list[str] = []
    if not isinstance(raw, str):
        raw = ''   # only text is read: str() of a list or dict would turn its repr into a request
    text = unicodedata.normalize('NFKC', raw)
    text = _TAGS.sub(' ', text)          # pasted markup is never part of a request
    text = _PLUS.sub(' and ', text)      # "usb+print+bt" is a list
    text = ''.join(_QUOTES.get(c, c) for c in text)
    text = ''.join(c if (c in '\n\t' or unicodedata.category(c)[0] != 'C') else ' ' for c in text)
    text = re.sub(r'[ \t]+', ' ', text).strip()
    if len(text) > MAX_CHARS:
        text = text[:MAX_CHARS]
        notes.append('INPUT_TRUNCATED')
    return text, notes


def tokenize(text: str) -> list[Token]:
    """Tokens of the (already cleaned) text. Offsets refer to `text`."""
    low = text.lower()
    out: list[Token] = []
    for m in _TOKEN_RE.finditer(low):
        kind = m.lastgroup
        s, e = m.span()
        t = m.group()
        if kind == 'quoted':
            inner = t[1:-1].strip()
            if inner:
                out.append(Token(t, s, e, 'quoted', inner))
            continue
        if kind == 'email':
            out.append(Token(t, s, e, 'email', t))
            continue
        if kind == 'url':
            host = re.sub(r'^https?://', '', t).split('/')[0].split(':')[0]
            out.append(Token(host, s, e, 'domain', host))
            continue
        if kind == 'path':
            out.append(Token(t, s, e, 'path', t))
            continue
        if kind == 'time':
            out.append(Token(t.replace(' ', ''), s, e, 'time', _parse_time(t)))
            continue
        if kind == 'number':
            try:
                v = float(t.replace(',', ''))
                v = int(v) if v.is_integer() else v
            except ValueError:
                v = None
            out.append(Token(t, s, e, 'number', v))
            continue
        if kind == 'punct':
            out.append(Token(t, s, e, 'punct'))
            continue
        # words
        exp = CONTRACTIONS.get(t)
        if exp:
            for part in exp.split():
                out.append(Token(part, s, e, 'word'))
            continue
        if t.endswith("'s") and len(t) > 3:
            t_base = t[:-2]
        else:
            t_base = t
        if t_base in NUMBER_WORDS:
            out.append(Token(t_base, s, e, 'number', NUMBER_WORDS[t_base]))
            continue
        if '.' in t_base and _DOMAIN.match(t_base) and _TLD.search(t_base):
            if t_base.rsplit('.', 1)[1] in FILE_EXTS:
                out.append(Token(t_base, s, e, 'path', t_base))
            else:
                out.append(Token(t_base, s, e, 'domain', t_base))
            continue
        out.append(Token(t_base if t_base != t and len(t_base) > 1 else t, s, e, 'word'))
    return _merge_number_words(out)


def _merge_number_words(tokens: list[Token]) -> list[Token]:
    """'twenty five' -> 25, 'two hundred' -> 200, 'a hundred' -> 100."""
    out: list[Token] = []
    for t in tokens:
        prev = out[-1] if out else None
        if t.kind == 'number' and t.text in NUMBER_WORDS and prev is not None:
            if prev.kind == 'number' and isinstance(prev.value, int):
                if t.value in (100, 1000):
                    prev.value = prev.value * t.value
                    prev.end, prev.text = t.end, f'{prev.text} {t.text}'
                    continue
                if prev.value >= 20 and prev.value % 10 == 0 and t.value < 10:
                    prev.value += t.value
                    prev.end, prev.text = t.end, f'{prev.text} {t.text}'
                    continue
            if prev.kind == 'word' and prev.text == 'a' and t.value in (100, 1000, 12):
                out[-1] = Token(f'a {t.text}', prev.start, t.end, 'number', t.value)
                continue
        out.append(t)
    return out


def _parse_time(t: str) -> int | None:
    """'6pm' -> 18, '9:30 am' -> 9, '18:00' -> 18 (hours only: the platform stores whole hours)."""
    m = re.match(r'(\d{1,2})(?::(\d{2}))?\s?(am|pm)?', t)
    if not m:
        return None
    h = int(m.group(1))
    ap = m.group(3)
    if ap == 'pm' and h < 12:
        h += 12
    if ap == 'am' and h == 12:
        h = 0
    return h if 0 <= h <= 23 else None


def is_domain(s: str) -> bool:
    return bool(_DOMAIN.match(s))


def is_serial(s: str) -> bool:
    return bool(_SERIAL.match(s))


def stem(word: str) -> str:
    """Plural to singular, conservatively. Only used when the exact word is not in the vocabulary."""
    if len(word) < 3 or not word.isalpha():
        return word
    if len(word) == 3:
        # "ids" -> "id", "dms" -> "dm"; leave words that only look plural ("bus", "gas", "his")
        return word[:-1] if word.endswith('s') and word not in ('bus', 'gas', 'his', 'has', 'was', 'yes', 'its',
                                                                'ios', 'aws', 'sms', 'ssn', 'pls') else word
    if word.endswith('ies') and len(word) > 4:
        return word[:-3] + 'y'
    if word.endswith(('ses', 'xes', 'ches', 'shes')):
        return word[:-2]
    if word.endswith('s') and not word.endswith(('ss', 'us', 'is')):
        return word[:-1]
    return word