"""Deterministic detectors for machine-format identifiers and secrets. Written from public format definitions (RFCs, checksum algorithms, vendor token prefixes), not from benchmark rows. Each rule returns typed character spans on the original text. """ import math import re TLD = (r'(?:com|org|net|edu|gov|mil|int|io|ai|co|dev|app|cloud|info|biz|xyz|online|site|tech|me|tv|cc|us|uk|de|fr|es|it|nl|pl|se|no|fi|dk|ch|at|be|cz|pt|gr|tr|ua|by|kz|ru|su|рф|cn|jp|kr|in|br|mx|ar|au|nz|ca|za|sg|hk|il|ae|eu|local|localdomain|internal|intranet|lan|corp|home|test|example|onion|svc|cluster)') LABEL = r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?' HOST = rf'(?"\'`,;)\]}]+', re.I) IPV4 = re.compile(r'(?|is|:=)\s*["\']?([^\s"\'<>,;]{6,})') BEARER = re.compile(r'(?i)(?]{1,64}:[^\s@/"\'<>]{1,128}@[^\s"\'<>`,;)\]}]+', re.I) TOKEN = re.compile(r'(? 4 else d * 2 total += d return total % 10 == 0 def iban_ok(value): s = value.replace(' ', '') if not 15 <= len(s) <= 34: return False s = s[4:] + s[:4] return int(''.join(str(int(c, 36)) for c in s)) % 97 == 1 def snils_ok(value): d = re.sub(r'\D', '', value) total = sum(int(c) * w for c, w in zip(d[:9], range(9, 0, -1))) check = 0 if total % 101 in (100, 0) and total >= 100 else total % 101 if total < 100: check = total elif total in (100, 101): check = 0 else: check = total % 101 % 100 return check == int(d[9:]) def entropy(s): return -sum(s.count(c) / len(s) * math.log2(s.count(c) / len(s)) for c in set(s)) def secretish(s): classes = sum(bool(re.search(p, s)) for p in (r'[a-z]', r'[A-Z]', r'\d')) return classes == 3 and entropy(s) >= 4.0 and not re.fullmatch(r'[0-9a-fA-F-]{32,36}', s) def find(text): out = [] def add(m, label, group=0, ok=True): if ok and m.end(group) > m.start(group): out.append(dict(start=m.start(group), end=m.end(group), label=label)) for m in CRED_URL.finditer(text): add(m, 'secret_url') for m in PEM.finditer(text): add(m, 'secret_pem') for m in JWT.finditer(text): add(m, 'secret_jwt') for m in VENDOR_KEY.finditer(text): add(m, 'secret_vendor') for m in KEYVAL.finditer(text): v = m.group(1) add(m, 'secret_kv', 1, not re.fullmatch(r'(?i)(true|false|null|none|required|changeme|string|undefined|\*+|x+|<.*>|\$\{?\w+\}?|%\w+%|\{\{.*\}\}|process\.env.*|os\.environ.*)', v)) for m in BEARER.finditer(text): add(m, 'secret_bearer', 1) for m in EMAIL.finditer(text): add(m, 'email') urls = [] for m in URL.finditer(text): end = m.end() while end > m.start() and text[end - 1] in '.,:;!?': end -= 1 urls.append((m.start(), end)) if not public(text[m.start():end]): out.append(dict(start=m.start(), end=end, label='url')) for m in TOKEN.finditer(text): # free-standing high-entropy strings only, never fragments of a URL add(m, 'secret_token', 0, secretish(m.group(0)) and not any(a <= m.start() < b for a, b in urls)) for m in IPV4.finditer(text): add(m, 'ip') for m in IPV6.finditer(text): add(m, 'ip', 0, m.group(0).count(':') >= 2) for m in MAC.finditer(text): add(m, 'mac') for m in HOSTNAME.finditer(text): add(m, 'hostname', 0, m.group(0).rsplit('.', 1)[-1].lower() not in FILE_EXT and not public(m.group(0)) and not any(a <= m.start() < b for a, b in urls)) for m in PHONE.finditer(text): add(m, 'phone', 0, 10 <= len(re.sub(r'\D', '', m.group(0))) <= 15) for m in CARD.finditer(text): d = re.sub(r'\D', '', m.group(0)); add(m, 'card_number', 0, 13 <= len(d) <= 19 and luhn(d)) for m in IBAN.finditer(text): add(m, 'iban', 0, iban_ok(m.group(0))) for m in SNILS.finditer(text): add(m, 'snils') for m in INN.finditer(text): add(m, 'inn', 1) for m in PASSPORT_RU.finditer(text): add(m, 'passport', 1) for m in SSN.finditer(text): add(m, 'ssn') return out