PIIX-560m / rules.py
adamrida's picture
PIIX-560m
26d1584
Raw History Blame Contribute Delete
8.23 kB
"""Deterministic detectors for machine-format identifiers and secrets.
Written from public format definitions (RFCs, checksum algorithms, vendor token prefixes),
not from benchmark rows. Each rule returns typed character spans on the original text.
"""
import math
import re
TLD = (r'(?:com|org|net|edu|gov|mil|int|io|ai|co|dev|app|cloud|info|biz|xyz|online|site|tech|me|tv|cc|us|uk|de|fr|es|it|nl|pl|se|no|fi|dk|ch|at|be|cz|pt|gr|tr|ua|by|kz|ru|su|рф|cn|jp|kr|in|br|mx|ar|au|nz|ca|za|sg|hk|il|ae|eu|local|localdomain|internal|intranet|lan|corp|home|test|example|onion|svc|cluster)')
LABEL = r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
HOST = rf'(?<![\w@.-])(?:{LABEL}\.)+{TLD}(?![\w-])(?!\.[A-Za-z0-9])'
FILE_EXT = {'txt', 'log', 'json', 'yaml', 'yml', 'xml', 'csv', 'py', 'js', 'ts', 'sh', 'md', 'conf', 'cfg', 'ini', 'env',
'pdf', 'doc', 'docx', 'xls', 'xlsx', 'png', 'jpg', 'gif', 'zip', 'gz', 'tar', 'sql', 'java', 'go', 'rb', 'php', 'html', 'css'}
EMAIL = re.compile(r'(?<![\w.+-])[\w.+-]+@(?:[\w-]+\.)+[\w-]{2,}(?![\w-])', re.U)
URL = re.compile(r'(?<![\w])(?:(?:https?|ftp|sftp|ssh|wss?|jdbc:[a-z]+|postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://|www\.)[^\s<>"\'`,;)\]}]+', re.I)
IPV4 = re.compile(r'(?<![\d.])(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:/\d{1,2}|:\d{2,5})?(?![\d.]*\d)')
IPV6 = re.compile(r'(?<![\w:])(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}(?![\w:])|(?<![\w:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4}(?![\w:])')
MAC = re.compile(r'(?<![\w:-])(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}(?![\w:-])|(?<![\w.])(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4}(?![\w.])')
HOSTNAME = re.compile(HOST, re.U)
PHONE = re.compile(r'(?<![\w+])\+\d{1,3}[\s.-]?\(?\d{2,4}\)?(?:[\s.-]?\d{2,4}){2,4}(?!\w)|(?<![\w+(])\(\d{3,4}\)[\s.-]?\d{3}[\s.-]?\d{2,4}(?:[\s.-]?\d{2})?(?!\w)'
r'|(?<![\w.-])\d{3}[.-]\d{3}[.-]\d{4}(?![\w-])|(?<![\w.-])8[\s-]?\(?\d{3}\)?[\s-]?\d{3}[\s-]?\d{2}[\s-]?\d{2}(?![\w-])')
CARD = re.compile(r'(?<![\w-])(?:\d[ -]?){12,18}\d(?![\w-])')
IBAN = re.compile(r'(?<!\w)[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?(?!\w)')
SNILS = re.compile(r'(?<![\w-])\d{3}-\d{3}-\d{3}[ -]\d{2}(?![\w-])')
INN = re.compile(r'(?i)(?<!\w)(?:инн|inn)\D{0,10}(\d{12}|\d{10})(?!\d)')
PASSPORT_RU = re.compile(r'(?i)(?<!\w)(?:паспорт\w*|passport)[^\n\d]{0,40}?(\d{2}\s?\d{2}\s?(?:№\s?)?\d{6})(?!\d)')
SSN = re.compile(r'(?<![\w-])(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}(?![\w-])')
PEM = re.compile(r'-----BEGIN [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE)[A-Z0-9 ]*-----.*?-----END [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE)[A-Z0-9 ]*-----', re.S)
JWT = re.compile(r'(?<![\w-])eyJ[\w-]{8,}\.eyJ[\w-]{8,}\.[\w-]{8,}(?![\w-])')
VENDOR_KEY = re.compile(
r'(?<![\w-])(?:A(?:KIA|SIA|GPA|IDA|ROA)[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,}|glpat-[\w-]{20,}'
r'|xox[abposr]-[A-Za-z0-9-]{10,}|sk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}|sk_(?:live|test)_[A-Za-z0-9]{16,}|pk_(?:live|test)_[A-Za-z0-9]{16,}'
r'|AIza[\w-]{35}|ya29\.[\w-]{20,}|SG\.[\w-]{16,}\.[\w-]{16,}|hf_[A-Za-z0-9]{30,}|npm_[A-Za-z0-9]{36}|dop_v1_[a-f0-9]{64}'
r'|\d{8,10}:AA[\w-]{33})(?![\w-])')
KEYVAL = re.compile(
r'(?i)(?<![A-Za-z])(?:pass(?:word|wd|phrase)?|pwd|secret(?:[_-]?key)?|api[_-]?key|access[_-]?key|auth[_-]?token|token|client[_-]?secret'
r'|private[_-]?key|credentials?|пароль|токен|секрет)\w*["\']?\s*(?:[:=]|=>|is|:=)\s*["\']?([^\s"\'<>,;]{6,})')
BEARER = re.compile(r'(?i)(?<!\w)(?:bearer|basic)\s+([A-Za-z0-9._~+/=-]{16,})')
# connection strings and URLs that embed a user name and password: the whole URL is a credential
CRED_URL = re.compile(r'(?<![\w])[a-z][a-z0-9+.-]{1,20}://[^\s:@/"\'<>]{1,64}:[^\s@/"\'<>]{1,128}@[^\s"\'<>`,;)\]}]+', re.I)
TOKEN = re.compile(r'(?<![\w/+=.-])[A-Za-z0-9+_=-]{28,}(?![\w/+=-])')
# Widely known public sites: a link to them identifies nobody, so URL and hostname rules skip them.
PUBLIC = re.compile(r'(?i)(?:^|[/.@])(?:github(?:usercontent)?\.com|gitlab\.com|bitbucket\.org|stackoverflow\.com|stackexchange\.com|google\.com|googleapis\.com|'
r'gstatic\.com|youtube\.com|microsoft\.com|windows\.net|azure\.com|amazonaws\.com|apple\.com|mozilla\.org|wikipedia\.org|npmjs\.(?:com|org)|'
r'pypi\.org|python\.org|nodejs\.org|docker\.(?:com|io)|kubernetes\.io|apache\.org|w3\.org|schema\.org|cloudflare\.com|jsdelivr\.net|unpkg\.com|'
r'readthedocs\.io|medium\.com|twitter\.com|x\.com|facebook\.com|linkedin\.com|reddit\.com|yandex\.ru|vk\.com|example\.(?:com|org|net)|'
r'localhost|shields\.io|travis-ci\.(?:org|com)|circleci\.com|sentry\.io|slack\.com|zoom\.us|atlassian\.(?:com|net)|oracle\.com|maven\.org|golang\.org|rust-lang\.org)(?=$|[/:?#])')
def public(value):
host = re.sub(r'^[a-z+.:-]*//', '', value, flags=re.I).split('/')[0].split('?')[0]
return bool(PUBLIC.search('/' + host.split('@')[-1].split(':')[0]))
def luhn(digits):
total = 0
for i, ch in enumerate(reversed(digits)):
d = int(ch)
if i % 2:
d = d * 2 - 9 if d > 4 else d * 2
total += d
return total % 10 == 0
def iban_ok(value):
s = value.replace(' ', '')
if not 15 <= len(s) <= 34:
return False
s = s[4:] + s[:4]
return int(''.join(str(int(c, 36)) for c in s)) % 97 == 1
def snils_ok(value):
d = re.sub(r'\D', '', value)
total = sum(int(c) * w for c, w in zip(d[:9], range(9, 0, -1)))
check = 0 if total % 101 in (100, 0) and total >= 100 else total % 101
if total < 100: check = total
elif total in (100, 101): check = 0
else: check = total % 101 % 100
return check == int(d[9:])
def entropy(s):
return -sum(s.count(c) / len(s) * math.log2(s.count(c) / len(s)) for c in set(s))
def secretish(s):
classes = sum(bool(re.search(p, s)) for p in (r'[a-z]', r'[A-Z]', r'\d'))
return classes == 3 and entropy(s) >= 4.0 and not re.fullmatch(r'[0-9a-fA-F-]{32,36}', s)
def find(text):
out = []
def add(m, label, group=0, ok=True):
if ok and m.end(group) > m.start(group):
out.append(dict(start=m.start(group), end=m.end(group), label=label))
for m in CRED_URL.finditer(text): add(m, 'secret_url')
for m in PEM.finditer(text): add(m, 'secret_pem')
for m in JWT.finditer(text): add(m, 'secret_jwt')
for m in VENDOR_KEY.finditer(text): add(m, 'secret_vendor')
for m in KEYVAL.finditer(text):
v = m.group(1)
add(m, 'secret_kv', 1, not re.fullmatch(r'(?i)(true|false|null|none|required|changeme|string|undefined|\*+|x+|<.*>|\$\{?\w+\}?|%\w+%|\{\{.*\}\}|process\.env.*|os\.environ.*)', v))
for m in BEARER.finditer(text): add(m, 'secret_bearer', 1)
for m in EMAIL.finditer(text): add(m, 'email')
urls = []
for m in URL.finditer(text):
end = m.end()
while end > m.start() and text[end - 1] in '.,:;!?': end -= 1
urls.append((m.start(), end))
if not public(text[m.start():end]): out.append(dict(start=m.start(), end=end, label='url'))
for m in TOKEN.finditer(text): # free-standing high-entropy strings only, never fragments of a URL
add(m, 'secret_token', 0, secretish(m.group(0)) and not any(a <= m.start() < b for a, b in urls))
for m in IPV4.finditer(text): add(m, 'ip')
for m in IPV6.finditer(text): add(m, 'ip', 0, m.group(0).count(':') >= 2)
for m in MAC.finditer(text): add(m, 'mac')
for m in HOSTNAME.finditer(text):
add(m, 'hostname', 0, m.group(0).rsplit('.', 1)[-1].lower() not in FILE_EXT and not public(m.group(0)) and not any(a <= m.start() < b for a, b in urls))
for m in PHONE.finditer(text):
add(m, 'phone', 0, 10 <= len(re.sub(r'\D', '', m.group(0))) <= 15)
for m in CARD.finditer(text):
d = re.sub(r'\D', '', m.group(0)); add(m, 'card_number', 0, 13 <= len(d) <= 19 and luhn(d))
for m in IBAN.finditer(text): add(m, 'iban', 0, iban_ok(m.group(0)))
for m in SNILS.finditer(text): add(m, 'snils')
for m in INN.finditer(text): add(m, 'inn', 1)
for m in PASSPORT_RU.finditer(text): add(m, 'passport', 1)
for m in SSN.finditer(text): add(m, 'ssn')
return out