File size: 8,232 Bytes
26d1584
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"""Deterministic detectors for machine-format identifiers and secrets.

Written from public format definitions (RFCs, checksum algorithms, vendor token prefixes),
not from benchmark rows. Each rule returns typed character spans on the original text.
"""
import math
import re

TLD = (r'(?:com|org|net|edu|gov|mil|int|io|ai|co|dev|app|cloud|info|biz|xyz|online|site|tech|me|tv|cc|us|uk|de|fr|es|it|nl|pl|se|no|fi|dk|ch|at|be|cz|pt|gr|tr|ua|by|kz|ru|su|рф|cn|jp|kr|in|br|mx|ar|au|nz|ca|za|sg|hk|il|ae|eu|local|localdomain|internal|intranet|lan|corp|home|test|example|onion|svc|cluster)')
LABEL = r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
HOST = rf'(?<![\w@.-])(?:{LABEL}\.)+{TLD}(?![\w-])(?!\.[A-Za-z0-9])'
FILE_EXT = {'txt', 'log', 'json', 'yaml', 'yml', 'xml', 'csv', 'py', 'js', 'ts', 'sh', 'md', 'conf', 'cfg', 'ini', 'env',
            'pdf', 'doc', 'docx', 'xls', 'xlsx', 'png', 'jpg', 'gif', 'zip', 'gz', 'tar', 'sql', 'java', 'go', 'rb', 'php', 'html', 'css'}

EMAIL = re.compile(r'(?<![\w.+-])[\w.+-]+@(?:[\w-]+\.)+[\w-]{2,}(?![\w-])', re.U)
URL = re.compile(r'(?<![\w])(?:(?:https?|ftp|sftp|ssh|wss?|jdbc:[a-z]+|postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://|www\.)[^\s<>"\'`,;)\]}]+', re.I)
IPV4 = re.compile(r'(?<![\d.])(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:/\d{1,2}|:\d{2,5})?(?![\d.]*\d)')
IPV6 = re.compile(r'(?<![\w:])(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}(?![\w:])|(?<![\w:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4}(?![\w:])')
MAC = re.compile(r'(?<![\w:-])(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}(?![\w:-])|(?<![\w.])(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4}(?![\w.])')
HOSTNAME = re.compile(HOST, re.U)
PHONE = re.compile(r'(?<![\w+])\+\d{1,3}[\s.-]?\(?\d{2,4}\)?(?:[\s.-]?\d{2,4}){2,4}(?!\w)|(?<![\w+(])\(\d{3,4}\)[\s.-]?\d{3}[\s.-]?\d{2,4}(?:[\s.-]?\d{2})?(?!\w)'
                   r'|(?<![\w.-])\d{3}[.-]\d{3}[.-]\d{4}(?![\w-])|(?<![\w.-])8[\s-]?\(?\d{3}\)?[\s-]?\d{3}[\s-]?\d{2}[\s-]?\d{2}(?![\w-])')
CARD = re.compile(r'(?<![\w-])(?:\d[ -]?){12,18}\d(?![\w-])')
IBAN = re.compile(r'(?<!\w)[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?(?!\w)')
SNILS = re.compile(r'(?<![\w-])\d{3}-\d{3}-\d{3}[ -]\d{2}(?![\w-])')
INN = re.compile(r'(?i)(?<!\w)(?:инн|inn)\D{0,10}(\d{12}|\d{10})(?!\d)')
PASSPORT_RU = re.compile(r'(?i)(?<!\w)(?:паспорт\w*|passport)[^\n\d]{0,40}?(\d{2}\s?\d{2}\s?(?:№\s?)?\d{6})(?!\d)')
SSN = re.compile(r'(?<![\w-])(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}(?![\w-])')
PEM = re.compile(r'-----BEGIN [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE)[A-Z0-9 ]*-----.*?-----END [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE)[A-Z0-9 ]*-----', re.S)
JWT = re.compile(r'(?<![\w-])eyJ[\w-]{8,}\.eyJ[\w-]{8,}\.[\w-]{8,}(?![\w-])')
VENDOR_KEY = re.compile(
    r'(?<![\w-])(?:A(?:KIA|SIA|GPA|IDA|ROA)[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,}|glpat-[\w-]{20,}'
    r'|xox[abposr]-[A-Za-z0-9-]{10,}|sk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}|sk_(?:live|test)_[A-Za-z0-9]{16,}|pk_(?:live|test)_[A-Za-z0-9]{16,}'
    r'|AIza[\w-]{35}|ya29\.[\w-]{20,}|SG\.[\w-]{16,}\.[\w-]{16,}|hf_[A-Za-z0-9]{30,}|npm_[A-Za-z0-9]{36}|dop_v1_[a-f0-9]{64}'
    r'|\d{8,10}:AA[\w-]{33})(?![\w-])')
KEYVAL = re.compile(
    r'(?i)(?<![A-Za-z])(?:pass(?:word|wd|phrase)?|pwd|secret(?:[_-]?key)?|api[_-]?key|access[_-]?key|auth[_-]?token|token|client[_-]?secret'
    r'|private[_-]?key|credentials?|пароль|токен|секрет)\w*["\']?\s*(?:[:=]|=>|is|:=)\s*["\']?([^\s"\'<>,;]{6,})')
BEARER = re.compile(r'(?i)(?<!\w)(?:bearer|basic)\s+([A-Za-z0-9._~+/=-]{16,})')
# connection strings and URLs that embed a user name and password: the whole URL is a credential
CRED_URL = re.compile(r'(?<![\w])[a-z][a-z0-9+.-]{1,20}://[^\s:@/"\'<>]{1,64}:[^\s@/"\'<>]{1,128}@[^\s"\'<>`,;)\]}]+', re.I)
TOKEN = re.compile(r'(?<![\w/+=.-])[A-Za-z0-9+_=-]{28,}(?![\w/+=-])')


# Widely known public sites: a link to them identifies nobody, so URL and hostname rules skip them.
PUBLIC = re.compile(r'(?i)(?:^|[/.@])(?:github(?:usercontent)?\.com|gitlab\.com|bitbucket\.org|stackoverflow\.com|stackexchange\.com|google\.com|googleapis\.com|'
                    r'gstatic\.com|youtube\.com|microsoft\.com|windows\.net|azure\.com|amazonaws\.com|apple\.com|mozilla\.org|wikipedia\.org|npmjs\.(?:com|org)|'
                    r'pypi\.org|python\.org|nodejs\.org|docker\.(?:com|io)|kubernetes\.io|apache\.org|w3\.org|schema\.org|cloudflare\.com|jsdelivr\.net|unpkg\.com|'
                    r'readthedocs\.io|medium\.com|twitter\.com|x\.com|facebook\.com|linkedin\.com|reddit\.com|yandex\.ru|vk\.com|example\.(?:com|org|net)|'
                    r'localhost|shields\.io|travis-ci\.(?:org|com)|circleci\.com|sentry\.io|slack\.com|zoom\.us|atlassian\.(?:com|net)|oracle\.com|maven\.org|golang\.org|rust-lang\.org)(?=$|[/:?#])')


def public(value):
    host = re.sub(r'^[a-z+.:-]*//', '', value, flags=re.I).split('/')[0].split('?')[0]
    return bool(PUBLIC.search('/' + host.split('@')[-1].split(':')[0]))


def luhn(digits):
    total = 0
    for i, ch in enumerate(reversed(digits)):
        d = int(ch)
        if i % 2:
            d = d * 2 - 9 if d > 4 else d * 2
        total += d
    return total % 10 == 0


def iban_ok(value):
    s = value.replace(' ', '')
    if not 15 <= len(s) <= 34:
        return False
    s = s[4:] + s[:4]
    return int(''.join(str(int(c, 36)) for c in s)) % 97 == 1


def snils_ok(value):
    d = re.sub(r'\D', '', value)
    total = sum(int(c) * w for c, w in zip(d[:9], range(9, 0, -1)))
    check = 0 if total % 101 in (100, 0) and total >= 100 else total % 101
    if total < 100: check = total
    elif total in (100, 101): check = 0
    else: check = total % 101 % 100
    return check == int(d[9:])


def entropy(s):
    return -sum(s.count(c) / len(s) * math.log2(s.count(c) / len(s)) for c in set(s))


def secretish(s):
    classes = sum(bool(re.search(p, s)) for p in (r'[a-z]', r'[A-Z]', r'\d'))
    return classes == 3 and entropy(s) >= 4.0 and not re.fullmatch(r'[0-9a-fA-F-]{32,36}', s)


def find(text):
    out = []

    def add(m, label, group=0, ok=True):
        if ok and m.end(group) > m.start(group):
            out.append(dict(start=m.start(group), end=m.end(group), label=label))

    for m in CRED_URL.finditer(text): add(m, 'secret_url')
    for m in PEM.finditer(text): add(m, 'secret_pem')
    for m in JWT.finditer(text): add(m, 'secret_jwt')
    for m in VENDOR_KEY.finditer(text): add(m, 'secret_vendor')
    for m in KEYVAL.finditer(text):
        v = m.group(1)
        add(m, 'secret_kv', 1, not re.fullmatch(r'(?i)(true|false|null|none|required|changeme|string|undefined|\*+|x+|<.*>|\$\{?\w+\}?|%\w+%|\{\{.*\}\}|process\.env.*|os\.environ.*)', v))
    for m in BEARER.finditer(text): add(m, 'secret_bearer', 1)
    for m in EMAIL.finditer(text): add(m, 'email')
    urls = []
    for m in URL.finditer(text):
        end = m.end()
        while end > m.start() and text[end - 1] in '.,:;!?': end -= 1
        urls.append((m.start(), end))
        if not public(text[m.start():end]): out.append(dict(start=m.start(), end=end, label='url'))
    for m in TOKEN.finditer(text):  # free-standing high-entropy strings only, never fragments of a URL
        add(m, 'secret_token', 0, secretish(m.group(0)) and not any(a <= m.start() < b for a, b in urls))
    for m in IPV4.finditer(text): add(m, 'ip')
    for m in IPV6.finditer(text): add(m, 'ip', 0, m.group(0).count(':') >= 2)
    for m in MAC.finditer(text): add(m, 'mac')
    for m in HOSTNAME.finditer(text):
        add(m, 'hostname', 0, m.group(0).rsplit('.', 1)[-1].lower() not in FILE_EXT and not public(m.group(0)) and not any(a <= m.start() < b for a, b in urls))
    for m in PHONE.finditer(text):
        add(m, 'phone', 0, 10 <= len(re.sub(r'\D', '', m.group(0))) <= 15)
    for m in CARD.finditer(text):
        d = re.sub(r'\D', '', m.group(0)); add(m, 'card_number', 0, 13 <= len(d) <= 19 and luhn(d))
    for m in IBAN.finditer(text): add(m, 'iban', 0, iban_ok(m.group(0)))
    for m in SNILS.finditer(text): add(m, 'snils')
    for m in INN.finditer(text): add(m, 'inn', 1)
    for m in PASSPORT_RU.finditer(text): add(m, 'passport', 1)
    for m in SSN.finditer(text): add(m, 'ssn')
    return out