Token Classification
Transformers
Safetensors
xlm-roberta
pii
privacy
redaction
anonymization
data-masking
named-entity-recognition
Instructions to use Tracer-ML/PIIX-560m with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Tracer-ML/PIIX-560m with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="Tracer-ML/PIIX-560m")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("Tracer-ML/PIIX-560m") model = AutoModelForTokenClassification.from_pretrained("Tracer-ML/PIIX-560m", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download rules.py from Tracer-ML/PIIX-560m: direct link, hf CLI and curl.
- Browser
- Download file 8.23 kB
-
https://huggingface.co/Tracer-ML/PIIX-560m/resolve/main/rules.py
- Command line
-
hf download hf://Tracer-ML/PIIX-560m/rules.py
-
curl -L -o rules.py https://huggingface.co/Tracer-ML/PIIX-560m/resolve/main/rules.py
8.23 kB
| """Deterministic detectors for machine-format identifiers and secrets. | |
| Written from public format definitions (RFCs, checksum algorithms, vendor token prefixes), | |
| not from benchmark rows. Each rule returns typed character spans on the original text. | |
| """ | |
| import math | |
| import re | |
| TLD = (r'(?:com|org|net|edu|gov|mil|int|io|ai|co|dev|app|cloud|info|biz|xyz|online|site|tech|me|tv|cc|us|uk|de|fr|es|it|nl|pl|se|no|fi|dk|ch|at|be|cz|pt|gr|tr|ua|by|kz|ru|su|рф|cn|jp|kr|in|br|mx|ar|au|nz|ca|za|sg|hk|il|ae|eu|local|localdomain|internal|intranet|lan|corp|home|test|example|onion|svc|cluster)') | |
| LABEL = r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?' | |
| HOST = rf'(?<![\w@.-])(?:{LABEL}\.)+{TLD}(?![\w-])(?!\.[A-Za-z0-9])' | |
| FILE_EXT = {'txt', 'log', 'json', 'yaml', 'yml', 'xml', 'csv', 'py', 'js', 'ts', 'sh', 'md', 'conf', 'cfg', 'ini', 'env', | |
| 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'png', 'jpg', 'gif', 'zip', 'gz', 'tar', 'sql', 'java', 'go', 'rb', 'php', 'html', 'css'} | |
| EMAIL = re.compile(r'(?<![\w.+-])[\w.+-]+@(?:[\w-]+\.)+[\w-]{2,}(?![\w-])', re.U) | |
| URL = re.compile(r'(?<![\w])(?:(?:https?|ftp|sftp|ssh|wss?|jdbc:[a-z]+|postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://|www\.)[^\s<>"\'`,;)\]}]+', re.I) | |
| IPV4 = re.compile(r'(?<![\d.])(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:/\d{1,2}|:\d{2,5})?(?![\d.]*\d)') | |
| IPV6 = re.compile(r'(?<![\w:])(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}(?![\w:])|(?<![\w:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4}(?![\w:])') | |
| MAC = re.compile(r'(?<![\w:-])(?:[0-9A-Fa-f]{2}[:-]){5}[0-9A-Fa-f]{2}(?![\w:-])|(?<![\w.])(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4}(?![\w.])') | |
| HOSTNAME = re.compile(HOST, re.U) | |
| PHONE = re.compile(r'(?<![\w+])\+\d{1,3}[\s.-]?\(?\d{2,4}\)?(?:[\s.-]?\d{2,4}){2,4}(?!\w)|(?<![\w+(])\(\d{3,4}\)[\s.-]?\d{3}[\s.-]?\d{2,4}(?:[\s.-]?\d{2})?(?!\w)' | |
| r'|(?<![\w.-])\d{3}[.-]\d{3}[.-]\d{4}(?![\w-])|(?<![\w.-])8[\s-]?\(?\d{3}\)?[\s-]?\d{3}[\s-]?\d{2}[\s-]?\d{2}(?![\w-])') | |
| CARD = re.compile(r'(?<![\w-])(?:\d[ -]?){12,18}\d(?![\w-])') | |
| IBAN = re.compile(r'(?<!\w)[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?(?!\w)') | |
| SNILS = re.compile(r'(?<![\w-])\d{3}-\d{3}-\d{3}[ -]\d{2}(?![\w-])') | |
| INN = re.compile(r'(?i)(?<!\w)(?:инн|inn)\D{0,10}(\d{12}|\d{10})(?!\d)') | |
| PASSPORT_RU = re.compile(r'(?i)(?<!\w)(?:паспорт\w*|passport)[^\n\d]{0,40}?(\d{2}\s?\d{2}\s?(?:№\s?)?\d{6})(?!\d)') | |
| SSN = re.compile(r'(?<![\w-])(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}(?![\w-])') | |
| PEM = re.compile(r'-----BEGIN [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE)[A-Z0-9 ]*-----.*?-----END [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE)[A-Z0-9 ]*-----', re.S) | |
| JWT = re.compile(r'(?<![\w-])eyJ[\w-]{8,}\.eyJ[\w-]{8,}\.[\w-]{8,}(?![\w-])') | |
| VENDOR_KEY = re.compile( | |
| r'(?<![\w-])(?:A(?:KIA|SIA|GPA|IDA|ROA)[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,}|glpat-[\w-]{20,}' | |
| r'|xox[abposr]-[A-Za-z0-9-]{10,}|sk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}|sk_(?:live|test)_[A-Za-z0-9]{16,}|pk_(?:live|test)_[A-Za-z0-9]{16,}' | |
| r'|AIza[\w-]{35}|ya29\.[\w-]{20,}|SG\.[\w-]{16,}\.[\w-]{16,}|hf_[A-Za-z0-9]{30,}|npm_[A-Za-z0-9]{36}|dop_v1_[a-f0-9]{64}' | |
| r'|\d{8,10}:AA[\w-]{33})(?![\w-])') | |
| KEYVAL = re.compile( | |
| r'(?i)(?<![A-Za-z])(?:pass(?:word|wd|phrase)?|pwd|secret(?:[_-]?key)?|api[_-]?key|access[_-]?key|auth[_-]?token|token|client[_-]?secret' | |
| r'|private[_-]?key|credentials?|пароль|токен|секрет)\w*["\']?\s*(?:[:=]|=>|is|:=)\s*["\']?([^\s"\'<>,;]{6,})') | |
| BEARER = re.compile(r'(?i)(?<!\w)(?:bearer|basic)\s+([A-Za-z0-9._~+/=-]{16,})') | |
| # connection strings and URLs that embed a user name and password: the whole URL is a credential | |
| CRED_URL = re.compile(r'(?<![\w])[a-z][a-z0-9+.-]{1,20}://[^\s:@/"\'<>]{1,64}:[^\s@/"\'<>]{1,128}@[^\s"\'<>`,;)\]}]+', re.I) | |
| TOKEN = re.compile(r'(?<![\w/+=.-])[A-Za-z0-9+_=-]{28,}(?![\w/+=-])') | |
| # Widely known public sites: a link to them identifies nobody, so URL and hostname rules skip them. | |
| PUBLIC = re.compile(r'(?i)(?:^|[/.@])(?:github(?:usercontent)?\.com|gitlab\.com|bitbucket\.org|stackoverflow\.com|stackexchange\.com|google\.com|googleapis\.com|' | |
| r'gstatic\.com|youtube\.com|microsoft\.com|windows\.net|azure\.com|amazonaws\.com|apple\.com|mozilla\.org|wikipedia\.org|npmjs\.(?:com|org)|' | |
| r'pypi\.org|python\.org|nodejs\.org|docker\.(?:com|io)|kubernetes\.io|apache\.org|w3\.org|schema\.org|cloudflare\.com|jsdelivr\.net|unpkg\.com|' | |
| r'readthedocs\.io|medium\.com|twitter\.com|x\.com|facebook\.com|linkedin\.com|reddit\.com|yandex\.ru|vk\.com|example\.(?:com|org|net)|' | |
| r'localhost|shields\.io|travis-ci\.(?:org|com)|circleci\.com|sentry\.io|slack\.com|zoom\.us|atlassian\.(?:com|net)|oracle\.com|maven\.org|golang\.org|rust-lang\.org)(?=$|[/:?#])') | |
| def public(value): | |
| host = re.sub(r'^[a-z+.:-]*//', '', value, flags=re.I).split('/')[0].split('?')[0] | |
| return bool(PUBLIC.search('/' + host.split('@')[-1].split(':')[0])) | |
| def luhn(digits): | |
| total = 0 | |
| for i, ch in enumerate(reversed(digits)): | |
| d = int(ch) | |
| if i % 2: | |
| d = d * 2 - 9 if d > 4 else d * 2 | |
| total += d | |
| return total % 10 == 0 | |
| def iban_ok(value): | |
| s = value.replace(' ', '') | |
| if not 15 <= len(s) <= 34: | |
| return False | |
| s = s[4:] + s[:4] | |
| return int(''.join(str(int(c, 36)) for c in s)) % 97 == 1 | |
| def snils_ok(value): | |
| d = re.sub(r'\D', '', value) | |
| total = sum(int(c) * w for c, w in zip(d[:9], range(9, 0, -1))) | |
| check = 0 if total % 101 in (100, 0) and total >= 100 else total % 101 | |
| if total < 100: check = total | |
| elif total in (100, 101): check = 0 | |
| else: check = total % 101 % 100 | |
| return check == int(d[9:]) | |
| def entropy(s): | |
| return -sum(s.count(c) / len(s) * math.log2(s.count(c) / len(s)) for c in set(s)) | |
| def secretish(s): | |
| classes = sum(bool(re.search(p, s)) for p in (r'[a-z]', r'[A-Z]', r'\d')) | |
| return classes == 3 and entropy(s) >= 4.0 and not re.fullmatch(r'[0-9a-fA-F-]{32,36}', s) | |
| def find(text): | |
| out = [] | |
| def add(m, label, group=0, ok=True): | |
| if ok and m.end(group) > m.start(group): | |
| out.append(dict(start=m.start(group), end=m.end(group), label=label)) | |
| for m in CRED_URL.finditer(text): add(m, 'secret_url') | |
| for m in PEM.finditer(text): add(m, 'secret_pem') | |
| for m in JWT.finditer(text): add(m, 'secret_jwt') | |
| for m in VENDOR_KEY.finditer(text): add(m, 'secret_vendor') | |
| for m in KEYVAL.finditer(text): | |
| v = m.group(1) | |
| add(m, 'secret_kv', 1, not re.fullmatch(r'(?i)(true|false|null|none|required|changeme|string|undefined|\*+|x+|<.*>|\$\{?\w+\}?|%\w+%|\{\{.*\}\}|process\.env.*|os\.environ.*)', v)) | |
| for m in BEARER.finditer(text): add(m, 'secret_bearer', 1) | |
| for m in EMAIL.finditer(text): add(m, 'email') | |
| urls = [] | |
| for m in URL.finditer(text): | |
| end = m.end() | |
| while end > m.start() and text[end - 1] in '.,:;!?': end -= 1 | |
| urls.append((m.start(), end)) | |
| if not public(text[m.start():end]): out.append(dict(start=m.start(), end=end, label='url')) | |
| for m in TOKEN.finditer(text): # free-standing high-entropy strings only, never fragments of a URL | |
| add(m, 'secret_token', 0, secretish(m.group(0)) and not any(a <= m.start() < b for a, b in urls)) | |
| for m in IPV4.finditer(text): add(m, 'ip') | |
| for m in IPV6.finditer(text): add(m, 'ip', 0, m.group(0).count(':') >= 2) | |
| for m in MAC.finditer(text): add(m, 'mac') | |
| for m in HOSTNAME.finditer(text): | |
| add(m, 'hostname', 0, m.group(0).rsplit('.', 1)[-1].lower() not in FILE_EXT and not public(m.group(0)) and not any(a <= m.start() < b for a, b in urls)) | |
| for m in PHONE.finditer(text): | |
| add(m, 'phone', 0, 10 <= len(re.sub(r'\D', '', m.group(0))) <= 15) | |
| for m in CARD.finditer(text): | |
| d = re.sub(r'\D', '', m.group(0)); add(m, 'card_number', 0, 13 <= len(d) <= 19 and luhn(d)) | |
| for m in IBAN.finditer(text): add(m, 'iban', 0, iban_ok(m.group(0))) | |
| for m in SNILS.finditer(text): add(m, 'snils') | |
| for m in INN.finditer(text): add(m, 'inn', 1) | |
| for m in PASSPORT_RU.finditer(text): add(m, 'passport', 1) | |
| for m in SSN.finditer(text): add(m, 'ssn') | |
| return out | |