Francesco Paolo Passaro commited on
Initial PrivacyGuardML checkpoint (16 categories, 39 languages)
Browse files- .gitattributes +35 -35
- LICENSE +21 -0
- MODEL_INFO.txt +12 -0
- README.md +110 -0
- config.json +92 -0
- model.bin +3 -0
- vocab.json +0 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,35 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Passaro Francesco Paolo
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
MODEL_INFO.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
PrivacyGuardML model file
|
| 2 |
+
==========================
|
| 3 |
+
Generated by Synthelion's own training pipeline:
|
| 4 |
+
devtools/train_privacyguardml.py
|
| 5 |
+
|
| 6 |
+
Training data: synthetic PII values (generated from Synthelion's own
|
| 7 |
+
checksum validators in synthelion/privacy_validators.py and rule
|
| 8 |
+
patterns in synthelion/privacy_rules.yaml) inserted into real
|
| 9 |
+
multilingual carrier sentences (39 languages, Wikipedia corpora).
|
| 10 |
+
Architecture: own transformer encoder, BIO tagging head.
|
| 11 |
+
Runtime: fully offline, CPU-only. This is the only privacy.use_ml
|
| 12 |
+
backend PrivacyGuard supports — no third-party model.
|
README.md
CHANGED
|
@@ -1,3 +1,113 @@
|
|
| 1 |
---
|
| 2 |
license: mit
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: mit
|
| 3 |
+
library_name: privacyguardml
|
| 4 |
+
tags:
|
| 5 |
+
- token-classification
|
| 6 |
+
- pii-detection
|
| 7 |
+
- ner
|
| 8 |
+
- transformer encoder
|
| 9 |
+
- multilingual
|
| 10 |
+
language:
|
| 11 |
+
- af
|
| 12 |
+
- be
|
| 13 |
+
- bg
|
| 14 |
+
- ca
|
| 15 |
+
- cs
|
| 16 |
+
- da
|
| 17 |
+
- de
|
| 18 |
+
- el
|
| 19 |
+
- en
|
| 20 |
+
- es
|
| 21 |
+
- et
|
| 22 |
+
- eu
|
| 23 |
+
- fi
|
| 24 |
+
- fr
|
| 25 |
+
- ga
|
| 26 |
+
- gl
|
| 27 |
+
- hi
|
| 28 |
+
- hr
|
| 29 |
+
- hu
|
| 30 |
+
- is
|
| 31 |
+
- it
|
| 32 |
+
- ja
|
| 33 |
+
- la
|
| 34 |
+
- lt
|
| 35 |
+
- lv
|
| 36 |
+
- mk
|
| 37 |
+
- nl
|
| 38 |
+
- no
|
| 39 |
+
- pl
|
| 40 |
+
- pt
|
| 41 |
+
- ro
|
| 42 |
+
- ru
|
| 43 |
+
- sk
|
| 44 |
+
- sl
|
| 45 |
+
- sq
|
| 46 |
+
- sr
|
| 47 |
+
- sv
|
| 48 |
+
- uk
|
| 49 |
+
- zh
|
| 50 |
---
|
| 51 |
+
|
| 52 |
+
# PrivacyGuardML
|
| 53 |
+
|
| 54 |
+
PII confirmation model for [Synthelion](https://github.com/francescopaolopassaro/Synthelion)'s PrivacyGuard — the opt-in ML-assisted confirmation tier (`privacy.use_ml`).
|
| 55 |
+
|
| 56 |
+
A small, fully offline transformer encoder trained with self-distillation: synthetic PII values are generated from Synthelion's own checksum validators and rule patterns, inserted into real multilingual sentences, and the encoder learns to find them back by shape and context via per-token BIO tagging. It is the sole ML backend PrivacyGuard supports — no third-party model.
|
| 57 |
+
|
| 58 |
+
**What it's for, precisely.** PrivacyGuard's core tier is regex + algorithmic checksum (or context keyword, for the rules with no checksum), zero-ML, and stays the default. This model is only the confirmation signal: a value that matches a rule's shape but has no context keyword nearby can be confirmed by an overlapping prediction from this model. A failed checksum still vetoes detection regardless of what the model says — it can only recover recall the strict context gate intentionally trades away, never introduce a false positive the validator would reject.
|
| 59 |
+
|
| 60 |
+
## Architecture
|
| 61 |
+
|
| 62 |
+
```
|
| 63 |
+
Char-ngram hashing -> word embedding + shape-feature embedding -> positional encoding
|
| 64 |
+
-> 2-layer TransformerEncoder (d=128, h=4) -> Linear(33) BIO-tag logits
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
| Parameter | Value |
|
| 68 |
+
|-----------|-------|
|
| 69 |
+
| d_model | 128 |
|
| 70 |
+
| n_heads | 4 |
|
| 71 |
+
| n_layers | 2 |
|
| 72 |
+
| ffn_dim | 512 |
|
| 73 |
+
| Vocabulary | 40,000 words + 8,192 char-ngram buckets |
|
| 74 |
+
| Max seq len | 96 |
|
| 75 |
+
| Input features | 21 (digit/at/plus/separator/uppercase flags + length buckets) |
|
| 76 |
+
| Output | 33-class BIO tag logits (16 categories x B/I + O) |
|
| 77 |
+
|
| 78 |
+
## Categories
|
| 79 |
+
|
| 80 |
+
Checksum-backed: `PHONE`, `EMAIL`, `CREDITCARD`, `IBAN`, `NATIONALID`, `TAXID`, `SSN`, `GPS`.
|
| 81 |
+
Context-only (no checksum, PrivacyGuard's remaining rule categories): `VEHICLEPLATE`, `BADGEID`, `BUSINESSID`, `SECRET`, `SOCIALHANDLE`, `LEGALCASE`, `PNRCODE`, `MINORDATA`.
|
| 82 |
+
|
| 83 |
+
## Quick start
|
| 84 |
+
|
| 85 |
+
```python
|
| 86 |
+
from synthelion.privacyguardml import PrivacyGuardMLDetector, resolve_model_path
|
| 87 |
+
|
| 88 |
+
detector = PrivacyGuardMLDetector(resolve_model_path(), min_confidence=0.6)
|
| 89 |
+
for span in detector.detect("Contact me at mario.rossi@example.com or IT60X0542811101000000123456"):
|
| 90 |
+
print(span) # MLSpan(start, end, value, label, confidence)
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
## Training
|
| 94 |
+
|
| 95 |
+
- **Data**: synthetic PII values (from Synthelion's checksum validators and `privacy_rules.yaml` patterns) inserted into real multilingual carrier sentences (Wikipedia corpora)
|
| 96 |
+
- **Labels**: self-distillation — ground truth is exact by construction (the inserted value's span is known at insertion time)
|
| 97 |
+
- **Sampling**: checksum-backed categories weighted 4x over context-only categories, to keep primary-category confidence high while still covering the full catalog
|
| 98 |
+
- **Languages**: 39 (see `config.json` -> `trained_languages`)
|
| 99 |
+
- **Examples**: 106,123 training / 5,585 eval
|
| 100 |
+
- **Runtime**: fully offline, CPU-only
|
| 101 |
+
|
| 102 |
+
## Files
|
| 103 |
+
|
| 104 |
+
| File | Description |
|
| 105 |
+
|------|-------------|
|
| 106 |
+
| `config.json` | Model hyperparameters, tag vocabulary, training metadata |
|
| 107 |
+
| `vocab.json` | Word vocabulary (40k words) |
|
| 108 |
+
| `model.bin` | PyTorch weights |
|
| 109 |
+
| `MODEL_INFO.txt` | Training provenance |
|
| 110 |
+
|
| 111 |
+
## License
|
| 112 |
+
|
| 113 |
+
MIT License. See [LICENSE](LICENSE) for details.
|
config.json
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_name": "PrivacyGuardML",
|
| 3 |
+
"model_type": "privacyguardml",
|
| 4 |
+
"d_model": 128,
|
| 5 |
+
"n_heads": 4,
|
| 6 |
+
"n_layers": 2,
|
| 7 |
+
"ffn_dim": 512,
|
| 8 |
+
"dropout": 0.1,
|
| 9 |
+
"n_features": 21,
|
| 10 |
+
"n_tags": 33,
|
| 11 |
+
"tags": [
|
| 12 |
+
"O",
|
| 13 |
+
"B-PHONE",
|
| 14 |
+
"I-PHONE",
|
| 15 |
+
"B-EMAIL",
|
| 16 |
+
"I-EMAIL",
|
| 17 |
+
"B-CREDITCARD",
|
| 18 |
+
"I-CREDITCARD",
|
| 19 |
+
"B-IBAN",
|
| 20 |
+
"I-IBAN",
|
| 21 |
+
"B-NATIONALID",
|
| 22 |
+
"I-NATIONALID",
|
| 23 |
+
"B-TAXID",
|
| 24 |
+
"I-TAXID",
|
| 25 |
+
"B-SSN",
|
| 26 |
+
"I-SSN",
|
| 27 |
+
"B-GPS",
|
| 28 |
+
"I-GPS",
|
| 29 |
+
"B-VEHICLEPLATE",
|
| 30 |
+
"I-VEHICLEPLATE",
|
| 31 |
+
"B-BADGEID",
|
| 32 |
+
"I-BADGEID",
|
| 33 |
+
"B-BUSINESSID",
|
| 34 |
+
"I-BUSINESSID",
|
| 35 |
+
"B-SECRET",
|
| 36 |
+
"I-SECRET",
|
| 37 |
+
"B-SOCIALHANDLE",
|
| 38 |
+
"I-SOCIALHANDLE",
|
| 39 |
+
"B-LEGALCASE",
|
| 40 |
+
"I-LEGALCASE",
|
| 41 |
+
"B-PNRCODE",
|
| 42 |
+
"I-PNRCODE",
|
| 43 |
+
"B-MINORDATA",
|
| 44 |
+
"I-MINORDATA"
|
| 45 |
+
],
|
| 46 |
+
"max_seq_len": 96,
|
| 47 |
+
"vocab_size": 40000,
|
| 48 |
+
"min_confidence": 0.6,
|
| 49 |
+
"trained_languages": [
|
| 50 |
+
"en",
|
| 51 |
+
"it",
|
| 52 |
+
"de",
|
| 53 |
+
"fr",
|
| 54 |
+
"es",
|
| 55 |
+
"ru",
|
| 56 |
+
"uk",
|
| 57 |
+
"hi",
|
| 58 |
+
"zh",
|
| 59 |
+
"ja",
|
| 60 |
+
"af",
|
| 61 |
+
"be",
|
| 62 |
+
"bg",
|
| 63 |
+
"ca",
|
| 64 |
+
"cs",
|
| 65 |
+
"da",
|
| 66 |
+
"el",
|
| 67 |
+
"et",
|
| 68 |
+
"eu",
|
| 69 |
+
"fi",
|
| 70 |
+
"ga",
|
| 71 |
+
"gl",
|
| 72 |
+
"hr",
|
| 73 |
+
"hu",
|
| 74 |
+
"is",
|
| 75 |
+
"la",
|
| 76 |
+
"lt",
|
| 77 |
+
"lv",
|
| 78 |
+
"mk",
|
| 79 |
+
"nl",
|
| 80 |
+
"no",
|
| 81 |
+
"pl",
|
| 82 |
+
"pt",
|
| 83 |
+
"ro",
|
| 84 |
+
"sk",
|
| 85 |
+
"sl",
|
| 86 |
+
"sq",
|
| 87 |
+
"sr",
|
| 88 |
+
"sv"
|
| 89 |
+
],
|
| 90 |
+
"training_examples": 106123,
|
| 91 |
+
"eval_examples": 5585
|
| 92 |
+
}
|
model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2d1b32267689d592cc330b626d451ae24c3a669e1791605dde2d0740967d6cbe
|
| 3 |
+
size 26346881
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|