Francesco Paolo Passaro commited on
Commit
3f922e0
·
verified ·
1 Parent(s): 3400562

Initial PrivacyGuardML checkpoint (16 categories, 39 languages)

Browse files
Files changed (7) hide show
  1. .gitattributes +35 -35
  2. LICENSE +21 -0
  3. MODEL_INFO.txt +12 -0
  4. README.md +110 -0
  5. config.json +92 -0
  6. model.bin +3 -0
  7. vocab.json +0 -0
.gitattributes CHANGED
@@ -1,35 +1,35 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Passaro Francesco Paolo
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
MODEL_INFO.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ PrivacyGuardML model file
2
+ ==========================
3
+ Generated by Synthelion's own training pipeline:
4
+ devtools/train_privacyguardml.py
5
+
6
+ Training data: synthetic PII values (generated from Synthelion's own
7
+ checksum validators in synthelion/privacy_validators.py and rule
8
+ patterns in synthelion/privacy_rules.yaml) inserted into real
9
+ multilingual carrier sentences (39 languages, Wikipedia corpora).
10
+ Architecture: own transformer encoder, BIO tagging head.
11
+ Runtime: fully offline, CPU-only. This is the only privacy.use_ml
12
+ backend PrivacyGuard supports — no third-party model.
README.md CHANGED
@@ -1,3 +1,113 @@
1
  ---
2
  license: mit
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: mit
3
+ library_name: privacyguardml
4
+ tags:
5
+ - token-classification
6
+ - pii-detection
7
+ - ner
8
+ - transformer encoder
9
+ - multilingual
10
+ language:
11
+ - af
12
+ - be
13
+ - bg
14
+ - ca
15
+ - cs
16
+ - da
17
+ - de
18
+ - el
19
+ - en
20
+ - es
21
+ - et
22
+ - eu
23
+ - fi
24
+ - fr
25
+ - ga
26
+ - gl
27
+ - hi
28
+ - hr
29
+ - hu
30
+ - is
31
+ - it
32
+ - ja
33
+ - la
34
+ - lt
35
+ - lv
36
+ - mk
37
+ - nl
38
+ - no
39
+ - pl
40
+ - pt
41
+ - ro
42
+ - ru
43
+ - sk
44
+ - sl
45
+ - sq
46
+ - sr
47
+ - sv
48
+ - uk
49
+ - zh
50
  ---
51
+
52
+ # PrivacyGuardML
53
+
54
+ PII confirmation model for [Synthelion](https://github.com/francescopaolopassaro/Synthelion)'s PrivacyGuard — the opt-in ML-assisted confirmation tier (`privacy.use_ml`).
55
+
56
+ A small, fully offline transformer encoder trained with self-distillation: synthetic PII values are generated from Synthelion's own checksum validators and rule patterns, inserted into real multilingual sentences, and the encoder learns to find them back by shape and context via per-token BIO tagging. It is the sole ML backend PrivacyGuard supports — no third-party model.
57
+
58
+ **What it's for, precisely.** PrivacyGuard's core tier is regex + algorithmic checksum (or context keyword, for the rules with no checksum), zero-ML, and stays the default. This model is only the confirmation signal: a value that matches a rule's shape but has no context keyword nearby can be confirmed by an overlapping prediction from this model. A failed checksum still vetoes detection regardless of what the model says — it can only recover recall the strict context gate intentionally trades away, never introduce a false positive the validator would reject.
59
+
60
+ ## Architecture
61
+
62
+ ```
63
+ Char-ngram hashing -> word embedding + shape-feature embedding -> positional encoding
64
+ -> 2-layer TransformerEncoder (d=128, h=4) -> Linear(33) BIO-tag logits
65
+ ```
66
+
67
+ | Parameter | Value |
68
+ |-----------|-------|
69
+ | d_model | 128 |
70
+ | n_heads | 4 |
71
+ | n_layers | 2 |
72
+ | ffn_dim | 512 |
73
+ | Vocabulary | 40,000 words + 8,192 char-ngram buckets |
74
+ | Max seq len | 96 |
75
+ | Input features | 21 (digit/at/plus/separator/uppercase flags + length buckets) |
76
+ | Output | 33-class BIO tag logits (16 categories x B/I + O) |
77
+
78
+ ## Categories
79
+
80
+ Checksum-backed: `PHONE`, `EMAIL`, `CREDITCARD`, `IBAN`, `NATIONALID`, `TAXID`, `SSN`, `GPS`.
81
+ Context-only (no checksum, PrivacyGuard's remaining rule categories): `VEHICLEPLATE`, `BADGEID`, `BUSINESSID`, `SECRET`, `SOCIALHANDLE`, `LEGALCASE`, `PNRCODE`, `MINORDATA`.
82
+
83
+ ## Quick start
84
+
85
+ ```python
86
+ from synthelion.privacyguardml import PrivacyGuardMLDetector, resolve_model_path
87
+
88
+ detector = PrivacyGuardMLDetector(resolve_model_path(), min_confidence=0.6)
89
+ for span in detector.detect("Contact me at mario.rossi@example.com or IT60X0542811101000000123456"):
90
+ print(span) # MLSpan(start, end, value, label, confidence)
91
+ ```
92
+
93
+ ## Training
94
+
95
+ - **Data**: synthetic PII values (from Synthelion's checksum validators and `privacy_rules.yaml` patterns) inserted into real multilingual carrier sentences (Wikipedia corpora)
96
+ - **Labels**: self-distillation — ground truth is exact by construction (the inserted value's span is known at insertion time)
97
+ - **Sampling**: checksum-backed categories weighted 4x over context-only categories, to keep primary-category confidence high while still covering the full catalog
98
+ - **Languages**: 39 (see `config.json` -> `trained_languages`)
99
+ - **Examples**: 106,123 training / 5,585 eval
100
+ - **Runtime**: fully offline, CPU-only
101
+
102
+ ## Files
103
+
104
+ | File | Description |
105
+ |------|-------------|
106
+ | `config.json` | Model hyperparameters, tag vocabulary, training metadata |
107
+ | `vocab.json` | Word vocabulary (40k words) |
108
+ | `model.bin` | PyTorch weights |
109
+ | `MODEL_INFO.txt` | Training provenance |
110
+
111
+ ## License
112
+
113
+ MIT License. See [LICENSE](LICENSE) for details.
config.json ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "PrivacyGuardML",
3
+ "model_type": "privacyguardml",
4
+ "d_model": 128,
5
+ "n_heads": 4,
6
+ "n_layers": 2,
7
+ "ffn_dim": 512,
8
+ "dropout": 0.1,
9
+ "n_features": 21,
10
+ "n_tags": 33,
11
+ "tags": [
12
+ "O",
13
+ "B-PHONE",
14
+ "I-PHONE",
15
+ "B-EMAIL",
16
+ "I-EMAIL",
17
+ "B-CREDITCARD",
18
+ "I-CREDITCARD",
19
+ "B-IBAN",
20
+ "I-IBAN",
21
+ "B-NATIONALID",
22
+ "I-NATIONALID",
23
+ "B-TAXID",
24
+ "I-TAXID",
25
+ "B-SSN",
26
+ "I-SSN",
27
+ "B-GPS",
28
+ "I-GPS",
29
+ "B-VEHICLEPLATE",
30
+ "I-VEHICLEPLATE",
31
+ "B-BADGEID",
32
+ "I-BADGEID",
33
+ "B-BUSINESSID",
34
+ "I-BUSINESSID",
35
+ "B-SECRET",
36
+ "I-SECRET",
37
+ "B-SOCIALHANDLE",
38
+ "I-SOCIALHANDLE",
39
+ "B-LEGALCASE",
40
+ "I-LEGALCASE",
41
+ "B-PNRCODE",
42
+ "I-PNRCODE",
43
+ "B-MINORDATA",
44
+ "I-MINORDATA"
45
+ ],
46
+ "max_seq_len": 96,
47
+ "vocab_size": 40000,
48
+ "min_confidence": 0.6,
49
+ "trained_languages": [
50
+ "en",
51
+ "it",
52
+ "de",
53
+ "fr",
54
+ "es",
55
+ "ru",
56
+ "uk",
57
+ "hi",
58
+ "zh",
59
+ "ja",
60
+ "af",
61
+ "be",
62
+ "bg",
63
+ "ca",
64
+ "cs",
65
+ "da",
66
+ "el",
67
+ "et",
68
+ "eu",
69
+ "fi",
70
+ "ga",
71
+ "gl",
72
+ "hr",
73
+ "hu",
74
+ "is",
75
+ "la",
76
+ "lt",
77
+ "lv",
78
+ "mk",
79
+ "nl",
80
+ "no",
81
+ "pl",
82
+ "pt",
83
+ "ro",
84
+ "sk",
85
+ "sl",
86
+ "sq",
87
+ "sr",
88
+ "sv"
89
+ ],
90
+ "training_examples": 106123,
91
+ "eval_examples": 5585
92
+ }
model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d1b32267689d592cc330b626d451ae24c3a669e1791605dde2d0740967d6cbe
3
+ size 26346881
vocab.json ADDED
The diff for this file is too large to render. See raw diff