v4: +v7 proxy in-distribution (gate F1 .60->.72), 32 types, #1 on all 4 clean held

README.md

@@ -54,7 +54,7 @@ _clean · synthetic · tests PII-vs-NER discrimination_
 
 | Model | P | R | F1 | F2 |
 |---|---|---|---|---|
-| trained:pii-detect-v3a | 0.304 | 0.742 | 0.431 | 0.576 |
+| trained:pii-detect-v4 | 0.284 | 0.706 | 0.405 | 0.544 |
 | regex baseline | 0.259 | 0.579 | 0.358 | 0.464 |
 | VAGOsolutions/SauerkrautLM-GLiNER | 0.171 | 0.785 | 0.280 | 0.456 |
 | openai/privacy-filter | 0.169 | 0.595 | 0.264 | 0.396 |
@@ -67,7 +67,7 @@ _clean · synthetic English_
 
 | Model | P | R | F1 | F2 |
 |---|---|---|---|---|
-| trained:pii-detect-v3a | 0.815 | 0.850 | 0.832 | 0.843 |
+| trained:pii-detect-v4 | 0.770 | 0.852 | 0.809 | 0.834 |
 | LiquidAI demo (LFM2.5-PII-Demo) | 0.887 | 0.735 | 0.804 | 0.761 |
 | OpenMed/privacy-filter-nemotron | 0.762 | 0.773 | 0.767 | 0.770 |
 | VAGOsolutions/SauerkrautLM-GLiNER | 0.605 | 0.734 | 0.663 | 0.704 |
@@ -80,7 +80,7 @@ _clean · human gold · English legal (ECHR)_
 
 | Model | P | R | F1 | F2 |
 |---|---|---|---|---|
-| trained:pii-detect-v3a | 0.897 | 0.746 | 0.815 | 0.772 |
+| trained:pii-detect-v4 | 0.860 | 0.782 | 0.819 | 0.797 |
 | VAGOsolutions/SauerkrautLM-GLiNER | 0.843 | 0.576 | 0.685 | 0.615 |
 | OpenMed/privacy-filter-nemotron | 0.728 | 0.625 | 0.673 | 0.643 |
 | openai/privacy-filter | 0.933 | 0.383 | 0.543 | 0.434 |
@@ -93,7 +93,7 @@ _clean · human gold · multilingual EU legal_
 
 | Model | P | R | F1 | F2 |
 |---|---|---|---|---|
-| trained:pii-detect-v3a | 0.499 | 0.473 | 0.486 | 0.478 |
+| trained:pii-detect-v4 | 0.445 | 0.541 | 0.488 | 0.518 |
 | VAGOsolutions/SauerkrautLM-GLiNER | 0.487 | 0.258 | 0.337 | 0.285 |
 | iiiorg/piiranha-v1 | 0.465 | 0.181 | 0.261 | 0.206 |
 | openai/privacy-filter | 0.372 | 0.186 | 0.248 | 0.207 |
@@ -112,7 +112,7 @@ _⚠ in-distribution for OpenMed-nemotron & nvidia GLiNER (read as ceilings)_
 | LiquidAI demo (LFM2.5-PII-Demo) | 0.894 | 0.518 | 0.656 | 0.566 |
 | iiiorg/piiranha-v1 | 0.841 | 0.538 | 0.656 | 0.580 |
 | VAGOsolutions/SauerkrautLM-GLiNER | 0.707 | 0.583 | 0.639 | 0.604 |
-| trained:pii-detect-v3a | 0.807 | 0.516 | 0.629 | 0.556 |
+| trained:pii-detect-v4 | 0.712 | 0.538 | 0.613 | 0.566 |
 | openai/privacy-filter | 0.699 | 0.484 | 0.572 | 0.516 |
 | regex baseline | 0.645 | 0.227 | 0.335 | 0.260 |
 
@@ -125,8 +125,8 @@ _⚠ in OUR backbone's pretraining — contaminated for us; reference only_
 | LiquidAI demo (LFM2.5-PII-Demo) | 0.533 | 0.464 | 0.496 | 0.476 |
 | VAGOsolutions/SauerkrautLM-GLiNER | 0.403 | 0.610 | 0.485 | 0.553 |
 | OpenMed/privacy-filter-nemotron | 0.341 | 0.541 | 0.418 | 0.484 |
+| trained:pii-detect-v4 | 0.330 | 0.488 | 0.394 | 0.445 |
 | openai/privacy-filter | 0.316 | 0.492 | 0.385 | 0.443 |
-| trained:pii-detect-v3a | 0.339 | 0.442 | 0.384 | 0.416 |
 | regex baseline | 0.269 | 0.147 | 0.190 | 0.162 |
 
 
@@ -134,3 +134,17 @@ _⚠ in OUR backbone's pretraining — contaminated for us; reference only_
 - **Strength:** best-in-class on **SPY** (PII-vs-NER discrimination) with the highest recall of any model tested.
 - **Gap:** legal-domain benchmarks (TAB/MAPA) — high precision but low recall; targeted for the next data iteration.
 - Benchmarks use differing entity taxonomies; our fine-grained types are mapped to each via an identity/crosswalk, so type-level F1 understates raw detection (binary detection is higher).
+
+## ShieldFlow proxy-traffic product gate (native ShieldFlow taxonomy, partial)
+
+Held-out 10% of the v7 proxy corpus (enterprise pastes); trained on the same 90% the
+incumbent did, evaluated on the identical untouched holdout.
+
+| dataset | ours (v4) P / R / F1 | q4 incumbent |
+|---|---|---|
+| proxy_traffic_holdout | 0.749 / 0.683 / **0.714** | F1 0.878 / R 0.982 |
+| proxy_traffic_holdout_xlong | 0.688 / 0.753 / **0.719** | F1 0.884 / R 0.940 |
+
+v4 adds the v7 train split + 3 native ShieldFlow types (`developer.login_credentials`,
+`developer.device_id`, `healthcare.health_plan_id`). Recall gap to the in-distribution
+incumbent remains; tuning the v7 mix weight is the lever.

config.json

@@ -56,94 +56,106 @@
     "26": "I-credential.password",
     "27": "E-credential.password",
     "28": "S-credential.password",
-    "29": "B-device.mac_address",
-    "30": "I-device.mac_address",
-    "31": "E-device.mac_address",
-    "32": "S-device.mac_address",
-    "33": "B-financial.bank_account",
-    "34": "I-financial.bank_account",
-    "35": "E-financial.bank_account",
-    "36": "S-financial.bank_account",
-    "37": "B-financial.credit_card",
-    "38": "I-financial.credit_card",
-    "39": "E-financial.credit_card",
-    "40": "S-financial.credit_card",
-    "41": "B-financial.iban",
-    "42": "I-financial.iban",
-    "43": "E-financial.iban",
-    "44": "S-financial.iban",
-    "45": "B-financial.swift_bic",
-    "46": "I-financial.swift_bic",
-    "47": "E-financial.swift_bic",
-    "48": "S-financial.swift_bic",
-    "49": "B-healthcare.condition",
-    "50": "I-healthcare.condition",
-    "51": "E-healthcare.condition",
-    "52": "S-healthcare.condition",
-    "53": "B-healthcare.medical_record",
-    "54": "I-healthcare.medical_record",
-    "55": "E-healthcare.medical_record",
-    "56": "S-healthcare.medical_record",
-    "57": "B-healthcare.medication",
-    "58": "I-healthcare.medication",
-    "59": "E-healthcare.medication",
-    "60": "S-healthcare.medication",
-    "61": "B-identity.date_of_birth",
-    "62": "I-identity.date_of_birth",
-    "63": "E-identity.date_of_birth",
-    "64": "S-identity.date_of_birth",
-    "65": "B-identity.drivers_license",
-    "66": "I-identity.drivers_license",
-    "67": "E-identity.drivers_license",
-    "68": "S-identity.drivers_license",
-    "69": "B-identity.national_id",
-    "70": "I-identity.national_id",
-    "71": "E-identity.national_id",
-    "72": "S-identity.national_id",
-    "73": "B-identity.passport",
-    "74": "I-identity.passport",
-    "75": "E-identity.passport",
-    "76": "S-identity.passport",
-    "77": "B-identity.person_name",
-    "78": "I-identity.person_name",
-    "79": "E-identity.person_name",
-    "80": "S-identity.person_name",
-    "81": "B-identity.ssn",
-    "82": "I-identity.ssn",
-    "83": "E-identity.ssn",
-    "84": "S-identity.ssn",
-    "85": "B-identity.tax_id",
-    "86": "I-identity.tax_id",
-    "87": "E-identity.tax_id",
-    "88": "S-identity.tax_id",
-    "89": "B-legal.case_number",
-    "90": "I-legal.case_number",
-    "91": "E-legal.case_number",
-    "92": "S-legal.case_number",
-    "93": "B-location.gps_coordinates",
-    "94": "I-location.gps_coordinates",
-    "95": "E-location.gps_coordinates",
-    "96": "S-location.gps_coordinates",
-    "97": "B-online.url",
-    "98": "I-online.url",
-    "99": "E-online.url",
-    "100": "S-online.url",
-    "101": "B-online.username",
-    "102": "I-online.username",
-    "103": "E-online.username",
-    "104": "S-online.username",
-    "105": "B-org.company_name",
-    "106": "I-org.company_name",
-    "107": "E-org.company_name",
-    "108": "S-org.company_name",
-    "109": "B-special.political",
-    "110": "I-special.political",
-    "111": "E-special.political",
-    "112": "S-special.political",
-    "113": "B-special.religion",
-    "114": "I-special.religion",
-    "115": "E-special.religion",
-    "116": "S-special.religion"
+    "29": "B-developer.device_id",
+    "30": "I-developer.device_id",
+    "31": "E-developer.device_id",
+    "32": "S-developer.device_id",
+    "33": "B-developer.login_credentials",
+    "34": "I-developer.login_credentials",
+    "35": "E-developer.login_credentials",
+    "36": "S-developer.login_credentials",
+    "37": "B-device.mac_address",
+    "38": "I-device.mac_address",
+    "39": "E-device.mac_address",
+    "40": "S-device.mac_address",
+    "41": "B-financial.bank_account",
+    "42": "I-financial.bank_account",
+    "43": "E-financial.bank_account",
+    "44": "S-financial.bank_account",
+    "45": "B-financial.credit_card",
+    "46": "I-financial.credit_card",
+    "47": "E-financial.credit_card",
+    "48": "S-financial.credit_card",
+    "49": "B-financial.iban",
+    "50": "I-financial.iban",
+    "51": "E-financial.iban",
+    "52": "S-financial.iban",
+    "53": "B-financial.swift_bic",
+    "54": "I-financial.swift_bic",
+    "55": "E-financial.swift_bic",
+    "56": "S-financial.swift_bic",
+    "57": "B-healthcare.condition",
+    "58": "I-healthcare.condition",
+    "59": "E-healthcare.condition",
+    "60": "S-healthcare.condition",
+    "61": "B-healthcare.health_plan_id",
+    "62": "I-healthcare.health_plan_id",
+    "63": "E-healthcare.health_plan_id",
+    "64": "S-healthcare.health_plan_id",
+    "65": "B-healthcare.medical_record",
+    "66": "I-healthcare.medical_record",
+    "67": "E-healthcare.medical_record",
+    "68": "S-healthcare.medical_record",
+    "69": "B-healthcare.medication",
+    "70": "I-healthcare.medication",
+    "71": "E-healthcare.medication",
+    "72": "S-healthcare.medication",
+    "73": "B-identity.date_of_birth",
+    "74": "I-identity.date_of_birth",
+    "75": "E-identity.date_of_birth",
+    "76": "S-identity.date_of_birth",
+    "77": "B-identity.drivers_license",
+    "78": "I-identity.drivers_license",
+    "79": "E-identity.drivers_license",
+    "80": "S-identity.drivers_license",
+    "81": "B-identity.national_id",
+    "82": "I-identity.national_id",
+    "83": "E-identity.national_id",
+    "84": "S-identity.national_id",
+    "85": "B-identity.passport",
+    "86": "I-identity.passport",
+    "87": "E-identity.passport",
+    "88": "S-identity.passport",
+    "89": "B-identity.person_name",
+    "90": "I-identity.person_name",
+    "91": "E-identity.person_name",
+    "92": "S-identity.person_name",
+    "93": "B-identity.ssn",
+    "94": "I-identity.ssn",
+    "95": "E-identity.ssn",
+    "96": "S-identity.ssn",
+    "97": "B-identity.tax_id",
+    "98": "I-identity.tax_id",
+    "99": "E-identity.tax_id",
+    "100": "S-identity.tax_id",
+    "101": "B-legal.case_number",
+    "102": "I-legal.case_number",
+    "103": "E-legal.case_number",
+    "104": "S-legal.case_number",
+    "105": "B-location.gps_coordinates",
+    "106": "I-location.gps_coordinates",
+    "107": "E-location.gps_coordinates",
+    "108": "S-location.gps_coordinates",
+    "109": "B-online.url",
+    "110": "I-online.url",
+    "111": "E-online.url",
+    "112": "S-online.url",
+    "113": "B-online.username",
+    "114": "I-online.username",
+    "115": "E-online.username",
+    "116": "S-online.username",
+    "117": "B-org.company_name",
+    "118": "I-org.company_name",
+    "119": "E-org.company_name",
+    "120": "S-org.company_name",
+    "121": "B-special.political",
+    "122": "I-special.political",
+    "123": "E-special.political",
+    "124": "S-special.political",
+    "125": "B-special.religion",
+    "126": "I-special.religion",
+    "127": "E-special.religion",
+    "128": "S-special.religion"
   },
   "initializer_range": 0.02,
   "intermediate_size": 6656,
@@ -155,28 +167,31 @@
     "B-contact.postal_code": 17,
     "B-credential.api_key": 21,
     "B-credential.password": 25,
-    "B-device.mac_address": 29,
-    "B-financial.bank_account": 33,
-    "B-financial.credit_card": 37,
-    "B-financial.iban": 41,
-    "B-financial.swift_bic": 45,
-    "B-healthcare.condition": 49,
-    "B-healthcare.medical_record": 53,
-    "B-healthcare.medication": 57,
-    "B-identity.date_of_birth": 61,
-    "B-identity.drivers_license": 65,
-    "B-identity.national_id": 69,
-    "B-identity.passport": 73,
-    "B-identity.person_name": 77,
-    "B-identity.ssn": 81,
-    "B-identity.tax_id": 85,
-    "B-legal.case_number": 89,
-    "B-location.gps_coordinates": 93,
-    "B-online.url": 97,
-    "B-online.username": 101,
-    "B-org.company_name": 105,
-    "B-special.political": 109,
-    "B-special.religion": 113,
+    "B-developer.device_id": 29,
+    "B-developer.login_credentials": 33,
+    "B-device.mac_address": 37,
+    "B-financial.bank_account": 41,
+    "B-financial.credit_card": 45,
+    "B-financial.iban": 49,
+    "B-financial.swift_bic": 53,
+    "B-healthcare.condition": 57,
+    "B-healthcare.health_plan_id": 61,
+    "B-healthcare.medical_record": 65,
+    "B-healthcare.medication": 69,
+    "B-identity.date_of_birth": 73,
+    "B-identity.drivers_license": 77,
+    "B-identity.national_id": 81,
+    "B-identity.passport": 85,
+    "B-identity.person_name": 89,
+    "B-identity.ssn": 93,
+    "B-identity.tax_id": 97,
+    "B-legal.case_number": 101,
+    "B-location.gps_coordinates": 105,
+    "B-online.url": 109,
+    "B-online.username": 113,
+    "B-org.company_name": 117,
+    "B-special.political": 121,
+    "B-special.religion": 125,
     "E-contact.address": 3,
     "E-contact.email": 7,
     "E-contact.ip_address": 11,
@@ -184,28 +199,31 @@
     "E-contact.postal_code": 19,
     "E-credential.api_key": 23,
     "E-credential.password": 27,
-    "E-device.mac_address": 31,
-    "E-financial.bank_account": 35,
-    "E-financial.credit_card": 39,
-    "E-financial.iban": 43,
-    "E-financial.swift_bic": 47,
-    "E-healthcare.condition": 51,
-    "E-healthcare.medical_record": 55,
-    "E-healthcare.medication": 59,
-    "E-identity.date_of_birth": 63,
-    "E-identity.drivers_license": 67,
-    "E-identity.national_id": 71,
-    "E-identity.passport": 75,
-    "E-identity.person_name": 79,
-    "E-identity.ssn": 83,
-    "E-identity.tax_id": 87,
-    "E-legal.case_number": 91,
-    "E-location.gps_coordinates": 95,
-    "E-online.url": 99,
-    "E-online.username": 103,
-    "E-org.company_name": 107,
-    "E-special.political": 111,
-    "E-special.religion": 115,
+    "E-developer.device_id": 31,
+    "E-developer.login_credentials": 35,
+    "E-device.mac_address": 39,
+    "E-financial.bank_account": 43,
+    "E-financial.credit_card": 47,
+    "E-financial.iban": 51,
+    "E-financial.swift_bic": 55,
+    "E-healthcare.condition": 59,
+    "E-healthcare.health_plan_id": 63,
+    "E-healthcare.medical_record": 67,
+    "E-healthcare.medication": 71,
+    "E-identity.date_of_birth": 75,
+    "E-identity.drivers_license": 79,
+    "E-identity.national_id": 83,
+    "E-identity.passport": 87,
+    "E-identity.person_name": 91,
+    "E-identity.ssn": 95,
+    "E-identity.tax_id": 99,
+    "E-legal.case_number": 103,
+    "E-location.gps_coordinates": 107,
+    "E-online.url": 111,
+    "E-online.username": 115,
+    "E-org.company_name": 119,
+    "E-special.political": 123,
+    "E-special.religion": 127,
     "I-contact.address": 2,
     "I-contact.email": 6,
     "I-contact.ip_address": 10,
@@ -213,28 +231,31 @@
     "I-contact.postal_code": 18,
     "I-credential.api_key": 22,
     "I-credential.password": 26,
-    "I-device.mac_address": 30,
-    "I-financial.bank_account": 34,
-    "I-financial.credit_card": 38,
-    "I-financial.iban": 42,
-    "I-financial.swift_bic": 46,
-    "I-healthcare.condition": 50,
-    "I-healthcare.medical_record": 54,
-    "I-healthcare.medication": 58,
-    "I-identity.date_of_birth": 62,
-    "I-identity.drivers_license": 66,
-    "I-identity.national_id": 70,
-    "I-identity.passport": 74,
-    "I-identity.person_name": 78,
-    "I-identity.ssn": 82,
-    "I-identity.tax_id": 86,
-    "I-legal.case_number": 90,
-    "I-location.gps_coordinates": 94,
-    "I-online.url": 98,
-    "I-online.username": 102,
-    "I-org.company_name": 106,
-    "I-special.political": 110,
-    "I-special.religion": 114,
+    "I-developer.device_id": 30,
+    "I-developer.login_credentials": 34,
+    "I-device.mac_address": 38,
+    "I-financial.bank_account": 42,
+    "I-financial.credit_card": 46,
+    "I-financial.iban": 50,
+    "I-financial.swift_bic": 54,
+    "I-healthcare.condition": 58,
+    "I-healthcare.health_plan_id": 62,
+    "I-healthcare.medical_record": 66,
+    "I-healthcare.medication": 70,
+    "I-identity.date_of_birth": 74,
+    "I-identity.drivers_license": 78,
+    "I-identity.national_id": 82,
+    "I-identity.passport": 86,
+    "I-identity.person_name": 90,
+    "I-identity.ssn": 94,
+    "I-identity.tax_id": 98,
+    "I-legal.case_number": 102,
+    "I-location.gps_coordinates": 106,
+    "I-online.url": 110,
+    "I-online.username": 114,
+    "I-org.company_name": 118,
+    "I-special.political": 122,
+    "I-special.religion": 126,
     "O": 0,
     "S-contact.address": 4,
     "S-contact.email": 8,
@@ -243,28 +264,31 @@
     "S-contact.postal_code": 20,
     "S-credential.api_key": 24,
     "S-credential.password": 28,
-    "S-device.mac_address": 32,
-    "S-financial.bank_account": 36,
-    "S-financial.credit_card": 40,
-    "S-financial.iban": 44,
-    "S-financial.swift_bic": 48,
-    "S-healthcare.condition": 52,
-    "S-healthcare.medical_record": 56,
-    "S-healthcare.medication": 60,
-    "S-identity.date_of_birth": 64,
-    "S-identity.drivers_license": 68,
-    "S-identity.national_id": 72,
-    "S-identity.passport": 76,
-    "S-identity.person_name": 80,
-    "S-identity.ssn": 84,
-    "S-identity.tax_id": 88,
-    "S-legal.case_number": 92,
-    "S-location.gps_coordinates": 96,
-    "S-online.url": 100,
-    "S-online.username": 104,
-    "S-org.company_name": 108,
-    "S-special.political": 112,
-    "S-special.religion": 116
+    "S-developer.device_id": 32,
+    "S-developer.login_credentials": 36,
+    "S-device.mac_address": 40,
+    "S-financial.bank_account": 44,
+    "S-financial.credit_card": 48,
+    "S-financial.iban": 52,
+    "S-financial.swift_bic": 56,
+    "S-healthcare.condition": 60,
+    "S-healthcare.health_plan_id": 64,
+    "S-healthcare.medical_record": 68,
+    "S-healthcare.medication": 72,
+    "S-identity.date_of_birth": 76,
+    "S-identity.drivers_license": 80,
+    "S-identity.national_id": 84,
+    "S-identity.passport": 88,
+    "S-identity.person_name": 92,
+    "S-identity.ssn": 96,
+    "S-identity.tax_id": 100,
+    "S-legal.case_number": 104,
+    "S-location.gps_coordinates": 108,
+    "S-online.url": 112,
+    "S-online.username": 116,
+    "S-org.company_name": 120,
+    "S-special.political": 124,
+    "S-special.religion": 128
   },
   "label_smoothing": 0.1,
   "layer_types": [

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c863eb56a6c0ddcf1ed1fc83d14ac07a4f182e0f1aeba6fd583c65978ee766d4
-size 1250657676
+oid sha256:a8e525b1856af936e8feb1b35fa1fb25e5f39bfaf78b7319f0f39fe8bfb15e58
+size 1250706876