Update README.md

3f54a28 verified 16 days ago

6.2 kB

	---
	license: apache-2.0
	datasets:
	- ai4privacy/pii-masking-openpii-1.5m
	language:
	- nl
	base_model:
	- DTAI-KULeuven/robbert-2023-dutch-large
	pipeline_tag: token-classification
	library_name: transformers
	tags:
	- pii
	- deidentification
	---


	Finetuning was done using [MedNER](https://github.com/UPOD-datascience/MedNER.nl).

	We limited the training to Dutch.

	The model was trained in a multilabel-sense, using a binary cross-entropy loss per label, which followed the standard IOB-schema
	(that is Outside the span, Beginning of the span, Inside the span)
	We replaced the standard 768-weight linear layer by 3x768 dense layers with 10\% dropout and ReLu activations.

	The weights are a result of chained SLERP over five cross-validated folds.

	Token classification scores (answering the question: *given the span, to which class does it belong?*):

	```json
	{
	"eval_AGE": {
	"f1": 0.958,
	"precision": 0.957,
	"recall": 0.959
	},
	"eval_BUILDINGNUM": {
	"f1": 0.968,
	"precision": 0.968,
	"recall": 0.969
	},
	"eval_CITY": {
	"f1": 0.974,
	"precision": 0.968,
	"recall": 0.980
	},
	"eval_CREDITCARDNUMBER": {
	"f1": 0.954,
	"precision": 0.951,
	"recall": 0.957
	},
	"eval_DATE": {
	"f1": 0.998,
	"precision": 0.996,
	"recall": 0.999
	},
	"eval_DRIVERLICENSENUM": {
	"f1": 0.961,
	"precision": 0.957,
	"recall": 0.966
	},
	"eval_EMAIL": {
	"f1": 0.970,
	"precision": 0.958,
	"recall": 0.984
	},
	"eval_GENDER": {
	"f1": 0.936,
	"precision": 0.928,
	"recall": 0.945
	},
	"eval_GIVENNAME": {
	"f1": 0.903,
	"precision": 0.898,
	"recall": 0.908
	},
	"eval_IDCARDNUM": {
	"f1": 0.815,
	"precision": 0.792,
	"recall": 0.841
	},
	"eval_PASSPORTNUM": {
	"f1": 0.976,
	"precision": 0.972,
	"recall": 0.981
	},
	"eval_SEX": {
	"f1": 0.950,
	"precision": 0.952,
	"recall": 0.948
	},
	"eval_SOCIALNUM": {
	"f1": 0.640,
	"precision": 0.650,
	"recall": 0.632
	},
	"eval_STREET": {
	"f1": 0.980,
	"precision": 0.979,
	"recall": 0.981
	},
	"eval_SURNAME": {
	"f1": 0.898,
	"precision": 0.894,
	"recall": 0.901
	},
	"eval_TAXNUM": {
	"f1": 0.746,
	"precision": 0.740,
	"recall": 0.751
	},
	"eval_TELEPHONENUM": {
	"f1": 0.984,
	"precision": 0.979,
	"recall": 0.989
	},
	"eval_TITLE": {
	"f1": 0.998,
	"precision": 0.997,
	"recall": 0.999
	},
	"eval_ZIPCODE": {
	"f1": 0.966,
	"precision": 0.968,
	"recall": 0.964
	},
	"eval_overall": {
	"accuracy": 0.985,
	"f1": 0.939,
	"precision": 0.935,
	"recall": 0.943
	}
	}
	```

	End2End scores (answering the question: given a text, what are the spans and* to which class do they belong?*):
	```json
	{
	"strict": {
	"per_category": {
	"AGE": {
	"Precision": 0.85,
	"Recall": 0.82,
	"F1": 0.83
	},
	"BUILDINGNUM": {
	"Precision": 0.91,
	"Recall": 0.84,
	"F1": 0.88
	},
	"CITY": {
	"Precision": 0.91,
	"Recall": 0.9,
	"F1": 0.9
	},
	"CREDITCARDNUMBER": {
	"Precision": 0.97,
	"Recall": 0.96,
	"F1": 0.96
	},
	"DATE": {
	"Precision": 0.89,
	"Recall": 0.87,
	"F1": 0.88
	},
	"DRIVERLICENSENUM": {
	"Precision": 0.81,
	"Recall": 0.75,
	"F1": 0.78
	},
	"EMAIL": {
	"Precision": 0.8,
	"Recall": 0.8,
	"F1": 0.8
	},
	"GENDER": {
	"Precision": 0.91,
	"Recall": 0.89,
	"F1": 0.9
	},
	"GIVENNAME": {
	"Precision": 0.85,
	"Recall": 0.88,
	"F1": 0.87
	},
	"IDCARDNUM": {
	"Precision": 0.78,
	"Recall": 0.78,
	"F1": 0.78
	},
	"PASSPORTNUM": {
	"Precision": 0.82,
	"Recall": 0.77,
	"F1": 0.8
	},
	"SEX": {
	"Precision": 0.93,
	"Recall": 0.88,
	"F1": 0.9
	},
	"SOCIALNUM": {
	"Precision": 0.65,
	"Recall": 0.7,
	"F1": 0.67
	},
	"STREET": {
	"Precision": 0.95,
	"Recall": 0.94,
	"F1": 0.94
	},
	"SURNAME": {
	"Precision": 0.87,
	"Recall": 0.86,
	"F1": 0.86
	},
	"TAXNUM": {
	"Precision": 0.79,
	"Recall": 0.69,
	"F1": 0.74
	},
	"TELEPHONENUM": {
	"Precision": 0.72,
	"Recall": 0.7,
	"F1": 0.71
	},
	"TITLE": {
	"Precision": 0.99,
	"Recall": 0.99,
	"F1": 0.99
	},
	"ZIPCODE": {
	"Precision": 0.86,
	"Recall": 0.94,
	"F1": 0.89
	}
	},
	"micro": {
	"Precision": 0.86,
	"Recall": 0.85,
	"F1": 0.86
	},
	"macro": {
	"Precision": 0.86,
	"Recall": 0.84,
	"F1": 0.85
	}
	},
	"relaxed": {
	"per_category": {
	"AGE": {
	"Precision": 0.94,
	"Recall": 0.91,
	"F1": 0.92
	},
	"BUILDINGNUM": {
	"Precision": 0.93,
	"Recall": 0.86,
	"F1": 0.89
	},
	"CITY": {
	"Precision": 0.96,
	"Recall": 0.96,
	"F1": 0.96
	},
	"CREDITCARDNUMBER": {
	"Precision": 0.98,
	"Recall": 0.97,
	"F1": 0.98
	},
	"DATE": {
	"Precision": 0.99,
	"Recall": 0.96,
	"F1": 0.98
	},
	"DRIVERLICENSENUM": {
	"Precision": 0.98,
	"Recall": 0.91,
	"F1": 0.94
	},
	"EMAIL": {
	"Precision": 0.99,
	"Recall": 0.99,
	"F1": 0.99
	},
	"GENDER": {
	"Precision": 0.94,
	"Recall": 0.92,
	"F1": 0.93
	},
	"GIVENNAME": {
	"Precision": 0.94,
	"Recall": 0.97,
	"F1": 0.95
	},
	"IDCARDNUM": {
	"Precision": 0.78,
	"Recall": 0.78,
	"F1": 0.78
	},
	"PASSPORTNUM": {
	"Precision": 0.98,
	"Recall": 0.93,
	"F1": 0.95
	},
	"SEX": {
	"Precision": 0.95,
	"Recall": 0.89,
	"F1": 0.92
	},
	"SOCIALNUM": {
	"Precision": 0.65,
	"Recall": 0.7,
	"F1": 0.67
	},
	"STREET": {
	"Precision": 0.98,
	"Recall": 0.96,
	"F1": 0.97
	},
	"SURNAME": {
	"Precision": 0.96,
	"Recall": 0.95,
	"F1": 0.96
	},
	"TAXNUM": {
	"Precision": 0.79,
	"Recall": 0.7,
	"F1": 0.74
	},
	"TELEPHONENUM": {
	"Precision": 0.97,
	"Recall": 0.95,
	"F1": 0.96
	},
	"TITLE": {
	"Precision": 0.99,
	"Recall": 0.99,
	"F1": 0.99
	},
	"ZIPCODE": {
	"Precision": 0.88,
	"Recall": 0.96,
	"F1": 0.92
	}
	},
	"micro": {
	"Precision": 0.95,
	"Recall": 0.93,
	"F1": 0.94
	},
	"macro": {
	"Precision": 0.93,
	"Recall": 0.91,
	"F1": 0.92
	}
	}
	}
	```