Delete anonymize_file_clear.py
Browse files- anonymize_file_clear.py +0 -112
anonymize_file_clear.py
DELETED
|
@@ -1,112 +0,0 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import re
|
| 3 |
-
import uuid
|
| 4 |
-
|
| 5 |
-
from transformers import AutoModel, AutoTokenizer
|
| 6 |
-
from concrete.ml.common.serialization.loaders import load
|
| 7 |
-
from utils_demo import *
|
| 8 |
-
|
| 9 |
-
def load_models():
|
| 10 |
-
|
| 11 |
-
# Load the tokenizer and the embedding model
|
| 12 |
-
try:
|
| 13 |
-
tokenizer = AutoTokenizer.from_pretrained("obi/deid_roberta_i2b2")
|
| 14 |
-
embeddings_model = AutoModel.from_pretrained("obi/deid_roberta_i2b2")
|
| 15 |
-
except:
|
| 16 |
-
print("Error while loading Roberta")
|
| 17 |
-
|
| 18 |
-
# Load the CML trained model
|
| 19 |
-
with open(LOGREG_MODEL_PATH, "r") as model_file:
|
| 20 |
-
cml_ner_model = load(file=model_file)
|
| 21 |
-
|
| 22 |
-
return embeddings_model, tokenizer, cml_ner_model
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
def anonymize_with_cml(text, embeddings_model, tokenizer, cml_ner_model):
|
| 26 |
-
token_pattern = r"(\b[\w\.\/\-@]+\b|[\s,.!?;:'\"-]+|\$\d+(?:\.\d+)?|\€\d+(?:\.\d+)?)"
|
| 27 |
-
|
| 28 |
-
tokens = re.findall(token_pattern, text)
|
| 29 |
-
uuid_map = {}
|
| 30 |
-
processed_tokens = []
|
| 31 |
-
|
| 32 |
-
for token in tokens:
|
| 33 |
-
if token.strip() and re.match(r"\w+", token): # If the token is a word
|
| 34 |
-
x = get_batch_text_representation([token], embeddings_model, tokenizer)
|
| 35 |
-
prediction_proba = cml_ner_model.predict_proba(x, fhe="disable")
|
| 36 |
-
probability = prediction_proba[0][1]
|
| 37 |
-
prediction = probability >= 0.77
|
| 38 |
-
if prediction:
|
| 39 |
-
if token not in uuid_map:
|
| 40 |
-
uuid_map[token] = str(uuid.uuid4())[:8]
|
| 41 |
-
processed_tokens.append(uuid_map[token])
|
| 42 |
-
else:
|
| 43 |
-
processed_tokens.append(token)
|
| 44 |
-
else:
|
| 45 |
-
processed_tokens.append(token) # Preserve punctuation and spaces as is
|
| 46 |
-
|
| 47 |
-
anonymized_text = "".join(processed_tokens)
|
| 48 |
-
return anonymized_text, uuid_map
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
def anonymize_text(text, verbose=False, save=False):
|
| 52 |
-
|
| 53 |
-
# Load models
|
| 54 |
-
if verbose:
|
| 55 |
-
print("Loading models..")
|
| 56 |
-
embeddings_model, tokenizer, cml_ner_model = load_models()
|
| 57 |
-
|
| 58 |
-
if verbose:
|
| 59 |
-
print(f"\nText to process:--------------------\n{text}\n--------------------\n")
|
| 60 |
-
|
| 61 |
-
# Save the original text to its specified file
|
| 62 |
-
if save:
|
| 63 |
-
write_txt(ORIGINAL_FILE_PATH, text)
|
| 64 |
-
|
| 65 |
-
# Anonymize the text
|
| 66 |
-
anonymized_text, uuid_map = anonymize_with_cml(text, embeddings_model, tokenizer, cml_ner_model)
|
| 67 |
-
|
| 68 |
-
# Save the anonymized text to its specified file
|
| 69 |
-
if save:
|
| 70 |
-
mapping = {o: (i, a) for i, (o, a) in enumerate(zip(text.split("\n\n"), anonymized_text.split("\n\n")))}
|
| 71 |
-
write_txt(ANONYMIZED_FILE_PATH, anonymized_text)
|
| 72 |
-
write_pickle(MAPPING_SENTENCES_PATH, mapping)
|
| 73 |
-
|
| 74 |
-
if verbose:
|
| 75 |
-
print(f"\nAnonymized text:--------------------\n{anonymized_text}\n--------------------\n")
|
| 76 |
-
|
| 77 |
-
# Save the UUID mapping to a JSON file
|
| 78 |
-
if save:
|
| 79 |
-
write_json(MAPPING_UUID_PATH, uuid_map)
|
| 80 |
-
|
| 81 |
-
if verbose and save:
|
| 82 |
-
print(f"Original text saved to :{ORIGINAL_FILE_PATH}")
|
| 83 |
-
print(f"Anonymized text saved to :{ANONYMIZED_FILE_PATH}")
|
| 84 |
-
print(f"UUID mapping saved to :{MAPPING_UUID_PATH}")
|
| 85 |
-
print(f"Sentence mapping saved to :{MAPPING_SENTENCES_PATH}")
|
| 86 |
-
|
| 87 |
-
return anonymized_text
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
if __name__ == "__main__":
|
| 91 |
-
parser = argparse.ArgumentParser(
|
| 92 |
-
description="Anonymize named entities in a text file and save the mapping to a JSON file."
|
| 93 |
-
)
|
| 94 |
-
parser.add_argument(
|
| 95 |
-
"--file_path",
|
| 96 |
-
type=str,
|
| 97 |
-
default="files/original_document.txt",
|
| 98 |
-
help="The path to the file to be processed.",
|
| 99 |
-
)
|
| 100 |
-
parser.add_argument(
|
| 101 |
-
"--verbose",
|
| 102 |
-
type=bool,
|
| 103 |
-
default=True,
|
| 104 |
-
help="This provides additional details about the program's execution.",
|
| 105 |
-
)
|
| 106 |
-
parser.add_argument("--save", type=bool, default=True, help="Save the files.")
|
| 107 |
-
|
| 108 |
-
args = parser.parse_args()
|
| 109 |
-
|
| 110 |
-
text = read_txt(args.file_path)
|
| 111 |
-
|
| 112 |
-
anonymize_text(text, verbose=args.verbose, save=args.save)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|