80cols commited on
Commit
78e08ec
·
verified ·
1 Parent(s): efc037d

Delete anonymize_file_clear.py

Browse files
Files changed (1) hide show
  1. anonymize_file_clear.py +0 -112
anonymize_file_clear.py DELETED
@@ -1,112 +0,0 @@
1
- import argparse
2
- import re
3
- import uuid
4
-
5
- from transformers import AutoModel, AutoTokenizer
6
- from concrete.ml.common.serialization.loaders import load
7
- from utils_demo import *
8
-
9
- def load_models():
10
-
11
- # Load the tokenizer and the embedding model
12
- try:
13
- tokenizer = AutoTokenizer.from_pretrained("obi/deid_roberta_i2b2")
14
- embeddings_model = AutoModel.from_pretrained("obi/deid_roberta_i2b2")
15
- except:
16
- print("Error while loading Roberta")
17
-
18
- # Load the CML trained model
19
- with open(LOGREG_MODEL_PATH, "r") as model_file:
20
- cml_ner_model = load(file=model_file)
21
-
22
- return embeddings_model, tokenizer, cml_ner_model
23
-
24
-
25
- def anonymize_with_cml(text, embeddings_model, tokenizer, cml_ner_model):
26
- token_pattern = r"(\b[\w\.\/\-@]+\b|[\s,.!?;:'\"-]+|\$\d+(?:\.\d+)?|\€\d+(?:\.\d+)?)"
27
-
28
- tokens = re.findall(token_pattern, text)
29
- uuid_map = {}
30
- processed_tokens = []
31
-
32
- for token in tokens:
33
- if token.strip() and re.match(r"\w+", token): # If the token is a word
34
- x = get_batch_text_representation([token], embeddings_model, tokenizer)
35
- prediction_proba = cml_ner_model.predict_proba(x, fhe="disable")
36
- probability = prediction_proba[0][1]
37
- prediction = probability >= 0.77
38
- if prediction:
39
- if token not in uuid_map:
40
- uuid_map[token] = str(uuid.uuid4())[:8]
41
- processed_tokens.append(uuid_map[token])
42
- else:
43
- processed_tokens.append(token)
44
- else:
45
- processed_tokens.append(token) # Preserve punctuation and spaces as is
46
-
47
- anonymized_text = "".join(processed_tokens)
48
- return anonymized_text, uuid_map
49
-
50
-
51
- def anonymize_text(text, verbose=False, save=False):
52
-
53
- # Load models
54
- if verbose:
55
- print("Loading models..")
56
- embeddings_model, tokenizer, cml_ner_model = load_models()
57
-
58
- if verbose:
59
- print(f"\nText to process:--------------------\n{text}\n--------------------\n")
60
-
61
- # Save the original text to its specified file
62
- if save:
63
- write_txt(ORIGINAL_FILE_PATH, text)
64
-
65
- # Anonymize the text
66
- anonymized_text, uuid_map = anonymize_with_cml(text, embeddings_model, tokenizer, cml_ner_model)
67
-
68
- # Save the anonymized text to its specified file
69
- if save:
70
- mapping = {o: (i, a) for i, (o, a) in enumerate(zip(text.split("\n\n"), anonymized_text.split("\n\n")))}
71
- write_txt(ANONYMIZED_FILE_PATH, anonymized_text)
72
- write_pickle(MAPPING_SENTENCES_PATH, mapping)
73
-
74
- if verbose:
75
- print(f"\nAnonymized text:--------------------\n{anonymized_text}\n--------------------\n")
76
-
77
- # Save the UUID mapping to a JSON file
78
- if save:
79
- write_json(MAPPING_UUID_PATH, uuid_map)
80
-
81
- if verbose and save:
82
- print(f"Original text saved to :{ORIGINAL_FILE_PATH}")
83
- print(f"Anonymized text saved to :{ANONYMIZED_FILE_PATH}")
84
- print(f"UUID mapping saved to :{MAPPING_UUID_PATH}")
85
- print(f"Sentence mapping saved to :{MAPPING_SENTENCES_PATH}")
86
-
87
- return anonymized_text
88
-
89
-
90
- if __name__ == "__main__":
91
- parser = argparse.ArgumentParser(
92
- description="Anonymize named entities in a text file and save the mapping to a JSON file."
93
- )
94
- parser.add_argument(
95
- "--file_path",
96
- type=str,
97
- default="files/original_document.txt",
98
- help="The path to the file to be processed.",
99
- )
100
- parser.add_argument(
101
- "--verbose",
102
- type=bool,
103
- default=True,
104
- help="This provides additional details about the program's execution.",
105
- )
106
- parser.add_argument("--save", type=bool, default=True, help="Save the files.")
107
-
108
- args = parser.parse_args()
109
-
110
- text = read_txt(args.file_path)
111
-
112
- anonymize_text(text, verbose=args.verbose, save=args.save)