Spaces:
Runtime error
Runtime error
Test updating ner process (#1)
Browse files- Test updating ner process (6b9a1e95db866a56b2074c363abad31870e78ec2)
Co-authored-by: Vin <IdjiotSandwiches@users.noreply.huggingface.co>
app.py
CHANGED
|
@@ -3,7 +3,8 @@ import torch
|
|
| 3 |
# import nltk
|
| 4 |
# from nltk.corpus import stopwords
|
| 5 |
# from nltk.stem import WordNetLemmatizer
|
| 6 |
-
|
|
|
|
| 7 |
|
| 8 |
from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoConfig
|
| 9 |
|
|
@@ -62,6 +63,9 @@ LABEL = ['O',
|
|
| 62 |
'I-button']
|
| 63 |
|
| 64 |
def analyze_ner(text):
|
|
|
|
|
|
|
|
|
|
| 65 |
predictions = clf_ner(text)
|
| 66 |
print(predictions) # Debugging statement to see the structure of the output
|
| 67 |
keys = []
|
|
|
|
| 3 |
# import nltk
|
| 4 |
# from nltk.corpus import stopwords
|
| 5 |
# from nltk.stem import WordNetLemmatizer
|
| 6 |
+
from nltk.tokenize import word_tokenize
|
| 7 |
+
import string
|
| 8 |
|
| 9 |
from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoConfig
|
| 10 |
|
|
|
|
| 63 |
'I-button']
|
| 64 |
|
| 65 |
def analyze_ner(text):
|
| 66 |
+
text = word_tokenize(text)
|
| 67 |
+
text = [word for word in text if word not in string.punctuation]
|
| 68 |
+
text = ' '.join(text)
|
| 69 |
predictions = clf_ner(text)
|
| 70 |
print(predictions) # Debugging statement to see the structure of the output
|
| 71 |
keys = []
|