Test updating ner process

#1
Files changed (1) hide show
  1. app.py +5 -1
app.py CHANGED
@@ -3,7 +3,8 @@ import torch
3
  # import nltk
4
  # from nltk.corpus import stopwords
5
  # from nltk.stem import WordNetLemmatizer
6
- # from nltk.tokenize import word_tokenize
 
7
 
8
  from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoConfig
9
 
@@ -62,6 +63,9 @@ LABEL = ['O',
62
  'I-button']
63
 
64
  def analyze_ner(text):
 
 
 
65
  predictions = clf_ner(text)
66
  print(predictions) # Debugging statement to see the structure of the output
67
  keys = []
 
3
  # import nltk
4
  # from nltk.corpus import stopwords
5
  # from nltk.stem import WordNetLemmatizer
6
+ from nltk.tokenize import word_tokenize
7
+ import string
8
 
9
  from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoConfig
10
 
 
63
  'I-button']
64
 
65
  def analyze_ner(text):
66
+ text = word_tokenize(text)
67
+ text = [word for word in text if word not in string.punctuation]
68
+ text = ' '.join(text)
69
  predictions = clf_ner(text)
70
  print(predictions) # Debugging statement to see the structure of the output
71
  keys = []