straightline777's picture
Update app.py
c1a74e5 verified
Raw History Blame Contribute Delete
3.54 kB
import gradio as gr
import torch
import nltk
# from nltk.corpus import stopwords
# from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
nltk.download('punkt')
nltk.download('stopwords')
from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, AutoConfig
# Load your model and tokenizer from local paths using SafeTensors
model_NER = AutoModelForTokenClassification.from_pretrained("./NER_model")
tokenizer_NER = AutoTokenizer.from_pretrained("./tokenizer_NER")
model_path = './model'
tokenizer_path = './tokenizer'
model_sentiment = AutoModelForSequenceClassification.from_pretrained(model_path)
tokenizer_sentiment = AutoTokenizer.from_pretrained(tokenizer_path)
clf_sentiment = pipeline("text-classification", model=model_sentiment, tokenizer=tokenizer_sentiment)
clf_ner = pipeline("ner", model=model_NER, tokenizer=tokenizer_NER)
config = AutoConfig.from_pretrained("./NER_model")
label_list = config.id2label
# def preprocess(text):
# stop_words = set(stopwords.words('english'))
# lemmatizer = WordNetLemmatizer()
# tokens = word_tokenize(text)
# processed = [lemmatizer.lemmatize(word.lower()) for word in tokens if word.isalpha() and word not in stop_words]
# return ' '.join(processed)
def analyze_sentiment(text):
# text = preprocess(text)
predictions = clf_sentiment(text)
label_mapping = {'LABEL_0': 'Negative', 'LABEL_1': 'Neutral', 'LABEL_2': 'Positive'}
predicted_label = label_mapping[predictions[0]['label']]
confidence = f"{predictions[0]['score'] * 100:.2f}%"
return f"Predicted Label: {predicted_label}\nConfidence: {confidence}"
LABEL = ['O',
'B-storage',
'I-storage',
'B-performance',
'I-performance',
'B-battery',
'I-battery',
'B-camera',
'I-camera',
'B-charging',
'I-charging',
'B-screen',
'I-screen',
'B-audio',
'I-audio',
'B-feature',
'I-feature',
'B-connection',
'I-connection',
'B-button',
'I-button']
def preprocess(text):
if not isinstance(text, str):
return ""
stop_words = set(stopwords.words('english'))
tokens = word_tokenize(text)
processed = [word.lower() for word in tokens if word.isalnum() and word not in stop_words]
return ' '.join(processed)
def analyze_ner(text):
text = preprocess(text)
predictions = clf_ner(text)
print(predictions) # Debugging statement to see the structure of the output
keys = []
values = []
for i, label in enumerate(LABEL):
keys.append(f'LABEL_{i}')
values.append(label)
label_mapping = { k:v for (k,v) in zip(keys, values) }
results = []
for pred in predictions:
result = f"""Word: {pred['word']}
Predicted Tag: {label_mapping[pred['entity']]}
Confidence Probability: {pred['score'] * 100:.2f}%\n"""
results.append(result)
return "\n".join(results)
def analyze_text(text):
sentiment_result = analyze_sentiment(text)
ner_result = analyze_ner(text)
return sentiment_result, ner_result
interface = gr.Interface(
fn=analyze_text,
inputs=gr.Textbox(lines=2, placeholder="Enter text here..."),
outputs=[
gr.Textbox(label="Sentiment Analysis Result"),
gr.Textbox(label="Named Entity Recognition Result")
],
title="Text Analysis",
description="This interface allows you to perform Sentiment Analysis and Named Entity Recognition (NER) on text."
)
if __name__ == "__main__":
interface.launch(share=True)