SentimentPrompting / sentiment_prompting.py
tuge's picture
Update sentiment_prompting.py
59742aa verified
Raw History Blame Contribute Delete
5.03 kB
import os
import time
import uuid
from typing import Any, Generator
import gradio as gr
import pandas as pd
from openai import OpenAI
from sklearn.metrics import classification_report
#df = pd.read_csv('tweets.csv')
df = pd.read_csv('cybercar_posts.csv')
df = df.sample(frac=1).reset_index(drop=True)
# df = df.head(10) # only use N samples for development
stopped_sessions = {}
openai_client = OpenAI(api_key=os.getenv("OPENAI_KEY"))
def get_llm_response(system_prompt: str, text: str) -> str:
response = openai_client.responses.create(
model="gpt-4o-mini",
instructions=system_prompt,
input=text,
temperature=0
)
return response.output_text.strip()
def classify(
system_prompt: str,
data: pd.DataFrame,
session_id: str,
progress=gr.Progress()
) -> Generator[tuple[pd.DataFrame, str], Any, tuple[pd.DataFrame, str]]:
predictions = []
n = len(data)
for i, (text, label) in enumerate(zip(data['text'], data['label'])):
# update progress bar
progress(i / n, desc=f"Processing {i+1}/{n}")
predicted_label = get_llm_response(system_prompt, text)
predictions.append(predicted_label)
time.sleep(0.1)
data = data.copy()
data["predictions"] = ""
data.loc[: len(predictions) - 1, "predictions"] = predictions
yield data, f"Completed {i+1}/{n}"
if session_id in stopped_sessions and stopped_sessions[session_id]:
break
# final return
return data, "Done!"
def evaluate(data: pd.DataFrame) -> str:
if 'predictions' not in data.columns:
return "There seem to be no predictions in the data. Classify all texts first."
return classification_report(
data['label'].tolist(),
data['predictions'].tolist()
)
def evaluate_file(excel_file: str) -> str:
eval_data = pd.read_excel(excel_file)
return evaluate(eval_data)
with gr.Blocks() as app:
gr.Markdown("# Text classification with prompts\n"
"You are given some data below with ground truth labels. Your task is to write a prompt that replicates these labels with an LLM.\n"
"1. Optimize the system prompt.\n"
"2. Test it on a custom input (you can change it).\n"
"3. Once you're satisfied, press 'Classify all texts 🎯'. The data will receive an additional column 'predicted' that shows the labels predicted by the LLM using your prompt incrementally.\n"
"3. Once finished, press 'Evaluate 🔎' to get evaluation results (F1 Scores). Note that the LLM output needs to be the same format as the label column. Does your prompt deliver the right format?\n"
"4. GOTO 1 to try to improve the prompt."
)
system_prompt = gr.Textbox(label="1. System Prompt", placeholder="Enter system prompt here.", value="Classify the text sentiment.", interactive=True)
with gr.Accordion(label="2. Classify custom text"):
with gr.Row():
custom_input = gr.Textbox(label="Custom text", placeholder="Enter your text here.", value="Schönes Wetter. NOT!")
custom_output = gr.Textbox(label="Prediction of custom input")
classify_custom_btn = gr.Button(value="Classify custom text 🎯")
with gr.Accordion(label="3. Classify all texts"):
progress_display = gr.Textbox(label="Progress")
with gr.Row():
classify_btn = gr.Button(value="Classify all texts 🎯", variant="primary")
stop_button = gr.Button(variant="stop", value="Stop")
data_display = gr.DataFrame(df, label="Data")
with gr.Accordion(label="4. Evaluate"):
score_display = gr.Textbox(label="Evaluation scores", lines=10, max_lines=20)
eval_btn = gr.Button(value="Evaluate 🔎")
file_upload = gr.File(label="Upload Excel file for Evaluation. Needs columns 'label' and 'predictions'.")
eval_custom_btn = gr.Button(value="Evaluate custom data 🔎")
session_id = gr.State()
def get_sessionid():
return str(uuid.uuid4())
classify_btn.click(
fn=get_sessionid,
outputs=session_id
).then(
fn=classify,
inputs=[system_prompt, data_display, session_id],
outputs=[data_display, progress_display],
show_progress="full",
show_progress_on=[progress_display]
)
eval_btn.click(
fn=evaluate,
inputs=data_display,
outputs=score_display
)
classify_custom_btn.click(
fn=get_llm_response,
inputs=[system_prompt, custom_input],
outputs=custom_output
)
def stop(session_id: str):
stopped_sessions[session_id] = True
stop_button.click(
fn=stop,
inputs=session_id
)
eval_custom_btn.click(
fn=evaluate_file,
inputs=file_upload,
outputs=score_display
)
app.queue(default_concurrency_limit=None)
app.launch(auth=(os.getenv("USER"), os.getenv("PASSWORD")), ssr_mode=False)