Spaces:
Sleeping
Sleeping
Download sentiment_prompting.py from tuge/SentimentPrompting: direct link, hf CLI and curl.
- Browser
- Download file 5.03 kB
-
https://huggingface.co/spaces/tuge/SentimentPrompting/resolve/main/sentiment_prompting.py
- Command line
-
hf download hf://spaces/tuge/SentimentPrompting/sentiment_prompting.py
-
curl -L -o sentiment_prompting.py https://huggingface.co/spaces/tuge/SentimentPrompting/resolve/main/sentiment_prompting.py
5.03 kB
| import os | |
| import time | |
| import uuid | |
| from typing import Any, Generator | |
| import gradio as gr | |
| import pandas as pd | |
| from openai import OpenAI | |
| from sklearn.metrics import classification_report | |
| #df = pd.read_csv('tweets.csv') | |
| df = pd.read_csv('cybercar_posts.csv') | |
| df = df.sample(frac=1).reset_index(drop=True) | |
| # df = df.head(10) # only use N samples for development | |
| stopped_sessions = {} | |
| openai_client = OpenAI(api_key=os.getenv("OPENAI_KEY")) | |
| def get_llm_response(system_prompt: str, text: str) -> str: | |
| response = openai_client.responses.create( | |
| model="gpt-4o-mini", | |
| instructions=system_prompt, | |
| input=text, | |
| temperature=0 | |
| ) | |
| return response.output_text.strip() | |
| def classify( | |
| system_prompt: str, | |
| data: pd.DataFrame, | |
| session_id: str, | |
| progress=gr.Progress() | |
| ) -> Generator[tuple[pd.DataFrame, str], Any, tuple[pd.DataFrame, str]]: | |
| predictions = [] | |
| n = len(data) | |
| for i, (text, label) in enumerate(zip(data['text'], data['label'])): | |
| # update progress bar | |
| progress(i / n, desc=f"Processing {i+1}/{n}") | |
| predicted_label = get_llm_response(system_prompt, text) | |
| predictions.append(predicted_label) | |
| time.sleep(0.1) | |
| data = data.copy() | |
| data["predictions"] = "" | |
| data.loc[: len(predictions) - 1, "predictions"] = predictions | |
| yield data, f"Completed {i+1}/{n}" | |
| if session_id in stopped_sessions and stopped_sessions[session_id]: | |
| break | |
| # final return | |
| return data, "Done!" | |
| def evaluate(data: pd.DataFrame) -> str: | |
| if 'predictions' not in data.columns: | |
| return "There seem to be no predictions in the data. Classify all texts first." | |
| return classification_report( | |
| data['label'].tolist(), | |
| data['predictions'].tolist() | |
| ) | |
| def evaluate_file(excel_file: str) -> str: | |
| eval_data = pd.read_excel(excel_file) | |
| return evaluate(eval_data) | |
| with gr.Blocks() as app: | |
| gr.Markdown("# Text classification with prompts\n" | |
| "You are given some data below with ground truth labels. Your task is to write a prompt that replicates these labels with an LLM.\n" | |
| "1. Optimize the system prompt.\n" | |
| "2. Test it on a custom input (you can change it).\n" | |
| "3. Once you're satisfied, press 'Classify all texts 🎯'. The data will receive an additional column 'predicted' that shows the labels predicted by the LLM using your prompt incrementally.\n" | |
| "3. Once finished, press 'Evaluate 🔎' to get evaluation results (F1 Scores). Note that the LLM output needs to be the same format as the label column. Does your prompt deliver the right format?\n" | |
| "4. GOTO 1 to try to improve the prompt." | |
| ) | |
| system_prompt = gr.Textbox(label="1. System Prompt", placeholder="Enter system prompt here.", value="Classify the text sentiment.", interactive=True) | |
| with gr.Accordion(label="2. Classify custom text"): | |
| with gr.Row(): | |
| custom_input = gr.Textbox(label="Custom text", placeholder="Enter your text here.", value="Schönes Wetter. NOT!") | |
| custom_output = gr.Textbox(label="Prediction of custom input") | |
| classify_custom_btn = gr.Button(value="Classify custom text 🎯") | |
| with gr.Accordion(label="3. Classify all texts"): | |
| progress_display = gr.Textbox(label="Progress") | |
| with gr.Row(): | |
| classify_btn = gr.Button(value="Classify all texts 🎯", variant="primary") | |
| stop_button = gr.Button(variant="stop", value="Stop") | |
| data_display = gr.DataFrame(df, label="Data") | |
| with gr.Accordion(label="4. Evaluate"): | |
| score_display = gr.Textbox(label="Evaluation scores", lines=10, max_lines=20) | |
| eval_btn = gr.Button(value="Evaluate 🔎") | |
| file_upload = gr.File(label="Upload Excel file for Evaluation. Needs columns 'label' and 'predictions'.") | |
| eval_custom_btn = gr.Button(value="Evaluate custom data 🔎") | |
| session_id = gr.State() | |
| def get_sessionid(): | |
| return str(uuid.uuid4()) | |
| classify_btn.click( | |
| fn=get_sessionid, | |
| outputs=session_id | |
| ).then( | |
| fn=classify, | |
| inputs=[system_prompt, data_display, session_id], | |
| outputs=[data_display, progress_display], | |
| show_progress="full", | |
| show_progress_on=[progress_display] | |
| ) | |
| eval_btn.click( | |
| fn=evaluate, | |
| inputs=data_display, | |
| outputs=score_display | |
| ) | |
| classify_custom_btn.click( | |
| fn=get_llm_response, | |
| inputs=[system_prompt, custom_input], | |
| outputs=custom_output | |
| ) | |
| def stop(session_id: str): | |
| stopped_sessions[session_id] = True | |
| stop_button.click( | |
| fn=stop, | |
| inputs=session_id | |
| ) | |
| eval_custom_btn.click( | |
| fn=evaluate_file, | |
| inputs=file_upload, | |
| outputs=score_display | |
| ) | |
| app.queue(default_concurrency_limit=None) | |
| app.launch(auth=(os.getenv("USER"), os.getenv("PASSWORD")), ssr_mode=False) | |