import os import time import uuid from typing import Any, Generator import gradio as gr import pandas as pd from openai import OpenAI from sklearn.metrics import classification_report #df = pd.read_csv('tweets.csv') df = pd.read_csv('cybercar_posts.csv') df = df.sample(frac=1).reset_index(drop=True) # df = df.head(10) # only use N samples for development stopped_sessions = {} openai_client = OpenAI(api_key=os.getenv("OPENAI_KEY")) def get_llm_response(system_prompt: str, text: str) -> str: response = openai_client.responses.create( model="gpt-4o-mini", instructions=system_prompt, input=text, temperature=0 ) return response.output_text.strip() def classify( system_prompt: str, data: pd.DataFrame, session_id: str, progress=gr.Progress() ) -> Generator[tuple[pd.DataFrame, str], Any, tuple[pd.DataFrame, str]]: predictions = [] n = len(data) for i, (text, label) in enumerate(zip(data['text'], data['label'])): # update progress bar progress(i / n, desc=f"Processing {i+1}/{n}") predicted_label = get_llm_response(system_prompt, text) predictions.append(predicted_label) time.sleep(0.1) data = data.copy() data["predictions"] = "" data.loc[: len(predictions) - 1, "predictions"] = predictions yield data, f"Completed {i+1}/{n}" if session_id in stopped_sessions and stopped_sessions[session_id]: break # final return return data, "Done!" def evaluate(data: pd.DataFrame) -> str: if 'predictions' not in data.columns: return "There seem to be no predictions in the data. Classify all texts first." return classification_report( data['label'].tolist(), data['predictions'].tolist() ) def evaluate_file(excel_file: str) -> str: eval_data = pd.read_excel(excel_file) return evaluate(eval_data) with gr.Blocks() as app: gr.Markdown("# Text classification with prompts\n" "You are given some data below with ground truth labels. Your task is to write a prompt that replicates these labels with an LLM.\n" "1. Optimize the system prompt.\n" "2. Test it on a custom input (you can change it).\n" "3. Once you're satisfied, press 'Classify all texts 🎯'. The data will receive an additional column 'predicted' that shows the labels predicted by the LLM using your prompt incrementally.\n" "3. Once finished, press 'Evaluate πŸ”Ž' to get evaluation results (F1 Scores). Note that the LLM output needs to be the same format as the label column. Does your prompt deliver the right format?\n" "4. GOTO 1 to try to improve the prompt." ) system_prompt = gr.Textbox(label="1. System Prompt", placeholder="Enter system prompt here.", value="Classify the text sentiment.", interactive=True) with gr.Accordion(label="2. Classify custom text"): with gr.Row(): custom_input = gr.Textbox(label="Custom text", placeholder="Enter your text here.", value="SchΓΆnes Wetter. NOT!") custom_output = gr.Textbox(label="Prediction of custom input") classify_custom_btn = gr.Button(value="Classify custom text 🎯") with gr.Accordion(label="3. Classify all texts"): progress_display = gr.Textbox(label="Progress") with gr.Row(): classify_btn = gr.Button(value="Classify all texts 🎯", variant="primary") stop_button = gr.Button(variant="stop", value="Stop") data_display = gr.DataFrame(df, label="Data") with gr.Accordion(label="4. Evaluate"): score_display = gr.Textbox(label="Evaluation scores", lines=10, max_lines=20) eval_btn = gr.Button(value="Evaluate πŸ”Ž") file_upload = gr.File(label="Upload Excel file for Evaluation. Needs columns 'label' and 'predictions'.") eval_custom_btn = gr.Button(value="Evaluate custom data πŸ”Ž") session_id = gr.State() def get_sessionid(): return str(uuid.uuid4()) classify_btn.click( fn=get_sessionid, outputs=session_id ).then( fn=classify, inputs=[system_prompt, data_display, session_id], outputs=[data_display, progress_display], show_progress="full", show_progress_on=[progress_display] ) eval_btn.click( fn=evaluate, inputs=data_display, outputs=score_display ) classify_custom_btn.click( fn=get_llm_response, inputs=[system_prompt, custom_input], outputs=custom_output ) def stop(session_id: str): stopped_sessions[session_id] = True stop_button.click( fn=stop, inputs=session_id ) eval_custom_btn.click( fn=evaluate_file, inputs=file_upload, outputs=score_display ) app.queue(default_concurrency_limit=None) app.launch(auth=(os.getenv("USER"), os.getenv("PASSWORD")), ssr_mode=False)