Spaces:
Sleeping
Sleeping
File size: 5,031 Bytes
99f4aae 9a37cdd 99f4aae 9a37cdd 99f4aae 9a37cdd 99f4aae 9a37cdd 99f4aae c4d1411 99f4aae 9a37cdd bf572c8 99f4aae c4d1411 9a37cdd 99f4aae 9a37cdd 99f4aae 9a37cdd 99f4aae 9a37cdd c4d1411 59742aa 7e5f3ba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | import os
import time
import uuid
from typing import Any, Generator
import gradio as gr
import pandas as pd
from openai import OpenAI
from sklearn.metrics import classification_report
#df = pd.read_csv('tweets.csv')
df = pd.read_csv('cybercar_posts.csv')
df = df.sample(frac=1).reset_index(drop=True)
# df = df.head(10) # only use N samples for development
stopped_sessions = {}
openai_client = OpenAI(api_key=os.getenv("OPENAI_KEY"))
def get_llm_response(system_prompt: str, text: str) -> str:
response = openai_client.responses.create(
model="gpt-4o-mini",
instructions=system_prompt,
input=text,
temperature=0
)
return response.output_text.strip()
def classify(
system_prompt: str,
data: pd.DataFrame,
session_id: str,
progress=gr.Progress()
) -> Generator[tuple[pd.DataFrame, str], Any, tuple[pd.DataFrame, str]]:
predictions = []
n = len(data)
for i, (text, label) in enumerate(zip(data['text'], data['label'])):
# update progress bar
progress(i / n, desc=f"Processing {i+1}/{n}")
predicted_label = get_llm_response(system_prompt, text)
predictions.append(predicted_label)
time.sleep(0.1)
data = data.copy()
data["predictions"] = ""
data.loc[: len(predictions) - 1, "predictions"] = predictions
yield data, f"Completed {i+1}/{n}"
if session_id in stopped_sessions and stopped_sessions[session_id]:
break
# final return
return data, "Done!"
def evaluate(data: pd.DataFrame) -> str:
if 'predictions' not in data.columns:
return "There seem to be no predictions in the data. Classify all texts first."
return classification_report(
data['label'].tolist(),
data['predictions'].tolist()
)
def evaluate_file(excel_file: str) -> str:
eval_data = pd.read_excel(excel_file)
return evaluate(eval_data)
with gr.Blocks() as app:
gr.Markdown("# Text classification with prompts\n"
"You are given some data below with ground truth labels. Your task is to write a prompt that replicates these labels with an LLM.\n"
"1. Optimize the system prompt.\n"
"2. Test it on a custom input (you can change it).\n"
"3. Once you're satisfied, press 'Classify all texts 🎯'. The data will receive an additional column 'predicted' that shows the labels predicted by the LLM using your prompt incrementally.\n"
"3. Once finished, press 'Evaluate 🔎' to get evaluation results (F1 Scores). Note that the LLM output needs to be the same format as the label column. Does your prompt deliver the right format?\n"
"4. GOTO 1 to try to improve the prompt."
)
system_prompt = gr.Textbox(label="1. System Prompt", placeholder="Enter system prompt here.", value="Classify the text sentiment.", interactive=True)
with gr.Accordion(label="2. Classify custom text"):
with gr.Row():
custom_input = gr.Textbox(label="Custom text", placeholder="Enter your text here.", value="Schönes Wetter. NOT!")
custom_output = gr.Textbox(label="Prediction of custom input")
classify_custom_btn = gr.Button(value="Classify custom text 🎯")
with gr.Accordion(label="3. Classify all texts"):
progress_display = gr.Textbox(label="Progress")
with gr.Row():
classify_btn = gr.Button(value="Classify all texts 🎯", variant="primary")
stop_button = gr.Button(variant="stop", value="Stop")
data_display = gr.DataFrame(df, label="Data")
with gr.Accordion(label="4. Evaluate"):
score_display = gr.Textbox(label="Evaluation scores", lines=10, max_lines=20)
eval_btn = gr.Button(value="Evaluate 🔎")
file_upload = gr.File(label="Upload Excel file for Evaluation. Needs columns 'label' and 'predictions'.")
eval_custom_btn = gr.Button(value="Evaluate custom data 🔎")
session_id = gr.State()
def get_sessionid():
return str(uuid.uuid4())
classify_btn.click(
fn=get_sessionid,
outputs=session_id
).then(
fn=classify,
inputs=[system_prompt, data_display, session_id],
outputs=[data_display, progress_display],
show_progress="full",
show_progress_on=[progress_display]
)
eval_btn.click(
fn=evaluate,
inputs=data_display,
outputs=score_display
)
classify_custom_btn.click(
fn=get_llm_response,
inputs=[system_prompt, custom_input],
outputs=custom_output
)
def stop(session_id: str):
stopped_sessions[session_id] = True
stop_button.click(
fn=stop,
inputs=session_id
)
eval_custom_btn.click(
fn=evaluate_file,
inputs=file_upload,
outputs=score_display
)
app.queue(default_concurrency_limit=None)
app.launch(auth=(os.getenv("USER"), os.getenv("PASSWORD")), ssr_mode=False)
|