File size: 5,031 Bytes
99f4aae
 
9a37cdd
99f4aae
 
 
 
 
 
 
 
 
 
 
 
9a37cdd
 
99f4aae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a37cdd
99f4aae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a37cdd
 
 
99f4aae
 
 
 
 
 
 
 
 
 
 
 
 
c4d1411
 
 
 
99f4aae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a37cdd
 
 
bf572c8
99f4aae
 
 
 
 
c4d1411
 
 
9a37cdd
 
 
 
 
 
99f4aae
9a37cdd
 
 
99f4aae
9a37cdd
99f4aae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9a37cdd
 
 
 
 
 
 
 
c4d1411
 
 
 
 
 
59742aa
7e5f3ba
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
import os
import time
import uuid
from typing import Any, Generator

import gradio as gr
import pandas as pd
from openai import OpenAI
from sklearn.metrics import classification_report

#df = pd.read_csv('tweets.csv')
df = pd.read_csv('cybercar_posts.csv')
df = df.sample(frac=1).reset_index(drop=True)
# df = df.head(10) # only use N samples for development

stopped_sessions = {}

openai_client = OpenAI(api_key=os.getenv("OPENAI_KEY"))

def get_llm_response(system_prompt: str, text: str) -> str:
    response = openai_client.responses.create(
        model="gpt-4o-mini",
        instructions=system_prompt,
        input=text,
        temperature=0
    )
    return response.output_text.strip()


def classify(
    system_prompt: str,
    data: pd.DataFrame,
    session_id: str,
    progress=gr.Progress()
) -> Generator[tuple[pd.DataFrame, str], Any, tuple[pd.DataFrame, str]]:

    predictions = []
    n = len(data)

    for i, (text, label) in enumerate(zip(data['text'], data['label'])):
        # update progress bar
        progress(i / n, desc=f"Processing {i+1}/{n}")
        predicted_label = get_llm_response(system_prompt, text)
        predictions.append(predicted_label)
        time.sleep(0.1)

        data = data.copy()
        data["predictions"] = ""
        data.loc[: len(predictions) - 1, "predictions"] = predictions
        yield data, f"Completed {i+1}/{n}"

        if session_id in stopped_sessions and stopped_sessions[session_id]:
            break

    # final return
    return data, "Done!"


def evaluate(data: pd.DataFrame) -> str:
    if 'predictions' not in data.columns:
        return "There seem to be no predictions in the data. Classify all texts first."
    return classification_report(
        data['label'].tolist(),
        data['predictions'].tolist()
    )


def evaluate_file(excel_file: str) -> str:
    eval_data = pd.read_excel(excel_file)
    return evaluate(eval_data)

with gr.Blocks() as app:

    gr.Markdown("# Text classification with prompts\n"
                "You are given some data below with ground truth labels. Your task is to write a prompt that replicates these labels with an LLM.\n"
                "1. Optimize the system prompt.\n"
                "2. Test it on a custom input (you can change it).\n"
                "3. Once you're satisfied, press 'Classify all texts 🎯'. The data will receive an additional column 'predicted' that shows the labels predicted by the LLM using your prompt incrementally.\n"
                "3. Once finished, press 'Evaluate 🔎' to get evaluation results (F1 Scores). Note that the LLM output needs to be the same format as the label column. Does your prompt deliver the right format?\n"
                "4. GOTO 1 to try to improve the prompt."
                )

    system_prompt = gr.Textbox(label="1. System Prompt", placeholder="Enter system prompt here.", value="Classify the text sentiment.", interactive=True)

    with gr.Accordion(label="2. Classify custom text"):
        with gr.Row():
            custom_input = gr.Textbox(label="Custom text", placeholder="Enter your text here.", value="Schönes Wetter. NOT!")
            custom_output = gr.Textbox(label="Prediction of custom input")
        classify_custom_btn = gr.Button(value="Classify custom text 🎯")

    with gr.Accordion(label="3. Classify all texts"):
        progress_display = gr.Textbox(label="Progress")
        with gr.Row():
            classify_btn = gr.Button(value="Classify all texts 🎯", variant="primary")
            stop_button = gr.Button(variant="stop", value="Stop")
        data_display = gr.DataFrame(df, label="Data")

    with gr.Accordion(label="4. Evaluate"):
        score_display = gr.Textbox(label="Evaluation scores", lines=10, max_lines=20)
        eval_btn = gr.Button(value="Evaluate 🔎")

        file_upload = gr.File(label="Upload Excel file for Evaluation. Needs columns 'label' and 'predictions'.")
        eval_custom_btn = gr.Button(value="Evaluate custom data 🔎")


    session_id = gr.State()

    def get_sessionid():
        return str(uuid.uuid4())

    classify_btn.click(
        fn=get_sessionid,
        outputs=session_id
    ).then(
        fn=classify,
        inputs=[system_prompt, data_display, session_id],
        outputs=[data_display, progress_display],
        show_progress="full",
        show_progress_on=[progress_display]
    )

    eval_btn.click(
        fn=evaluate,
        inputs=data_display,
        outputs=score_display
    )

    classify_custom_btn.click(
        fn=get_llm_response,
        inputs=[system_prompt, custom_input],
        outputs=custom_output
    )

    def stop(session_id: str):
        stopped_sessions[session_id] = True

    stop_button.click(
        fn=stop,
        inputs=session_id
    )

    eval_custom_btn.click(
        fn=evaluate_file,
        inputs=file_upload,
        outputs=score_display
    )

app.queue(default_concurrency_limit=None)
app.launch(auth=(os.getenv("USER"), os.getenv("PASSWORD")), ssr_mode=False)