Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import numpy as np | |
| import matplotlib.pyplot as plt | |
| from openai import OpenAI | |
| import random | |
| import time | |
| from dotenv import load_dotenv | |
| import os | |
| # Load environment variables | |
| load_dotenv() | |
| # Initialize OpenAI client | |
| client = OpenAI() | |
| # Define benchmark prompt | |
| PROMPT_A = "Benchmark Human-like Template" | |
| PROMPT_B = "Custom Template" | |
| template_messages_A = [ | |
| { | |
| "role": "system", | |
| "content": "You are a helpful assistant that always answers questions. Keep it short. Answer like you are a real human. For example, you can use emotions, metaphors and proverbs. Try to always be positive, and help the user with their questions, doubts and problems. Don't be pessimistic." | |
| }, | |
| { | |
| "role": "user", | |
| "content": "{question}" | |
| } | |
| ] | |
| def format_messages(template, question): | |
| return [ | |
| { | |
| "role": msg["role"], | |
| "content": msg["content"].format(question=question) | |
| } | |
| for msg in template | |
| ] | |
| def run_agent(question: str, group: str, custom_template: str): | |
| if group == "A": | |
| messages = format_messages(template_messages_A, question) | |
| else: | |
| # Use custom template for group B | |
| template_messages_B = [ | |
| { | |
| "role": "system", | |
| "content": custom_template | |
| }, | |
| { | |
| "role": "user", | |
| "content": "{question}" | |
| } | |
| ] | |
| messages = format_messages(template_messages_B, question) | |
| # Run GPT | |
| completion = client.chat.completions.create( | |
| model="gpt-4o", | |
| messages=messages | |
| ) | |
| return completion.choices[0].message.content | |
| def analyze_response(text): | |
| messages = [ | |
| {"role": "system", "content": "You are trained to analyze and detect the sentiment of given text."}, | |
| {"role": "user", "content": f"""Analyze the following recommendation and determine if the output is human-like. Check if there are emotions used, and metaphors and figure of speech. | |
| Assign a score: Based on your evaluation assign a score to the agent's performans using the following scale: | |
| - 1 (Poor): The agent is very machine like, doesn't use emotions, methaphors and figure of speech. | |
| - 2 (Fair): The agent is some human-likeness, some emotions, methaphors and figure of speech are used | |
| - 3 (Good): The agent is is human-like, uses enough emotions, methaphors and figure of speech. | |
| - 4 (Very Good): The agent very human-like, uses multiple emotions, methaphors and figure of speech. | |
| - 5 (Excellent): You almost cannot distinguish between the machine and the human, a lot emotions, methaphors and figure of speech are used. | |
| After evaluating the conversation based on the criteria above, provide your score as an integer between 1 and 5. Only answer with a single character in the following value {1, 2, 3, 4, 5}. | |
| Don't provide explanations, only the single integer value. | |
| Text to evaluate: | |
| {text} | |
| Scoring Output: | |
| """} | |
| ] | |
| response = client.chat.completions.create( | |
| model="gpt-4o", | |
| messages=messages, | |
| max_tokens=1, | |
| n=1, | |
| stop=None, | |
| temperature=0 | |
| ) | |
| return int(response.choices[0].message.content) | |
| def create_plot(scores_A, scores_B): | |
| labels = ['Benchmark', 'Custom'] | |
| colors = ['#2DD4BF', '#F43F5E'] | |
| fig, ax = plt.subplots() | |
| ax.set_ylabel('Human-like score') | |
| ax.set_ylim([0, 5]) | |
| bplot = ax.boxplot([scores_A, scores_B], | |
| patch_artist=True, | |
| tick_labels=labels) | |
| for patch, color in zip(bplot['boxes'], colors): | |
| patch.set_facecolor(color) | |
| return fig | |
| def run_experiment(questions, custom_template): | |
| results_A = [] | |
| results_B = [] | |
| all_responses = [] | |
| for question in questions: | |
| # Randomly assign group | |
| group = "A" if random.random() < 0.5 else "B" | |
| # Get response | |
| response = run_agent(question, group, custom_template) | |
| # Analyze response | |
| score = analyze_response(response) | |
| # Store results | |
| if group == "A": | |
| results_A.append(score) | |
| else: | |
| results_B.append(score) | |
| all_responses.append({ | |
| "question": question, | |
| "group": "Benchmark" if group == "A" else "Custom", | |
| "response": response, | |
| "score": score | |
| }) | |
| # Create visualization | |
| fig = create_plot(results_A, results_B) | |
| return results_A, results_B, all_responses, fig | |
| def gradio_interface(questions, custom_template): | |
| # Split questions into list | |
| question_list = [q.strip() for q in questions.split('\n') if q.strip()] | |
| # Run experiment | |
| scores_A, scores_B, responses, fig = run_experiment(question_list, custom_template) | |
| # Format detailed results | |
| detailed_results = "" | |
| for r in responses: | |
| detailed_results += f"Question: {r['question']}\n" | |
| detailed_results += f"Template: {r['group']}\n" | |
| detailed_results += f"Response: {r['response']}\n" | |
| detailed_results += f"Score: {r['score']}\n" | |
| detailed_results += "-" * 50 + "\n" | |
| # Calculate averages | |
| avg_A = sum(scores_A) / len(scores_A) if scores_A else 0 | |
| avg_B = sum(scores_B) / len(scores_B) if scores_B else 0 | |
| summary = f""" | |
| Summary: | |
| Benchmark Template - Average Score: {avg_A:.2f} | |
| Custom Template - Average Score: {avg_B:.2f} | |
| Number of responses: | |
| Benchmark Template: {len(scores_A)} | |
| Custom Template: {len(scores_B)} | |
| """ | |
| return fig, summary, detailed_results | |
| # Create Gradio interface | |
| iface = gr.Interface( | |
| fn=gradio_interface, | |
| inputs=[ | |
| gr.Textbox( | |
| lines=5, | |
| placeholder="Enter questions (one per line)...", | |
| label="Questions" | |
| ), | |
| gr.Textbox( | |
| lines=3, | |
| placeholder="Enter your custom template prompt design...", | |
| label="Check How Human Your Template Prompt (different GPTs could have different scores)", | |
| value="You are a helpful assistant that always answers questions. Keep it short." | |
| ) | |
| ], | |
| outputs=[ | |
| gr.Plot(label="Results Visualization"), | |
| gr.Textbox(label="Summary", lines=6), | |
| gr.Textbox(label="Detailed Results", lines=10) | |
| ], | |
| title="A/B Testing Prompt Template Design Analysis", | |
| description="Compare prompt template design of your chatbot against a benchmark human-like template design and analyze your chatbot human-likeness scores.", | |
| examples=[ | |
| ["What should I do when I feel sad?\nWhat do you think about falling in love?\nWhat do you think about getting divorced?\nWhat should I do when I feel happy?", | |
| "You are a helpful assistant that always answers questions. Keep it short and professional."] | |
| ] | |
| ) | |
| if __name__ == "__main__": | |
| iface.launch() | |