improve-eval / app.py
mdkrasnow
Add logging and client init
b2e2264
Raw History Blame Contribute Delete
25.6 kB
import gradio as gr
import random
import json
import uuid
import datetime
import os
from typing import Dict, List, Any, Tuple, Optional
# Import our modules
import data_utils
import elo
# Initialize Argilla and data for first cycle
data_utils.init_argilla()
data_utils.initialize_cycle()
def initialize_state():
"""Initialize the application state"""
return {
"current_stage": 1, # 1 = Individual assessment, 2 = Comparison
"examples_completed": 0,
"current_example_index": 0,
"elo_comparisons_done": 0,
"elo_pair": None,
"current_choice": None # Track the user's current choice (good, fine, bad)
}
def load_example(state):
"""Load the current example for display"""
try:
# Get the current output
current_output = data_utils.all_outputs[state["current_example_index"]]
# Return values in the same order as the outputs parameter in demo.load
return current_output["component_id"], current_output["low_inference_notes"], current_output["output"]["summary"]
except (IndexError, KeyError) as e:
# Handle errors by initializing a new cycle
data_utils.initialize_cycle()
state["current_example_index"] = 0
current_output = data_utils.all_outputs[0]
# Return values in the same order as the outputs parameter in demo.load
return current_output["component_id"], current_output["low_inference_notes"], current_output["output"]["summary"]
def select_choice(choice, state):
"""Set the choice in the state and provide visual feedback"""
state["current_choice"] = choice
print(f"User selected: {choice}")
# Update button states
good_btn_update = gr.update(
variant="primary" if choice == "good" else "secondary",
elem_classes=["good-button", "selected-button"] if choice == "good" else ["good-button"]
)
fine_btn_update = gr.update(
variant="primary" if choice == "fine" else "secondary",
elem_classes=["fine-button", "selected-button"] if choice == "fine" else ["fine-button"]
)
bad_btn_update = gr.update(
variant="primary" if choice == "bad" else "secondary",
elem_classes=["bad-button", "selected-button"] if choice == "bad" else ["bad-button"]
)
log_msg_update = gr.update(value=f"Selected rating: {choice.upper()}", visible=True)
# Return individual updates for each component
return good_btn_update, fine_btn_update, bad_btn_update, log_msg_update, state
def stage1_submit(explanation, state):
"""Process Stage 1 submission (individual assessment)"""
# Validate input
if not state["current_choice"]:
error_msg = gr.update(value="Please select a rating (Good, Fine, or Bad).", visible=True)
log_msg = gr.update(value="Submission failed: No rating selected", visible=True)
# Return empty/unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state
)
if not explanation.strip():
error_msg = gr.update(value="Please provide an explanation for your choice.", visible=True)
log_msg = gr.update(value="Submission failed: No explanation provided", visible=True)
# Return empty/unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state
)
# Get current output
current_output = data_utils.all_outputs[state["current_example_index"]]
# Log the action
print(f"Submitting feedback for output ID: {current_output['id']}")
print(f"Choice: {state['current_choice']}, Explanation: {explanation}")
# Reinitialize Argilla client to ensure fresh connection
data_utils.init_argilla()
# Store feedback in Argilla
try:
success = data_utils.store_feedback(
output_id=current_output["id"],
choice=state["current_choice"],
explanation=explanation,
stage="individual"
)
except Exception as e:
print(f"[ERROR] Exception during store_feedback: {str(e)}")
import traceback
print(f"[ERROR] Traceback: {traceback.format_exc()}")
success = False
if not success:
error_msg = gr.update(value=f"Failed to store feedback in Argilla. Please check logs for details.", visible=True)
log_msg = gr.update(value="Submission failed: Error storing in Argilla", visible=True)
# Return empty/unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state
)
# Increment counter
state["examples_completed"] += 1
# Reset current choice
state["current_choice"] = None
# Determine if we should move to next example or to stage 2
if state["examples_completed"] >= len(data_utils.all_outputs):
# Move to stage 2 (ELO comparisons)
state["current_stage"] = 2
state["elo_comparisons_done"] = 0
# Select first pair for comparison
output_a, output_b = elo.select_elo_pair(data_utils.all_outputs)
# Check if we have a valid pair
if output_a is None or output_b is None:
# Not enough rated outputs, start a new cycle
data_utils.initialize_cycle()
state = initialize_state()
current_output = data_utils.all_outputs[state["current_example_index"]]
return (
gr.update(visible=True), # stage1_group
gr.update(visible=False), # stage2_group
gr.update(value=current_output["component_id"]), # component_id_display
gr.update(value=current_output["low_inference_notes"]), # notes_display
gr.update(value=current_output["output"]["summary"]), # output_display
gr.update(value=""), # explanation_input
gr.update(value="", visible=False), # error_message
gr.update(value="Started new cycle - not enough rated outputs for comparison", visible=True), # log_message
gr.update(variant="secondary"), # good_btn
gr.update(variant="secondary"), # fine_btn
gr.update(variant="secondary"), # bad_btn
state
)
state["elo_pair"] = (output_a, output_b)
return (
gr.update(visible=False), # stage1_group
gr.update(visible=True), # stage2_group
gr.update(), # component_id_display (unchanged)
gr.update(), # notes_display (unchanged)
gr.update(), # output_display (unchanged)
gr.update(value=""), # explanation_input
gr.update(value="", visible=False), # error_message
gr.update(value="Moving to comparison stage", visible=True), # log_message
gr.update(variant="secondary"), # good_btn
gr.update(variant="secondary"), # fine_btn
gr.update(variant="secondary"), # bad_btn
state
)
else:
# Move to next example
state["current_example_index"] = state["examples_completed"]
current_output = data_utils.all_outputs[state["current_example_index"]]
return (
gr.update(visible=True), # stage1_group
gr.update(visible=False), # stage2_group
gr.update(value=current_output["component_id"]), # component_id_display
gr.update(value=current_output["low_inference_notes"]), # notes_display
gr.update(value=current_output["output"]["summary"]), # output_display
gr.update(value=""), # explanation_input
gr.update(value="", visible=False), # error_message
gr.update(value=f"Submission successful! Moving to next example ({state['examples_completed']+1}/{len(data_utils.all_outputs)})", visible=True), # log_message
gr.update(variant="secondary"), # good_btn
gr.update(variant="secondary"), # fine_btn
gr.update(variant="secondary"), # bad_btn
state
)
def select_comparison_choice(choice, state):
"""Set the comparison choice in the state and provide visual feedback"""
state["current_choice"] = choice
print(f"User selected output: {choice}")
# Update button states
choose_a_btn_update = gr.update(
variant="primary" if choice == "a" else "secondary",
elem_classes=["selected-button"] if choice == "a" else []
)
choose_b_btn_update = gr.update(
variant="primary" if choice == "b" else "secondary",
elem_classes=["selected-button"] if choice == "b" else []
)
log_msg_update = gr.update(value=f"Selected output: {choice.upper()}", visible=True)
# Return individual updates for each component
return choose_a_btn_update, choose_b_btn_update, log_msg_update, state
def stage2_submit(explanation, state):
"""Process Stage 2 submission (comparative assessment)"""
# Validate input
if not state["current_choice"]:
error_msg = gr.update(value="Please select which output is better (A or B).", visible=True)
log_msg = gr.update(value="Submission failed: No output selected", visible=True)
# Return unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), gr.update(), gr.update(), gr.update(),
error_msg, log_msg, gr.update(), gr.update(), state
)
if not explanation.strip():
error_msg = gr.update(value="Please provide an explanation for your choice.", visible=True)
log_msg = gr.update(value="Submission failed: No explanation provided", visible=True)
# Return unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), gr.update(), gr.update(), gr.update(),
error_msg, log_msg, gr.update(), gr.update(), state
)
# Get current comparison pair
output_a = state["elo_pair"][0]
output_b = state["elo_pair"][1]
# Determine which output was chosen
a_wins = (state["current_choice"] == "a")
# Log the action
print(f"Comparing outputs: {output_a['id']} vs {output_b['id']}")
print(f"User chose: {'A' if a_wins else 'B'}, Explanation: {explanation}")
# Reinitialize Argilla client to ensure fresh connection
data_utils.init_argilla()
# Update ELO ratings
try:
new_rating_a, new_rating_b = elo.update_ratings(output_a["id"], output_b["id"], a_wins)
except Exception as e:
print(f"[ERROR] Exception during ELO rating update: {str(e)}")
import traceback
print(f"[ERROR] Traceback: {traceback.format_exc()}")
new_rating_a, new_rating_b = None, None
if new_rating_a is None or new_rating_b is None:
error_msg = gr.update(value="Failed to update ELO ratings. Please check logs for details.", visible=True)
log_msg = gr.update(value="Submission failed: Error updating ELO ratings", visible=True)
# Return unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), gr.update(), gr.update(), gr.update(),
error_msg, log_msg, gr.update(), gr.update(), state
)
# Store feedback
try:
success = data_utils.store_feedback(
output_id=output_a["id"] if a_wins else output_b["id"],
choice=f"preferred_over_{output_b['id'] if a_wins else output_a['id']}",
explanation=explanation,
stage="comparison"
)
except Exception as e:
print(f"[ERROR] Exception during store_feedback for comparison: {str(e)}")
import traceback
print(f"[ERROR] Traceback: {traceback.format_exc()}")
success = False
if not success:
error_msg = gr.update(value="Failed to store comparison feedback in Argilla. Please check logs for details.", visible=True)
log_msg = gr.update(value="Submission failed: Error storing comparison in Argilla", visible=True)
# Return unchanged values for all other outputs
return (
gr.update(), gr.update(), gr.update(), gr.update(), gr.update(),
gr.update(), gr.update(), gr.update(), gr.update(),
error_msg, log_msg, gr.update(), gr.update(), state
)
# Reset current choice
state["current_choice"] = None
# Increment counter
state["elo_comparisons_done"] += 1
# Determine if we should continue with more ELO comparisons or restart
if state["elo_comparisons_done"] >= 5: # Do 5 comparisons per cycle
# Restart the process with a new cycle
data_utils.initialize_cycle()
state = initialize_state()
current_output = data_utils.all_outputs[state["current_example_index"]]
return (
gr.update(visible=True), # stage1_group
gr.update(visible=False), # stage2_group
gr.update(value=current_output["component_id"]), # component_id_display
gr.update(value=current_output["low_inference_notes"]), # notes_display
gr.update(value=current_output["output"]["summary"]), # output_display
gr.update(), # notes_display_elo (unchanged)
gr.update(), # output_a (unchanged)
gr.update(), # output_b (unchanged)
gr.update(value=""), # comparison_explanation
gr.update(value="", visible=False), # error_message
gr.update(value="Completed comparison cycle. Starting new evaluation cycle.", visible=True), # log_message
gr.update(variant="secondary"), # choose_a_btn
gr.update(variant="secondary"), # choose_b_btn
state
)
else:
# Get a new comparison pair
output_a, output_b = elo.select_elo_pair(data_utils.all_outputs)
if output_a is None or output_b is None:
# Not enough rated outputs, start a new cycle
data_utils.initialize_cycle()
state = initialize_state()
current_output = data_utils.all_outputs[state["current_example_index"]]
return (
gr.update(visible=True), # stage1_group
gr.update(visible=False), # stage2_group
gr.update(value=current_output["component_id"]), # component_id_display
gr.update(value=current_output["low_inference_notes"]), # notes_display
gr.update(value=current_output["output"]["summary"]), # output_display
gr.update(), # notes_display_elo (unchanged)
gr.update(), # output_a (unchanged)
gr.update(), # output_b (unchanged)
gr.update(value=""), # comparison_explanation
gr.update(value="", visible=False), # error_message
gr.update(value="Not enough rated outputs for comparison. Starting new cycle.", visible=True), # log_message
gr.update(variant="secondary"), # choose_a_btn
gr.update(variant="secondary"), # choose_b_btn
state
)
state["elo_pair"] = (output_a, output_b)
return (
gr.update(visible=False), # stage1_group
gr.update(visible=True), # stage2_group
gr.update(), # component_id_display (unchanged)
gr.update(), # notes_display (unchanged)
gr.update(), # output_display (unchanged)
gr.update(value=output_a["low_inference_notes"]), # notes_display_elo
gr.update(value=output_a["output"]["summary"]), # output_a
gr.update(value=output_b["output"]["summary"]), # output_b
gr.update(value=""), # comparison_explanation
gr.update(value="", visible=False), # error_message
gr.update(value=f"Comparison submitted successfully! Moving to next comparison ({state['elo_comparisons_done']+1}/5)", visible=True), # log_message
gr.update(variant="secondary"), # choose_a_btn
gr.update(variant="secondary"), # choose_b_btn
state
)
def get_leaderboard_display():
"""Get the leaderboard display for top outputs"""
leaderboard = elo.get_leaderboard(top_n=10)
if not leaderboard:
return "No rated outputs yet."
leaderboard_text = "## Top Rated Outputs\n\n"
for i, (output_id, rating) in enumerate(leaderboard):
leaderboard_text += f"{i+1}. Output ID: {output_id[:8]}... - Rating: {rating:.1f}\n"
return leaderboard_text
# Create Gradio interface
with gr.Blocks(theme=gr.themes.Soft(
primary_hue="indigo",
secondary_hue="blue",
neutral_hue="slate",
radius_size="lg",
font=["Inter", "ui-sans-serif", "system-ui", "sans-serif"]
),
css="""
.scrollable-notes {
max-height: 300px;
overflow-y: auto;
border: 1px solid #ddd;
padding: 10px;
border-radius: 5px;
}
.good-button {
background-color: #4CAF50 !important;
color: white !important;
}
.fine-button {
background-color: #FFC107 !important;
color: black !important;
}
.bad-button {
background-color: #F44336 !important;
color: white !important;
}
.selected-button {
opacity: 0.8;
box-shadow: 0 0 5px rgba(0,0,0,0.5);
}
.log-message {
padding: 8px;
margin-top: 10px;
border-radius: 4px;
background-color: #f0f0f0;
font-size: 14px;
}
""") as demo:
# Initialize state
state = gr.State(initialize_state())
# Error message area
error_message = gr.Markdown(visible=False)
log_message = gr.Markdown(visible=False, elem_classes=["log-message"])
with gr.Tabs():
with gr.TabItem("Data Labeling"):
# Stage 1: Individual Assessment
with gr.Group(visible=True) as stage1_group:
gr.Markdown("# Output Evaluation")
gr.Markdown("### Please rate the quality of this output")
# Component ID display
component_id_display = gr.Markdown("### Component ID: Loading...")
with gr.Row():
# Left column for notes - make it scrollable and smaller
with gr.Column(scale=1):
gr.Markdown("### Low Inference Notes")
notes_display = gr.Markdown(elem_classes=["scrollable-notes"])
# Right column for output and controls
with gr.Column(scale=2):
output_display = gr.Textbox(
label="AI Generated Output",
lines=8,
max_lines=15,
interactive=False
)
# Rating buttons in a row with colors
with gr.Row():
good_btn = gr.Button(
"GOOD - Output is excellent",
variant="secondary",
elem_classes=["good-button"]
)
fine_btn = gr.Button(
"FINE - Output needs minor improvements",
variant="secondary",
elem_classes=["fine-button"]
)
bad_btn = gr.Button(
"BAD - Output needs major improvements",
variant="secondary",
elem_classes=["bad-button"]
)
# Explanation input
explanation_input = gr.Textbox(
label="Please explain your choice in one sentence",
lines=2,
placeholder="Explain why you selected this rating..."
)
# Submit button
submit_btn = gr.Button("Submit and Continue", variant="primary", size="lg")
# Stage 2: Comparative Assessment
with gr.Group(visible=False) as stage2_group:
gr.Markdown("# Compare Outputs")
gr.Markdown("### Which of these two outputs is better?")
# Make notes scrollable here too
gr.Markdown("### Low Inference Notes (Same for both outputs)")
notes_display_elo = gr.Markdown(elem_classes=["scrollable-notes"])
with gr.Row():
with gr.Column():
output_a = gr.Textbox(
label="Output A",
lines=8,
interactive=False
)
choose_a_btn = gr.Button("Output A is better", variant="secondary")
with gr.Column():
output_b = gr.Textbox(
label="Output B",
lines=8,
interactive=False
)
choose_b_btn = gr.Button("Output B is better", variant="secondary")
comparison_explanation = gr.Textbox(
label="Please explain why you chose this output",
lines=2,
placeholder="Explain your preference in one sentence..."
)
# Submit button for comparison
compare_submit_btn = gr.Button("Submit and Continue", variant="primary", size="lg")
with gr.TabItem("Leaderboard"):
leaderboard_display = gr.Markdown()
refresh_btn = gr.Button("Refresh Leaderboard")
# Load initial data
demo.load(
fn=load_example,
inputs=[state],
outputs=[component_id_display, notes_display, output_display]
)
# Also load the leaderboard on startup
demo.load(
fn=get_leaderboard_display,
inputs=[],
outputs=[leaderboard_display]
)
# Connect the buttons to their functions
good_btn.click(
fn=lambda state: select_choice("good", state),
inputs=[state],
outputs=[good_btn, fine_btn, bad_btn, log_message, state]
)
fine_btn.click(
fn=lambda state: select_choice("fine", state),
inputs=[state],
outputs=[good_btn, fine_btn, bad_btn, log_message, state]
)
bad_btn.click(
fn=lambda state: select_choice("bad", state),
inputs=[state],
outputs=[good_btn, fine_btn, bad_btn, log_message, state]
)
submit_btn.click(
fn=stage1_submit,
inputs=[explanation_input, state],
outputs=[
stage1_group, stage2_group, component_id_display, notes_display, output_display,
explanation_input, error_message, log_message, good_btn, fine_btn, bad_btn, state
]
)
choose_a_btn.click(
fn=lambda state: select_comparison_choice("a", state),
inputs=[state],
outputs=[choose_a_btn, choose_b_btn, log_message, state]
)
choose_b_btn.click(
fn=lambda state: select_comparison_choice("b", state),
inputs=[state],
outputs=[choose_a_btn, choose_b_btn, log_message, state]
)
compare_submit_btn.click(
fn=stage2_submit,
inputs=[comparison_explanation, state],
outputs=[
stage1_group, stage2_group, component_id_display, notes_display, output_display,
notes_display_elo, output_a, output_b, comparison_explanation,
error_message, log_message, choose_a_btn, choose_b_btn, state
]
)
refresh_btn.click(
fn=get_leaderboard_display,
inputs=[],
outputs=[leaderboard_display]
)
# For Hugging Face Spaces deployment
if __name__ == "__main__":
# Initialize Argilla on startup
data_utils.init_argilla()
# Launch the app
demo.launch()