import gradio as gr import random import json import uuid import datetime import os from typing import Dict, List, Any, Tuple, Optional # Import our modules import data_utils import elo # Initialize Argilla and data for first cycle data_utils.init_argilla() data_utils.initialize_cycle() def initialize_state(): """Initialize the application state""" return { "current_stage": 1, # 1 = Individual assessment, 2 = Comparison "examples_completed": 0, "current_example_index": 0, "elo_comparisons_done": 0, "elo_pair": None, "current_choice": None # Track the user's current choice (good, fine, bad) } def load_example(state): """Load the current example for display""" try: # Get the current output current_output = data_utils.all_outputs[state["current_example_index"]] # Return values in the same order as the outputs parameter in demo.load return current_output["component_id"], current_output["low_inference_notes"], current_output["output"]["summary"] except (IndexError, KeyError) as e: # Handle errors by initializing a new cycle data_utils.initialize_cycle() state["current_example_index"] = 0 current_output = data_utils.all_outputs[0] # Return values in the same order as the outputs parameter in demo.load return current_output["component_id"], current_output["low_inference_notes"], current_output["output"]["summary"] def select_choice(choice, state): """Set the choice in the state and provide visual feedback""" state["current_choice"] = choice print(f"User selected: {choice}") # Update button states good_btn_update = gr.update( variant="primary" if choice == "good" else "secondary", elem_classes=["good-button", "selected-button"] if choice == "good" else ["good-button"] ) fine_btn_update = gr.update( variant="primary" if choice == "fine" else "secondary", elem_classes=["fine-button", "selected-button"] if choice == "fine" else ["fine-button"] ) bad_btn_update = gr.update( variant="primary" if choice == "bad" else "secondary", elem_classes=["bad-button", "selected-button"] if choice == "bad" else ["bad-button"] ) log_msg_update = gr.update(value=f"Selected rating: {choice.upper()}", visible=True) # Return individual updates for each component return good_btn_update, fine_btn_update, bad_btn_update, log_msg_update, state def stage1_submit(explanation, state): """Process Stage 1 submission (individual assessment)""" # Validate input if not state["current_choice"]: error_msg = gr.update(value="Please select a rating (Good, Fine, or Bad).", visible=True) log_msg = gr.update(value="Submission failed: No rating selected", visible=True) # Return empty/unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state ) if not explanation.strip(): error_msg = gr.update(value="Please provide an explanation for your choice.", visible=True) log_msg = gr.update(value="Submission failed: No explanation provided", visible=True) # Return empty/unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state ) # Get current output current_output = data_utils.all_outputs[state["current_example_index"]] # Log the action print(f"Submitting feedback for output ID: {current_output['id']}") print(f"Choice: {state['current_choice']}, Explanation: {explanation}") # Reinitialize Argilla client to ensure fresh connection data_utils.init_argilla() # Store feedback in Argilla try: success = data_utils.store_feedback( output_id=current_output["id"], choice=state["current_choice"], explanation=explanation, stage="individual" ) except Exception as e: print(f"[ERROR] Exception during store_feedback: {str(e)}") import traceback print(f"[ERROR] Traceback: {traceback.format_exc()}") success = False if not success: error_msg = gr.update(value=f"Failed to store feedback in Argilla. Please check logs for details.", visible=True) log_msg = gr.update(value="Submission failed: Error storing in Argilla", visible=True) # Return empty/unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), gr.update(), state ) # Increment counter state["examples_completed"] += 1 # Reset current choice state["current_choice"] = None # Determine if we should move to next example or to stage 2 if state["examples_completed"] >= len(data_utils.all_outputs): # Move to stage 2 (ELO comparisons) state["current_stage"] = 2 state["elo_comparisons_done"] = 0 # Select first pair for comparison output_a, output_b = elo.select_elo_pair(data_utils.all_outputs) # Check if we have a valid pair if output_a is None or output_b is None: # Not enough rated outputs, start a new cycle data_utils.initialize_cycle() state = initialize_state() current_output = data_utils.all_outputs[state["current_example_index"]] return ( gr.update(visible=True), # stage1_group gr.update(visible=False), # stage2_group gr.update(value=current_output["component_id"]), # component_id_display gr.update(value=current_output["low_inference_notes"]), # notes_display gr.update(value=current_output["output"]["summary"]), # output_display gr.update(value=""), # explanation_input gr.update(value="", visible=False), # error_message gr.update(value="Started new cycle - not enough rated outputs for comparison", visible=True), # log_message gr.update(variant="secondary"), # good_btn gr.update(variant="secondary"), # fine_btn gr.update(variant="secondary"), # bad_btn state ) state["elo_pair"] = (output_a, output_b) return ( gr.update(visible=False), # stage1_group gr.update(visible=True), # stage2_group gr.update(), # component_id_display (unchanged) gr.update(), # notes_display (unchanged) gr.update(), # output_display (unchanged) gr.update(value=""), # explanation_input gr.update(value="", visible=False), # error_message gr.update(value="Moving to comparison stage", visible=True), # log_message gr.update(variant="secondary"), # good_btn gr.update(variant="secondary"), # fine_btn gr.update(variant="secondary"), # bad_btn state ) else: # Move to next example state["current_example_index"] = state["examples_completed"] current_output = data_utils.all_outputs[state["current_example_index"]] return ( gr.update(visible=True), # stage1_group gr.update(visible=False), # stage2_group gr.update(value=current_output["component_id"]), # component_id_display gr.update(value=current_output["low_inference_notes"]), # notes_display gr.update(value=current_output["output"]["summary"]), # output_display gr.update(value=""), # explanation_input gr.update(value="", visible=False), # error_message gr.update(value=f"Submission successful! Moving to next example ({state['examples_completed']+1}/{len(data_utils.all_outputs)})", visible=True), # log_message gr.update(variant="secondary"), # good_btn gr.update(variant="secondary"), # fine_btn gr.update(variant="secondary"), # bad_btn state ) def select_comparison_choice(choice, state): """Set the comparison choice in the state and provide visual feedback""" state["current_choice"] = choice print(f"User selected output: {choice}") # Update button states choose_a_btn_update = gr.update( variant="primary" if choice == "a" else "secondary", elem_classes=["selected-button"] if choice == "a" else [] ) choose_b_btn_update = gr.update( variant="primary" if choice == "b" else "secondary", elem_classes=["selected-button"] if choice == "b" else [] ) log_msg_update = gr.update(value=f"Selected output: {choice.upper()}", visible=True) # Return individual updates for each component return choose_a_btn_update, choose_b_btn_update, log_msg_update, state def stage2_submit(explanation, state): """Process Stage 2 submission (comparative assessment)""" # Validate input if not state["current_choice"]: error_msg = gr.update(value="Please select which output is better (A or B).", visible=True) log_msg = gr.update(value="Submission failed: No output selected", visible=True) # Return unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), state ) if not explanation.strip(): error_msg = gr.update(value="Please provide an explanation for your choice.", visible=True) log_msg = gr.update(value="Submission failed: No explanation provided", visible=True) # Return unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), state ) # Get current comparison pair output_a = state["elo_pair"][0] output_b = state["elo_pair"][1] # Determine which output was chosen a_wins = (state["current_choice"] == "a") # Log the action print(f"Comparing outputs: {output_a['id']} vs {output_b['id']}") print(f"User chose: {'A' if a_wins else 'B'}, Explanation: {explanation}") # Reinitialize Argilla client to ensure fresh connection data_utils.init_argilla() # Update ELO ratings try: new_rating_a, new_rating_b = elo.update_ratings(output_a["id"], output_b["id"], a_wins) except Exception as e: print(f"[ERROR] Exception during ELO rating update: {str(e)}") import traceback print(f"[ERROR] Traceback: {traceback.format_exc()}") new_rating_a, new_rating_b = None, None if new_rating_a is None or new_rating_b is None: error_msg = gr.update(value="Failed to update ELO ratings. Please check logs for details.", visible=True) log_msg = gr.update(value="Submission failed: Error updating ELO ratings", visible=True) # Return unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), state ) # Store feedback try: success = data_utils.store_feedback( output_id=output_a["id"] if a_wins else output_b["id"], choice=f"preferred_over_{output_b['id'] if a_wins else output_a['id']}", explanation=explanation, stage="comparison" ) except Exception as e: print(f"[ERROR] Exception during store_feedback for comparison: {str(e)}") import traceback print(f"[ERROR] Traceback: {traceback.format_exc()}") success = False if not success: error_msg = gr.update(value="Failed to store comparison feedback in Argilla. Please check logs for details.", visible=True) log_msg = gr.update(value="Submission failed: Error storing comparison in Argilla", visible=True) # Return unchanged values for all other outputs return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), error_msg, log_msg, gr.update(), gr.update(), state ) # Reset current choice state["current_choice"] = None # Increment counter state["elo_comparisons_done"] += 1 # Determine if we should continue with more ELO comparisons or restart if state["elo_comparisons_done"] >= 5: # Do 5 comparisons per cycle # Restart the process with a new cycle data_utils.initialize_cycle() state = initialize_state() current_output = data_utils.all_outputs[state["current_example_index"]] return ( gr.update(visible=True), # stage1_group gr.update(visible=False), # stage2_group gr.update(value=current_output["component_id"]), # component_id_display gr.update(value=current_output["low_inference_notes"]), # notes_display gr.update(value=current_output["output"]["summary"]), # output_display gr.update(), # notes_display_elo (unchanged) gr.update(), # output_a (unchanged) gr.update(), # output_b (unchanged) gr.update(value=""), # comparison_explanation gr.update(value="", visible=False), # error_message gr.update(value="Completed comparison cycle. Starting new evaluation cycle.", visible=True), # log_message gr.update(variant="secondary"), # choose_a_btn gr.update(variant="secondary"), # choose_b_btn state ) else: # Get a new comparison pair output_a, output_b = elo.select_elo_pair(data_utils.all_outputs) if output_a is None or output_b is None: # Not enough rated outputs, start a new cycle data_utils.initialize_cycle() state = initialize_state() current_output = data_utils.all_outputs[state["current_example_index"]] return ( gr.update(visible=True), # stage1_group gr.update(visible=False), # stage2_group gr.update(value=current_output["component_id"]), # component_id_display gr.update(value=current_output["low_inference_notes"]), # notes_display gr.update(value=current_output["output"]["summary"]), # output_display gr.update(), # notes_display_elo (unchanged) gr.update(), # output_a (unchanged) gr.update(), # output_b (unchanged) gr.update(value=""), # comparison_explanation gr.update(value="", visible=False), # error_message gr.update(value="Not enough rated outputs for comparison. Starting new cycle.", visible=True), # log_message gr.update(variant="secondary"), # choose_a_btn gr.update(variant="secondary"), # choose_b_btn state ) state["elo_pair"] = (output_a, output_b) return ( gr.update(visible=False), # stage1_group gr.update(visible=True), # stage2_group gr.update(), # component_id_display (unchanged) gr.update(), # notes_display (unchanged) gr.update(), # output_display (unchanged) gr.update(value=output_a["low_inference_notes"]), # notes_display_elo gr.update(value=output_a["output"]["summary"]), # output_a gr.update(value=output_b["output"]["summary"]), # output_b gr.update(value=""), # comparison_explanation gr.update(value="", visible=False), # error_message gr.update(value=f"Comparison submitted successfully! Moving to next comparison ({state['elo_comparisons_done']+1}/5)", visible=True), # log_message gr.update(variant="secondary"), # choose_a_btn gr.update(variant="secondary"), # choose_b_btn state ) def get_leaderboard_display(): """Get the leaderboard display for top outputs""" leaderboard = elo.get_leaderboard(top_n=10) if not leaderboard: return "No rated outputs yet." leaderboard_text = "## Top Rated Outputs\n\n" for i, (output_id, rating) in enumerate(leaderboard): leaderboard_text += f"{i+1}. Output ID: {output_id[:8]}... - Rating: {rating:.1f}\n" return leaderboard_text # Create Gradio interface with gr.Blocks(theme=gr.themes.Soft( primary_hue="indigo", secondary_hue="blue", neutral_hue="slate", radius_size="lg", font=["Inter", "ui-sans-serif", "system-ui", "sans-serif"] ), css=""" .scrollable-notes { max-height: 300px; overflow-y: auto; border: 1px solid #ddd; padding: 10px; border-radius: 5px; } .good-button { background-color: #4CAF50 !important; color: white !important; } .fine-button { background-color: #FFC107 !important; color: black !important; } .bad-button { background-color: #F44336 !important; color: white !important; } .selected-button { opacity: 0.8; box-shadow: 0 0 5px rgba(0,0,0,0.5); } .log-message { padding: 8px; margin-top: 10px; border-radius: 4px; background-color: #f0f0f0; font-size: 14px; } """) as demo: # Initialize state state = gr.State(initialize_state()) # Error message area error_message = gr.Markdown(visible=False) log_message = gr.Markdown(visible=False, elem_classes=["log-message"]) with gr.Tabs(): with gr.TabItem("Data Labeling"): # Stage 1: Individual Assessment with gr.Group(visible=True) as stage1_group: gr.Markdown("# Output Evaluation") gr.Markdown("### Please rate the quality of this output") # Component ID display component_id_display = gr.Markdown("### Component ID: Loading...") with gr.Row(): # Left column for notes - make it scrollable and smaller with gr.Column(scale=1): gr.Markdown("### Low Inference Notes") notes_display = gr.Markdown(elem_classes=["scrollable-notes"]) # Right column for output and controls with gr.Column(scale=2): output_display = gr.Textbox( label="AI Generated Output", lines=8, max_lines=15, interactive=False ) # Rating buttons in a row with colors with gr.Row(): good_btn = gr.Button( "GOOD - Output is excellent", variant="secondary", elem_classes=["good-button"] ) fine_btn = gr.Button( "FINE - Output needs minor improvements", variant="secondary", elem_classes=["fine-button"] ) bad_btn = gr.Button( "BAD - Output needs major improvements", variant="secondary", elem_classes=["bad-button"] ) # Explanation input explanation_input = gr.Textbox( label="Please explain your choice in one sentence", lines=2, placeholder="Explain why you selected this rating..." ) # Submit button submit_btn = gr.Button("Submit and Continue", variant="primary", size="lg") # Stage 2: Comparative Assessment with gr.Group(visible=False) as stage2_group: gr.Markdown("# Compare Outputs") gr.Markdown("### Which of these two outputs is better?") # Make notes scrollable here too gr.Markdown("### Low Inference Notes (Same for both outputs)") notes_display_elo = gr.Markdown(elem_classes=["scrollable-notes"]) with gr.Row(): with gr.Column(): output_a = gr.Textbox( label="Output A", lines=8, interactive=False ) choose_a_btn = gr.Button("Output A is better", variant="secondary") with gr.Column(): output_b = gr.Textbox( label="Output B", lines=8, interactive=False ) choose_b_btn = gr.Button("Output B is better", variant="secondary") comparison_explanation = gr.Textbox( label="Please explain why you chose this output", lines=2, placeholder="Explain your preference in one sentence..." ) # Submit button for comparison compare_submit_btn = gr.Button("Submit and Continue", variant="primary", size="lg") with gr.TabItem("Leaderboard"): leaderboard_display = gr.Markdown() refresh_btn = gr.Button("Refresh Leaderboard") # Load initial data demo.load( fn=load_example, inputs=[state], outputs=[component_id_display, notes_display, output_display] ) # Also load the leaderboard on startup demo.load( fn=get_leaderboard_display, inputs=[], outputs=[leaderboard_display] ) # Connect the buttons to their functions good_btn.click( fn=lambda state: select_choice("good", state), inputs=[state], outputs=[good_btn, fine_btn, bad_btn, log_message, state] ) fine_btn.click( fn=lambda state: select_choice("fine", state), inputs=[state], outputs=[good_btn, fine_btn, bad_btn, log_message, state] ) bad_btn.click( fn=lambda state: select_choice("bad", state), inputs=[state], outputs=[good_btn, fine_btn, bad_btn, log_message, state] ) submit_btn.click( fn=stage1_submit, inputs=[explanation_input, state], outputs=[ stage1_group, stage2_group, component_id_display, notes_display, output_display, explanation_input, error_message, log_message, good_btn, fine_btn, bad_btn, state ] ) choose_a_btn.click( fn=lambda state: select_comparison_choice("a", state), inputs=[state], outputs=[choose_a_btn, choose_b_btn, log_message, state] ) choose_b_btn.click( fn=lambda state: select_comparison_choice("b", state), inputs=[state], outputs=[choose_a_btn, choose_b_btn, log_message, state] ) compare_submit_btn.click( fn=stage2_submit, inputs=[comparison_explanation, state], outputs=[ stage1_group, stage2_group, component_id_display, notes_display, output_display, notes_display_elo, output_a, output_b, comparison_explanation, error_message, log_message, choose_a_btn, choose_b_btn, state ] ) refresh_btn.click( fn=get_leaderboard_display, inputs=[], outputs=[leaderboard_display] ) # For Hugging Face Spaces deployment if __name__ == "__main__": # Initialize Argilla on startup data_utils.init_argilla() # Launch the app demo.launch()