import spaces import torch import gradio as gr from transformers import AutoTokenizer, AutoModelForCausalLM from peft import PeftModel # ============================================================ # CONFIGURATION # ============================================================ BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct" ADAPTER_MODEL = "Anmol2507/LLM_as_judge_fine_tuned_Qwen_7B" MAX_LENGTH = 768 MAX_NEW_TOKENS = 32 # ============================================================ # LOAD TOKENIZER # ============================================================ print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained( BASE_MODEL, trust_remote_code=True ) print("Tokenizer loaded.") # ============================================================ # LOAD BASE MODEL # ============================================================ # Load on CPU first because ZeroGPU does not provide a real # CUDA device during application startup. # ============================================================ print("Loading base model...") base_model = AutoModelForCausalLM.from_pretrained( BASE_MODEL, dtype=torch.bfloat16, trust_remote_code=True ) base_model.eval() print("Base model loaded.") # ============================================================ # LOAD LoRA ADAPTER # ============================================================ # torch_device="cpu" is important. # It prevents PEFT from trying to load adapter weights onto # CUDA before ZeroGPU allocates a GPU. # ============================================================ print("Loading LoRA adapter...") model = PeftModel.from_pretrained( base_model, ADAPTER_MODEL, torch_device="cpu" ) model.eval() print("LoRA adapter loaded.") # ============================================================ # REGISTER MODEL WITH ZEROGPU # ============================================================ # Keep this at module level. # ZeroGPU intercepts this CUDA placement. # ============================================================ print("Registering model with ZeroGPU...") model = model.to("cuda") model.eval() print("Model registered with ZeroGPU.") # ============================================================ # BUILD JUDGE PROMPT # ============================================================ def build_prompt(problem, answer_a, answer_b): messages = [ { "role": "system", "content": ( "You are an expert AI judge evaluating two answers " "to the same programming or technical problem.\n\n" "Determine whether each answer is correct.\n" "Focus primarily on correctness, logical validity, " "and whether the answer actually solves the problem.\n\n" "Return ONLY valid JSON using exactly this format:\n" '{"A":"correct","B":"incorrect"}\n\n' "Allowed labels are only:\n" '"correct"\n' '"incorrect"\n\n' "If both answers are correct, mark both as correct.\n" "If both answers are incorrect, mark both as incorrect." ) }, { "role": "user", "content": ( f"Problem:\n{problem}\n\n" f"Answer A:\n{answer_a}\n\n" f"Answer B:\n{answer_b}\n\n" "Evaluate both answers." ) } ] return tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) # ============================================================ # JUDGE / INFERENCE FUNCTION # ============================================================ @spaces.GPU(duration=60) def judge(problem, answer_a, answer_b): # -------------------------------------------------------- # Validate inputs # -------------------------------------------------------- if not problem or not problem.strip(): return '{"error":"Problem is required."}' if not answer_a or not answer_a.strip(): return '{"error":"Answer A is required."}' if not answer_b or not answer_b.strip(): return '{"error":"Answer B is required."}' # -------------------------------------------------------- # Create prompt # -------------------------------------------------------- prompt = build_prompt( problem, answer_a, answer_b ) # -------------------------------------------------------- # Tokenize # -------------------------------------------------------- inputs = tokenizer( prompt, return_tensors="pt", truncation=True, max_length=MAX_LENGTH ) # -------------------------------------------------------- # Move input tensors to GPU # -------------------------------------------------------- inputs = { key: value.to("cuda") for key, value in inputs.items() } # -------------------------------------------------------- # Generate response # -------------------------------------------------------- with torch.inference_mode(): outputs = model.generate( **inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=False, pad_token_id=tokenizer.eos_token_id ) # -------------------------------------------------------- # Remove prompt tokens # -------------------------------------------------------- input_length = inputs["input_ids"].shape[1] generated_tokens = outputs[0][input_length:] # -------------------------------------------------------- # Decode model response # -------------------------------------------------------- response = tokenizer.decode( generated_tokens, skip_special_tokens=True ).strip() return response # ============================================================ # GRADIO APPLICATION # ============================================================ with gr.Blocks( title="LLM as Judge" ) as demo: # -------------------------------------------------------- # Header # -------------------------------------------------------- gr.Markdown( """ # 🧑‍⚖️ LLM as Judge ### Fine-tuned Qwen2.5-7B Judge Evaluate two answers to the same programming or technical problem. """ ) # -------------------------------------------------------- # Problem # -------------------------------------------------------- problem = gr.Textbox( label="Problem", placeholder=( "Enter the programming or technical problem..." ), lines=6 ) # -------------------------------------------------------- # Answers # -------------------------------------------------------- with gr.Row(): answer_a = gr.Textbox( label="Answer A", placeholder="Enter Answer A...", lines=12 ) answer_b = gr.Textbox( label="Answer B", placeholder="Enter Answer B...", lines=12 ) # -------------------------------------------------------- # Judge button # -------------------------------------------------------- judge_button = gr.Button( "⚖️ Evaluate", variant="primary" ) # -------------------------------------------------------- # Result # -------------------------------------------------------- result = gr.Code( label="Judge Result", language="json" ) # -------------------------------------------------------- # API EVENT # -------------------------------------------------------- # IMPORTANT: # This must remain INSIDE the Blocks context. # api_name="judge" exposes the endpoint as /judge. # -------------------------------------------------------- judge_button.click( fn=judge, inputs=[ problem, answer_a, answer_b ], outputs=result, api_name="judge" ) # ============================================================ # START APPLICATION # ============================================================ if __name__ == "__main__": demo.launch()