Anmol2507's picture
Update app.py
b9f0050 verified
Raw History Blame Contribute Delete
8.39 kB
import spaces
import torch
import gradio as gr
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel
# ============================================================
# CONFIGURATION
# ============================================================
BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct"
ADAPTER_MODEL = "Anmol2507/LLM_as_judge_fine_tuned_Qwen_7B"
MAX_LENGTH = 768
MAX_NEW_TOKENS = 32
# ============================================================
# LOAD TOKENIZER
# ============================================================
print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(
BASE_MODEL,
trust_remote_code=True
)
print("Tokenizer loaded.")
# ============================================================
# LOAD BASE MODEL
# ============================================================
# Load on CPU first because ZeroGPU does not provide a real
# CUDA device during application startup.
# ============================================================
print("Loading base model...")
base_model = AutoModelForCausalLM.from_pretrained(
BASE_MODEL,
dtype=torch.bfloat16,
trust_remote_code=True
)
base_model.eval()
print("Base model loaded.")
# ============================================================
# LOAD LoRA ADAPTER
# ============================================================
# torch_device="cpu" is important.
# It prevents PEFT from trying to load adapter weights onto
# CUDA before ZeroGPU allocates a GPU.
# ============================================================
print("Loading LoRA adapter...")
model = PeftModel.from_pretrained(
base_model,
ADAPTER_MODEL,
torch_device="cpu"
)
model.eval()
print("LoRA adapter loaded.")
# ============================================================
# REGISTER MODEL WITH ZEROGPU
# ============================================================
# Keep this at module level.
# ZeroGPU intercepts this CUDA placement.
# ============================================================
print("Registering model with ZeroGPU...")
model = model.to("cuda")
model.eval()
print("Model registered with ZeroGPU.")
# ============================================================
# BUILD JUDGE PROMPT
# ============================================================
def build_prompt(problem, answer_a, answer_b):
messages = [
{
"role": "system",
"content": (
"You are an expert AI judge evaluating two answers "
"to the same programming or technical problem.\n\n"
"Determine whether each answer is correct.\n"
"Focus primarily on correctness, logical validity, "
"and whether the answer actually solves the problem.\n\n"
"Return ONLY valid JSON using exactly this format:\n"
'{"A":"correct","B":"incorrect"}\n\n'
"Allowed labels are only:\n"
'"correct"\n'
'"incorrect"\n\n'
"If both answers are correct, mark both as correct.\n"
"If both answers are incorrect, mark both as incorrect."
)
},
{
"role": "user",
"content": (
f"Problem:\n{problem}\n\n"
f"Answer A:\n{answer_a}\n\n"
f"Answer B:\n{answer_b}\n\n"
"Evaluate both answers."
)
}
]
return tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
# ============================================================
# JUDGE / INFERENCE FUNCTION
# ============================================================
@spaces.GPU(duration=60)
def judge(problem, answer_a, answer_b):
# --------------------------------------------------------
# Validate inputs
# --------------------------------------------------------
if not problem or not problem.strip():
return '{"error":"Problem is required."}'
if not answer_a or not answer_a.strip():
return '{"error":"Answer A is required."}'
if not answer_b or not answer_b.strip():
return '{"error":"Answer B is required."}'
# --------------------------------------------------------
# Create prompt
# --------------------------------------------------------
prompt = build_prompt(
problem,
answer_a,
answer_b
)
# --------------------------------------------------------
# Tokenize
# --------------------------------------------------------
inputs = tokenizer(
prompt,
return_tensors="pt",
truncation=True,
max_length=MAX_LENGTH
)
# --------------------------------------------------------
# Move input tensors to GPU
# --------------------------------------------------------
inputs = {
key: value.to("cuda")
for key, value in inputs.items()
}
# --------------------------------------------------------
# Generate response
# --------------------------------------------------------
with torch.inference_mode():
outputs = model.generate(
**inputs,
max_new_tokens=MAX_NEW_TOKENS,
do_sample=False,
pad_token_id=tokenizer.eos_token_id
)
# --------------------------------------------------------
# Remove prompt tokens
# --------------------------------------------------------
input_length = inputs["input_ids"].shape[1]
generated_tokens = outputs[0][input_length:]
# --------------------------------------------------------
# Decode model response
# --------------------------------------------------------
response = tokenizer.decode(
generated_tokens,
skip_special_tokens=True
).strip()
return response
# ============================================================
# GRADIO APPLICATION
# ============================================================
with gr.Blocks(
title="LLM as Judge"
) as demo:
# --------------------------------------------------------
# Header
# --------------------------------------------------------
gr.Markdown(
"""
# 🧑‍⚖️ LLM as Judge
### Fine-tuned Qwen2.5-7B Judge
Evaluate two answers to the same programming or
technical problem.
"""
)
# --------------------------------------------------------
# Problem
# --------------------------------------------------------
problem = gr.Textbox(
label="Problem",
placeholder=(
"Enter the programming or technical problem..."
),
lines=6
)
# --------------------------------------------------------
# Answers
# --------------------------------------------------------
with gr.Row():
answer_a = gr.Textbox(
label="Answer A",
placeholder="Enter Answer A...",
lines=12
)
answer_b = gr.Textbox(
label="Answer B",
placeholder="Enter Answer B...",
lines=12
)
# --------------------------------------------------------
# Judge button
# --------------------------------------------------------
judge_button = gr.Button(
"⚖️ Evaluate",
variant="primary"
)
# --------------------------------------------------------
# Result
# --------------------------------------------------------
result = gr.Code(
label="Judge Result",
language="json"
)
# --------------------------------------------------------
# API EVENT
# --------------------------------------------------------
# IMPORTANT:
# This must remain INSIDE the Blocks context.
# api_name="judge" exposes the endpoint as /judge.
# --------------------------------------------------------
judge_button.click(
fn=judge,
inputs=[
problem,
answer_a,
answer_b
],
outputs=result,
api_name="judge"
)
# ============================================================
# START APPLICATION
# ============================================================
if __name__ == "__main__":
demo.launch()