Spaces:
Running on Zero
Running on Zero
Download app.py from Anmol2507/LLM_as_Judge_API: direct link, hf CLI and curl.
- Browser
- Download file 8.39 kB
-
https://huggingface.co/spaces/Anmol2507/LLM_as_Judge_API/resolve/main/app.py
- Command line
-
hf download hf://spaces/Anmol2507/LLM_as_Judge_API/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Anmol2507/LLM_as_Judge_API/resolve/main/app.py
8.39 kB
| import spaces | |
| import torch | |
| import gradio as gr | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| from peft import PeftModel | |
| # ============================================================ | |
| # CONFIGURATION | |
| # ============================================================ | |
| BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct" | |
| ADAPTER_MODEL = "Anmol2507/LLM_as_judge_fine_tuned_Qwen_7B" | |
| MAX_LENGTH = 768 | |
| MAX_NEW_TOKENS = 32 | |
| # ============================================================ | |
| # LOAD TOKENIZER | |
| # ============================================================ | |
| print("Loading tokenizer...") | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| BASE_MODEL, | |
| trust_remote_code=True | |
| ) | |
| print("Tokenizer loaded.") | |
| # ============================================================ | |
| # LOAD BASE MODEL | |
| # ============================================================ | |
| # Load on CPU first because ZeroGPU does not provide a real | |
| # CUDA device during application startup. | |
| # ============================================================ | |
| print("Loading base model...") | |
| base_model = AutoModelForCausalLM.from_pretrained( | |
| BASE_MODEL, | |
| dtype=torch.bfloat16, | |
| trust_remote_code=True | |
| ) | |
| base_model.eval() | |
| print("Base model loaded.") | |
| # ============================================================ | |
| # LOAD LoRA ADAPTER | |
| # ============================================================ | |
| # torch_device="cpu" is important. | |
| # It prevents PEFT from trying to load adapter weights onto | |
| # CUDA before ZeroGPU allocates a GPU. | |
| # ============================================================ | |
| print("Loading LoRA adapter...") | |
| model = PeftModel.from_pretrained( | |
| base_model, | |
| ADAPTER_MODEL, | |
| torch_device="cpu" | |
| ) | |
| model.eval() | |
| print("LoRA adapter loaded.") | |
| # ============================================================ | |
| # REGISTER MODEL WITH ZEROGPU | |
| # ============================================================ | |
| # Keep this at module level. | |
| # ZeroGPU intercepts this CUDA placement. | |
| # ============================================================ | |
| print("Registering model with ZeroGPU...") | |
| model = model.to("cuda") | |
| model.eval() | |
| print("Model registered with ZeroGPU.") | |
| # ============================================================ | |
| # BUILD JUDGE PROMPT | |
| # ============================================================ | |
| def build_prompt(problem, answer_a, answer_b): | |
| messages = [ | |
| { | |
| "role": "system", | |
| "content": ( | |
| "You are an expert AI judge evaluating two answers " | |
| "to the same programming or technical problem.\n\n" | |
| "Determine whether each answer is correct.\n" | |
| "Focus primarily on correctness, logical validity, " | |
| "and whether the answer actually solves the problem.\n\n" | |
| "Return ONLY valid JSON using exactly this format:\n" | |
| '{"A":"correct","B":"incorrect"}\n\n' | |
| "Allowed labels are only:\n" | |
| '"correct"\n' | |
| '"incorrect"\n\n' | |
| "If both answers are correct, mark both as correct.\n" | |
| "If both answers are incorrect, mark both as incorrect." | |
| ) | |
| }, | |
| { | |
| "role": "user", | |
| "content": ( | |
| f"Problem:\n{problem}\n\n" | |
| f"Answer A:\n{answer_a}\n\n" | |
| f"Answer B:\n{answer_b}\n\n" | |
| "Evaluate both answers." | |
| ) | |
| } | |
| ] | |
| return tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=False, | |
| add_generation_prompt=True | |
| ) | |
| # ============================================================ | |
| # JUDGE / INFERENCE FUNCTION | |
| # ============================================================ | |
| def judge(problem, answer_a, answer_b): | |
| # -------------------------------------------------------- | |
| # Validate inputs | |
| # -------------------------------------------------------- | |
| if not problem or not problem.strip(): | |
| return '{"error":"Problem is required."}' | |
| if not answer_a or not answer_a.strip(): | |
| return '{"error":"Answer A is required."}' | |
| if not answer_b or not answer_b.strip(): | |
| return '{"error":"Answer B is required."}' | |
| # -------------------------------------------------------- | |
| # Create prompt | |
| # -------------------------------------------------------- | |
| prompt = build_prompt( | |
| problem, | |
| answer_a, | |
| answer_b | |
| ) | |
| # -------------------------------------------------------- | |
| # Tokenize | |
| # -------------------------------------------------------- | |
| inputs = tokenizer( | |
| prompt, | |
| return_tensors="pt", | |
| truncation=True, | |
| max_length=MAX_LENGTH | |
| ) | |
| # -------------------------------------------------------- | |
| # Move input tensors to GPU | |
| # -------------------------------------------------------- | |
| inputs = { | |
| key: value.to("cuda") | |
| for key, value in inputs.items() | |
| } | |
| # -------------------------------------------------------- | |
| # Generate response | |
| # -------------------------------------------------------- | |
| with torch.inference_mode(): | |
| outputs = model.generate( | |
| **inputs, | |
| max_new_tokens=MAX_NEW_TOKENS, | |
| do_sample=False, | |
| pad_token_id=tokenizer.eos_token_id | |
| ) | |
| # -------------------------------------------------------- | |
| # Remove prompt tokens | |
| # -------------------------------------------------------- | |
| input_length = inputs["input_ids"].shape[1] | |
| generated_tokens = outputs[0][input_length:] | |
| # -------------------------------------------------------- | |
| # Decode model response | |
| # -------------------------------------------------------- | |
| response = tokenizer.decode( | |
| generated_tokens, | |
| skip_special_tokens=True | |
| ).strip() | |
| return response | |
| # ============================================================ | |
| # GRADIO APPLICATION | |
| # ============================================================ | |
| with gr.Blocks( | |
| title="LLM as Judge" | |
| ) as demo: | |
| # -------------------------------------------------------- | |
| # Header | |
| # -------------------------------------------------------- | |
| gr.Markdown( | |
| """ | |
| # 🧑⚖️ LLM as Judge | |
| ### Fine-tuned Qwen2.5-7B Judge | |
| Evaluate two answers to the same programming or | |
| technical problem. | |
| """ | |
| ) | |
| # -------------------------------------------------------- | |
| # Problem | |
| # -------------------------------------------------------- | |
| problem = gr.Textbox( | |
| label="Problem", | |
| placeholder=( | |
| "Enter the programming or technical problem..." | |
| ), | |
| lines=6 | |
| ) | |
| # -------------------------------------------------------- | |
| # Answers | |
| # -------------------------------------------------------- | |
| with gr.Row(): | |
| answer_a = gr.Textbox( | |
| label="Answer A", | |
| placeholder="Enter Answer A...", | |
| lines=12 | |
| ) | |
| answer_b = gr.Textbox( | |
| label="Answer B", | |
| placeholder="Enter Answer B...", | |
| lines=12 | |
| ) | |
| # -------------------------------------------------------- | |
| # Judge button | |
| # -------------------------------------------------------- | |
| judge_button = gr.Button( | |
| "⚖️ Evaluate", | |
| variant="primary" | |
| ) | |
| # -------------------------------------------------------- | |
| # Result | |
| # -------------------------------------------------------- | |
| result = gr.Code( | |
| label="Judge Result", | |
| language="json" | |
| ) | |
| # -------------------------------------------------------- | |
| # API EVENT | |
| # -------------------------------------------------------- | |
| # IMPORTANT: | |
| # This must remain INSIDE the Blocks context. | |
| # api_name="judge" exposes the endpoint as /judge. | |
| # -------------------------------------------------------- | |
| judge_button.click( | |
| fn=judge, | |
| inputs=[ | |
| problem, | |
| answer_a, | |
| answer_b | |
| ], | |
| outputs=result, | |
| api_name="judge" | |
| ) | |
| # ============================================================ | |
| # START APPLICATION | |
| # ============================================================ | |
| if __name__ == "__main__": | |
| demo.launch() |