PranjalZetsu's picture
Update app.py
1b8d2a5 verified
Raw History Blame Contribute Delete
2.04 kB
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
# --- CONFIGURATION ---
ADAPTER_REPO = "PranjalZetsu/Fab_Yield_Agent_Qwen-q4"
BASE_MODEL = "unsloth/Qwen2.5-7B-Instruct"
print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(ADAPTER_REPO)
# --- NEW: BitsAndBytes Configuration ---
print("Setting up 4-bit quantization config...")
bnb_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.float16
)
print("Loading base model in 4-bit to save VRAM...")
base_model = AutoModelForCausalLM.from_pretrained(
BASE_MODEL,
quantization_config=bnb_config, # <-- This is the crucial change
device_map="auto",
torch_dtype=torch.float16,
)
print("Attaching your 42-Epoch LoRA Adapter...")
model = PeftModel.from_pretrained(base_model, ADAPTER_REPO)
print("Model ready!")
def generate_response(message, history):
# Format the prompt exactly how your agent was trained
prompt = f"System: You are an AI Semiconductor Process Engineer. Output your reasoning in <think> tags, your diagnosis in <diagnosis> tags, and your parameter adjustments in <experiment> tags.\n\nUser: {message}\n\nAssistant:"
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
# Generate the response
outputs = model.generate(
**inputs,
max_new_tokens=512,
temperature=0.3, # Keep it low so it focuses on logic, not creativity
pad_token_id=tokenizer.eos_token_id
)
response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
return response
# --- THE UI ---
demo = gr.ChatInterface(
fn=generate_response,
title="🏭 TensorTitans Fab Yield Optimizer",
description="Live inference of our GRPO-trained Qwen-7B agent. Type a scenario like: 'Target: H200X Flagship. Thermal constraints are tight. What is our recipe?'"
)
if __name__ == "__main__":
demo.launch(ssr_mode=False)