import gradio as gr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig from peft import PeftModel # --- CONFIGURATION --- ADAPTER_REPO = "PranjalZetsu/Fab_Yield_Agent_Qwen-q4" BASE_MODEL = "unsloth/Qwen2.5-7B-Instruct" print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained(ADAPTER_REPO) # --- NEW: BitsAndBytes Configuration --- print("Setting up 4-bit quantization config...") bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16 ) print("Loading base model in 4-bit to save VRAM...") base_model = AutoModelForCausalLM.from_pretrained( BASE_MODEL, quantization_config=bnb_config, # <-- This is the crucial change device_map="auto", torch_dtype=torch.float16, ) print("Attaching your 42-Epoch LoRA Adapter...") model = PeftModel.from_pretrained(base_model, ADAPTER_REPO) print("Model ready!") def generate_response(message, history): # Format the prompt exactly how your agent was trained prompt = f"System: You are an AI Semiconductor Process Engineer. Output your reasoning in tags, your diagnosis in tags, and your parameter adjustments in tags.\n\nUser: {message}\n\nAssistant:" inputs = tokenizer(prompt, return_tensors="pt").to("cuda") # Generate the response outputs = model.generate( **inputs, max_new_tokens=512, temperature=0.3, # Keep it low so it focuses on logic, not creativity pad_token_id=tokenizer.eos_token_id ) response = tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True) return response # --- THE UI --- demo = gr.ChatInterface( fn=generate_response, title="🏭 TensorTitans Fab Yield Optimizer", description="Live inference of our GRPO-trained Qwen-7B agent. Type a scenario like: 'Target: H200X Flagship. Thermal constraints are tight. What is our recipe?'" ) if __name__ == "__main__": demo.launch(ssr_mode=False)