import gradio as gr import torch from transformers import AutoTokenizer, AutoModelForCausalLM # Small Granite model (best for CPU / low RAM) MODEL_NAME = "ibm-granite/granite-1b-instruct" # Load tokenizer tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) # Load model with low memory settings model = AutoModelForCausalLM.from_pretrained( MODEL_NAME, torch_dtype=torch.float32, # safer for CPU low_cpu_mem_usage=True, device_map="cpu" ) model.eval() def generate_response(prompt): if not prompt.strip(): return "Please enter a prompt." inputs = tokenizer( prompt, return_tensors="pt", truncation=True, max_length=512 ) with torch.no_grad(): output_ids = model.generate( **inputs, max_new_tokens=200, temperature=0.7, top_p=0.9, do_sample=True, repetition_penalty=1.1 ) response = tokenizer.decode( output_ids[0], skip_special_tokens=True ) return response # Minimal Gradio UI demo = gr.Interface( fn=generate_response, inputs=gr.Textbox( lines=5, label="Prompt", placeholder="Ask a question..." ), outputs=gr.Textbox( lines=10, label="Granite Response" ), title="IBM Granite (Low-Memory CPU Demo)", description="Optimized for Hugging Face Spaces with limited RAM" ) if __name__ == "__main__": demo.launch()