import os from transformers import AutoTokenizer, AutoModelForCausalLM import torch class AutoModelForCausalLMWrapper: def __init__(self): self.model_name = "facebook/opt-1.3b" # or your preferred model hf_token = os.getenv("HUGGINGFACE_HUB_TOKEN") # Set your HF token in env var HF_TOKEN self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, use_auth_token=hf_token) self.model = AutoModelForCausalLM.from_pretrained( self.model_name, device_map="auto", torch_dtype=torch.float16, use_auth_token=hf_token, ) self.model.eval() self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.model.to(self.device) def chat_response(self, user_message: str) -> str: # Format prompt clearly prompt = f"Q: {user_message.strip()}\nA:" inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device) outputs = self.model.generate( **inputs, max_new_tokens=100, temperature=0.2, top_p=0.9, do_sample=True, eos_token_id=self.tokenizer.eos_token_id, pad_token_id=self.tokenizer.pad_token_id, ) decoded = self.tokenizer.decode(outputs[0], skip_special_tokens=True) # Return only answer part after 'A:' print(decoded) answer = decoded.split("A:")[-1].strip() return answer