File size: 1,483 Bytes
735344a
 
735de60
 
a589eee
604b846
735344a
0872a11
 
 
 
 
 
 
 
 
735344a
0872a11
 
 
 
 
 
 
 
 
 
 
91f5e66
0872a11
 
 
 
 
 
 
6cd0daf
4b78404
122e560
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import os
from transformers import AutoTokenizer, AutoModelForCausalLM
import torch


class AutoModelForCausalLMWrapper:
    def __init__(self):
        self.model_name = "facebook/opt-1.3b"  # or your preferred model
        hf_token = os.getenv("HUGGINGFACE_HUB_TOKEN")  # Set your HF token in env var HF_TOKEN
        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, use_auth_token=hf_token)
        self.model = AutoModelForCausalLM.from_pretrained(
            self.model_name,
            device_map="auto",
            torch_dtype=torch.float16,
            use_auth_token=hf_token,
        )
        self.model.eval()
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.model.to(self.device)

    def chat_response(self, user_message: str) -> str:
        # Format prompt clearly
        prompt = f"Q: {user_message.strip()}\nA:"

        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)
        outputs = self.model.generate(
            **inputs,
            max_new_tokens=100,
            temperature=0.2,
            top_p=0.9,
            do_sample=True,
            eos_token_id=self.tokenizer.eos_token_id,
            pad_token_id=self.tokenizer.pad_token_id,
        )
        decoded = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
        # Return only answer part after 'A:'
        print(decoded)
        answer = decoded.split("A:")[-1].strip()
        return answer