Spaces:
Sleeping
Sleeping
Download autoModelForCausalLM.py from abstikatech/AbstikaAI: direct link, hf CLI and curl.
- Browser
- Download file 1.48 kB
-
https://huggingface.co/spaces/abstikatech/AbstikaAI/resolve/main/autoModelForCausalLM.py
- Command line
-
hf download hf://spaces/abstikatech/AbstikaAI/autoModelForCausalLM.py
-
curl -L -o autoModelForCausalLM.py https://huggingface.co/spaces/abstikatech/AbstikaAI/resolve/main/autoModelForCausalLM.py
1.48 kB
| import os | |
| from transformers import AutoTokenizer, AutoModelForCausalLM | |
| import torch | |
| class AutoModelForCausalLMWrapper: | |
| def __init__(self): | |
| self.model_name = "facebook/opt-1.3b" # or your preferred model | |
| hf_token = os.getenv("HUGGINGFACE_HUB_TOKEN") # Set your HF token in env var HF_TOKEN | |
| self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, use_auth_token=hf_token) | |
| self.model = AutoModelForCausalLM.from_pretrained( | |
| self.model_name, | |
| device_map="auto", | |
| torch_dtype=torch.float16, | |
| use_auth_token=hf_token, | |
| ) | |
| self.model.eval() | |
| self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| self.model.to(self.device) | |
| def chat_response(self, user_message: str) -> str: | |
| # Format prompt clearly | |
| prompt = f"Q: {user_message.strip()}\nA:" | |
| inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device) | |
| outputs = self.model.generate( | |
| **inputs, | |
| max_new_tokens=100, | |
| temperature=0.2, | |
| top_p=0.9, | |
| do_sample=True, | |
| eos_token_id=self.tokenizer.eos_token_id, | |
| pad_token_id=self.tokenizer.pad_token_id, | |
| ) | |
| decoded = self.tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| # Return only answer part after 'A:' | |
| print(decoded) | |
| answer = decoded.split("A:")[-1].strip() | |
| return answer | |