shieldx-model / Service.py
Nagraj23's picture
Update Service.py
f96407a verified
Raw History Blame Contribute Delete
933 Bytes
from transformers import AutoTokenizer, AutoModelForCausalLM
import torch
def load_model():
model_name = "Nagraj23/shieldx-falcon-merged"
tokenizer = AutoTokenizer.from_pretrained(model_name)
# Force model to run on CPU even if torch thinks CUDA is available
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32).to("cpu")
return model, tokenizer
def format_prompt(user_input):
return (
"<|system|>\nYou are ShieldX, a helpful safety bot.\n<|end|>\n"
f"<|user|>\n{user_input}\n<|end|>\n"
"<|assistant|>"
)
def generate_response(model, tokenizer, user_input):
prompt = format_prompt(user_input)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=100)
reply = tokenizer.decode(outputs[0], skip_special_tokens=True)
return reply.split("<|assistant|>")[-1].strip()