balaji commited on
Commit
a589eee
Β·
1 Parent(s): b1d5c60

update endpoint

Browse files
Files changed (1) hide show
  1. autoModelForCausalLM.py +38 -12
autoModelForCausalLM.py CHANGED
@@ -11,15 +11,41 @@ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
11
  model = AutoModelForCausalLM.from_pretrained(MODEL_ID).to("cpu")
12
 
13
 
14
- def chat_response(prompt: str):
15
- """Generate a response from the small model."""
16
- inputs = tokenizer(prompt, return_tensors="pt")
17
- with torch.no_grad():
18
- outputs = model.generate(
19
- **inputs,
20
- max_length=200,
21
- do_sample=True,
22
- temperature=0.7,
23
- top_k=50
24
- )
25
- return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  model = AutoModelForCausalLM.from_pretrained(MODEL_ID).to("cpu")
12
 
13
 
14
+ def chat_response(user_message: str) -> str:
15
+ """
16
+ Generates a concise response from the Mistral model.
17
+ Cleans up repetitions and removes the prompt from the final answer.
18
+ """
19
+
20
+ # πŸ—οΈ Build a clear prompt for the model
21
+ prompt = f"Answer briefly and accurately:\nQuestion: {user_message}\nAnswer:"
22
+
23
+ # πŸ”„ Tokenize and send to model
24
+ inputs = tokenizer(prompt, return_tensors="pt").to(device)
25
+
26
+ # πŸš€ Generate response with controlled length & stopping
27
+ outputs = model.generate(
28
+ **inputs,
29
+ max_new_tokens=64, # βœ… Limit so it doesn't ramble
30
+ do_sample=True,
31
+ temperature=0.7,
32
+ top_p=0.9,
33
+ eos_token_id=tokenizer.eos_token_id # βœ… Stops at </s>
34
+ )
35
+
36
+ # πŸ” Decode and clean up response
37
+ response_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
38
+
39
+ # 🧹 Remove the original prompt text
40
+ cleaned_response = response_text.replace(prompt, "").strip()
41
+
42
+ # πŸ›  Remove obvious repetitions (e.g., "the capital is the capital is...")
43
+ import re
44
+ cleaned_response = re.sub(r'(\b\w+\b)( \1\b)+', r'\1', cleaned_response)
45
+
46
+ # βœ‚ If the model still outputs multiple sentences, keep only the first 2
47
+ sentences = cleaned_response.split(". ")
48
+ final_answer = ". ".join(sentences[:2]).strip()
49
+
50
+ return final_answer
51
+