Royrotem100 commited on
Commit
f2a39ce
·
1 Parent(s): 792d64d

Add DictaLM 2.0 instruct model

Browse files
Files changed (1) hide show
  1. app.py +14 -1
app.py CHANGED
@@ -55,11 +55,24 @@ def predict():
55
  inputs = tokenizer(formatted_text, return_tensors='pt')
56
 
57
  # Generate the output
58
- outputs = model.generate(inputs['input_ids'], max_length=1024, temperature=0.7, top_p=0.9)
 
 
 
 
 
 
 
 
 
59
 
60
  # Decode the output
61
  prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).replace(formatted_text, '').strip()
62
 
 
 
 
 
63
  return jsonify({"prediction": prediction})
64
 
65
  def run_flask():
 
55
  inputs = tokenizer(formatted_text, return_tensors='pt')
56
 
57
  # Generate the output
58
+ outputs = model.generate(
59
+ inputs['input_ids'],
60
+ attention_mask=inputs['attention_mask'],
61
+ max_length=1024,
62
+ temperature=0.7,
63
+ top_p=0.9,
64
+ do_sample=True,
65
+ pad_token_id=tokenizer.eos_token_id
66
+ )
67
+
68
 
69
  # Decode the output
70
  prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).replace(formatted_text, '').strip()
71
 
72
+ # Remove the user input part from the response
73
+ if "[/INST]" in prediction:
74
+ prediction = prediction.split("[/INST]", 1)[-1].strip()
75
+
76
  return jsonify({"prediction": prediction})
77
 
78
  def run_flask():