Test502 commited on
Commit
12bf817
·
verified ·
1 Parent(s): 9c3311c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +57 -21
app.py CHANGED
@@ -1,27 +1,53 @@
1
  import gradio as gr
2
- from transformers import AutoTokenizer, AutoModelForCausalLM
3
  import torch
 
4
 
5
  MODEL_ID = "saai-sa/ASL-4B-v1"
6
 
 
 
7
  tokenizer = AutoTokenizer.from_pretrained(
8
  MODEL_ID,
9
  trust_remote_code=True
10
  )
11
 
12
- model = AutoModelForCausalLM.from_pretrained(
13
- MODEL_ID,
14
- torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
15
- device_map="auto",
16
- trust_remote_code=True
17
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
- def respond(message, history):
20
  messages = []
21
 
22
- for item in history:
23
- if isinstance(item, dict):
24
- messages.append(item)
 
 
 
 
 
 
 
 
25
 
26
  messages.append({
27
  "role": "user",
@@ -37,29 +63,39 @@ def respond(message, history):
37
  inputs = tokenizer(
38
  prompt,
39
  return_tensors="pt"
40
- ).to(model.device)
41
 
42
- with torch.no_grad():
43
- outputs = model.generate(
44
  **inputs,
45
- max_new_tokens=300,
 
46
  temperature=0.7,
47
  top_p=0.8,
48
- do_sample=True
 
49
  )
50
 
51
- new_tokens = outputs[0][inputs.input_ids.shape[-1]:]
52
 
53
- return tokenizer.decode(
54
- new_tokens,
55
  skip_special_tokens=True
56
  )
57
 
 
 
58
 
59
  demo = gr.ChatInterface(
60
- fn=respond,
61
  title="ASL-4B-v1",
62
  description="Saudi Arabic Language Model"
63
  )
64
 
65
- demo.launch()
 
 
 
 
 
 
 
1
  import gradio as gr
2
+ import spaces
3
  import torch
4
+ from transformers import AutoTokenizer, AutoModelForCausalLM
5
 
6
  MODEL_ID = "saai-sa/ASL-4B-v1"
7
 
8
+ print("Loading tokenizer...")
9
+
10
  tokenizer = AutoTokenizer.from_pretrained(
11
  MODEL_ID,
12
  trust_remote_code=True
13
  )
14
 
15
+ model = None
16
+
17
+
18
+ @spaces.GPU(duration=120)
19
+ def chat(message, history):
20
+ global model
21
+
22
+ # تحميل المودل أول مرة فقط بعد الحصول على GPU
23
+ if model is None:
24
+ print("Loading ASL-4B on GPU...")
25
+
26
+ model = AutoModelForCausalLM.from_pretrained(
27
+ MODEL_ID,
28
+ dtype=torch.bfloat16,
29
+ trust_remote_code=True
30
+ )
31
+
32
+ model = model.to("cuda")
33
+ model.eval()
34
+
35
+ print("ASL-4B loaded!")
36
+
37
 
 
38
  messages = []
39
 
40
+ if history:
41
+ for item in history:
42
+ if isinstance(item, dict):
43
+ role = item.get("role")
44
+ content = item.get("content")
45
+
46
+ if role in ["user", "assistant"] and content:
47
+ messages.append({
48
+ "role": role,
49
+ "content": content
50
+ })
51
 
52
  messages.append({
53
  "role": "user",
 
63
  inputs = tokenizer(
64
  prompt,
65
  return_tensors="pt"
66
+ ).to("cuda")
67
 
68
+ with torch.inference_mode():
69
+ output = model.generate(
70
  **inputs,
71
+ max_new_tokens=512,
72
+ do_sample=True,
73
  temperature=0.7,
74
  top_p=0.8,
75
+ top_k=20,
76
+ repetition_penalty=1.0
77
  )
78
 
79
+ generated_tokens = output[0][inputs.input_ids.shape[-1]:]
80
 
81
+ response = tokenizer.decode(
82
+ generated_tokens,
83
  skip_special_tokens=True
84
  )
85
 
86
+ return response
87
+
88
 
89
  demo = gr.ChatInterface(
90
+ fn=chat,
91
  title="ASL-4B-v1",
92
  description="Saudi Arabic Language Model"
93
  )
94
 
95
+ demo.queue()
96
+
97
+ demo.launch(
98
+ server_name="0.0.0.0",
99
+ server_port=7860,
100
+ ssr_mode=False
101
+ )