PlusBIntelligenceLab commited on
Commit
66f025c
·
verified ·
1 Parent(s): 97a7be4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +413 -72
app.py CHANGED
@@ -1,7 +1,6 @@
1
  """
2
- Treevik-2 2.5B Instruct (HF Spaces Ready)
3
- Auto-detects GPU/CPU automatically.
4
- Uses GPU if available, otherwise CPU fallback.
5
  """
6
 
7
  import gc
@@ -17,28 +16,28 @@ import torch.nn.functional as F
17
  from huggingface_hub import hf_hub_download
18
 
19
  # ============================================================================
20
- # SETTINGS
21
  # ============================================================================
22
 
23
- REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct"
24
-
25
  torch.set_grad_enabled(False)
26
 
27
  if torch.cuda.is_available():
28
- DEVICE = "cuda"
29
- DTYPE = torch.float16
30
  torch.backends.cuda.matmul.allow_tf32 = True
31
  else:
32
- DEVICE = "cpu"
33
- DTYPE = torch.float16
34
 
35
- print(f"Using device: {DEVICE}")
36
- print(f"Using dtype: {DTYPE}")
37
 
38
  # ============================================================================
39
  # DOWNLOAD MODEL
40
  # ============================================================================
41
 
 
 
42
  print(f"Downloading model from {REPO_ID}...")
43
 
44
  meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
@@ -69,13 +68,32 @@ class CausalSelfAttention(nn.Module):
69
  self.n_head = config.n_head
70
  self.n_kv_head = config.n_kv_head
71
  self.n_embd = config.n_embd
 
72
  self.head_dim = self.n_embd // self.n_head
73
 
74
- self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False)
75
- self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
76
- self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
 
 
77
 
78
- self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
 
80
  self.ve_gate = (
81
  nn.Linear(32, self.n_kv_head, bias=False)
@@ -83,17 +101,42 @@ class CausalSelfAttention(nn.Module):
83
  else None
84
  )
85
 
86
- def forward(self, x, ve, cos_sin, window_size, kv_cache=None, position_id=0):
 
 
 
 
 
 
 
 
 
87
  B, T, C = x.size()
88
 
89
- q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
90
- k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
91
- v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim)
 
 
 
 
 
 
 
 
92
 
93
  if ve is not None:
94
- ve = ve.view(B, T, self.n_kv_head, self.head_dim)
 
 
 
 
 
 
95
  v = v + (
96
- 2 * torch.sigmoid(self.ve_gate(x[..., :32]))
 
 
97
  ).unsqueeze(-1) * ve
98
 
99
  cos, sin = cos_sin
@@ -105,12 +148,21 @@ class CausalSelfAttention(nn.Module):
105
  k = norm(k)
106
 
107
  if kv_cache is not None:
 
108
  if position_id == 0:
109
  kv_cache["k"] = k
110
  kv_cache["v"] = v
 
111
  else:
112
- kv_cache["k"] = torch.cat([kv_cache["k"], k], dim=1)
113
- kv_cache["v"] = torch.cat([kv_cache["v"], v], dim=1)
 
 
 
 
 
 
 
114
 
115
  k = kv_cache["k"]
116
  v = kv_cache["v"]
@@ -118,16 +170,39 @@ class CausalSelfAttention(nn.Module):
118
  nr = self.n_head // self.n_kv_head
119
 
120
  if nr > 1:
 
121
  k = (
122
  k.unsqueeze(3)
123
- .expand(B, k.size(1), self.n_kv_head, nr, self.head_dim)
124
- .reshape(B, k.size(1), self.n_head, self.head_dim)
 
 
 
 
 
 
 
 
 
 
 
125
  )
126
 
127
  v = (
128
  v.unsqueeze(3)
129
- .expand(B, v.size(1), self.n_kv_head, nr, self.head_dim)
130
- .reshape(B, v.size(1), self.n_head, self.head_dim)
 
 
 
 
 
 
 
 
 
 
 
131
  )
132
 
133
  q = q.transpose(1, 2)
@@ -141,19 +216,34 @@ class CausalSelfAttention(nn.Module):
141
  is_causal=(T > 1),
142
  )
143
 
144
- return self.c_proj(
145
- y.transpose(1, 2).contiguous().view(B, T, -1)
 
 
146
  )
147
 
 
 
148
  class MLP(nn.Module):
149
  def __init__(self, config):
150
  super().__init__()
151
 
152
- self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
153
- self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
 
 
 
 
 
 
 
 
 
154
 
155
  def forward(self, x):
156
- return self.c_proj(F.relu(self.c_fc(x)).square())
 
 
157
 
158
  class Block(nn.Module):
159
  def __init__(self, config, li):
@@ -162,9 +252,27 @@ class Block(nn.Module):
162
  self.attn = CausalSelfAttention(config, li)
163
  self.mlp = MLP(config)
164
 
165
- def forward(self, x, ve, cs, ws, kv_cache=None, position_id=0):
166
- x = x + self.attn(norm(x), ve, cs, ws, kv_cache, position_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  x = x + self.mlp(norm(x))
 
168
  return x
169
 
170
  @dataclass
@@ -183,28 +291,80 @@ class GPT(nn.Module):
183
 
184
  self.config = config
185
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  pad = ((config.vocab_size + 63) // 64) * 64
187
 
188
  self.transformer = nn.ModuleDict({
189
- "wte": nn.Embedding(pad, config.n_embd),
 
 
 
 
190
  "h": nn.ModuleList([
191
- Block(config, i) for i in range(config.n_layer)
192
- ])
 
193
  })
194
 
195
- self.lm_head = nn.Linear(config.n_embd, pad, bias=False)
 
 
 
 
 
 
 
 
 
 
 
 
196
 
197
  hd = config.n_embd // config.n_head
198
 
 
 
 
 
 
 
 
 
 
 
 
199
  inv = 1.0 / (
200
  10000 ** (
201
- torch.arange(0, hd, 2, dtype=torch.float32) / hd
 
 
 
 
 
202
  )
203
  )
204
 
205
  freqs = torch.outer(
206
- torch.arange(config.sequence_len * 10, dtype=torch.float32),
207
- inv
 
 
 
208
  )
209
 
210
  self.register_buffer(
@@ -219,47 +379,126 @@ class GPT(nn.Module):
219
  persistent=False,
220
  )
221
 
222
- def forward(self, idx):
 
 
 
 
 
 
223
  B, T = idx.size()
224
 
225
  cs = (
226
- self.cos[:, :T],
227
- self.sin[:, :T],
 
 
 
 
228
  )
229
 
230
- x = norm(self.transformer["wte"](idx))
 
 
 
 
 
 
 
 
 
 
 
 
 
231
 
232
- for block in self.transformer["h"]:
233
- x = block(x, None, cs, (0, 0))
 
 
 
 
 
 
 
 
 
 
 
 
234
 
235
  logits = self.lm_head(norm(x))
236
- return logits[..., :self.config.vocab_size]
 
 
237
 
238
  @torch.inference_mode()
239
- def generate(self, tokens, max_tokens=256, temperature=0.7, top_k=50):
 
 
 
 
 
 
240
 
241
  ids = torch.tensor(
242
  [tokens],
243
  dtype=torch.long,
244
- device=DEVICE,
245
  )
246
 
 
 
 
 
 
 
 
247
  for _ in range(max_tokens):
248
 
249
- logits = self.forward(ids)[:, -1, :]
 
 
 
 
 
 
 
 
 
 
250
 
251
  if top_k > 0:
252
  v, _ = torch.topk(
253
  logits,
254
  min(top_k, logits.size(-1))
255
  )
256
- logits[logits < v[:, [-1]]] = -float("Inf")
257
 
258
- probs = F.softmax(logits / temperature, dim=-1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
259
 
260
- nid = torch.multinomial(probs, num_samples=1)
 
 
 
261
 
262
- ids = torch.cat((ids, nid), dim=1)
263
 
264
  yield nid.item()
265
 
@@ -283,6 +522,11 @@ config = GPTConfig(**{
283
  ]
284
  })
285
 
 
 
 
 
 
286
  model = GPT(config)
287
 
288
  print("Loading weights...")
@@ -307,23 +551,29 @@ for key in list(model_state.keys()):
307
  scale_key = key + "_scale"
308
 
309
  if scale_key in raw_state_dict:
 
310
  tensor = (
311
  tensor.float()
312
  * raw_state_dict[scale_key].float()
313
  )
314
 
315
- model_state[key].copy_(tensor.to(model_state[key].dtype))
 
 
316
 
317
  print("Moving model to device...")
318
 
319
- model = model.to(device=DEVICE, dtype=DTYPE)
 
 
 
320
 
321
  model.eval()
322
 
323
  del raw_state_dict
324
  gc.collect()
325
 
326
- if DEVICE == "cuda":
327
  torch.cuda.empty_cache()
328
 
329
  # ============================================================================
@@ -339,14 +589,18 @@ USER_END = enc.encode_single_token("<|user_end|>")
339
  ASST_START = enc.encode_single_token("<|assistant_start|>")
340
  ASST_END = enc.encode_single_token("<|assistant_end|>")
341
 
342
- param_count = sum(p.numel() for p in model.parameters()) / 1e9
 
 
 
343
 
344
  print(
345
- f"✅ Loaded model ({param_count:.1f}B params) on {DEVICE}"
 
346
  )
347
 
348
  # ============================================================================
349
- # INFERENCE
350
  # ============================================================================
351
 
352
  @spaces.GPU
@@ -357,16 +611,30 @@ def respond(message, history):
357
  return
358
 
359
  ids = [BOS]
 
 
 
 
 
 
 
 
 
 
 
360
  ids += [USER_START]
361
  ids += enc.encode_ordinary(message)
362
  ids += [USER_END]
 
363
  ids += [ASST_START]
364
 
 
 
365
  partial = ""
366
 
367
  for token in model.generate(
368
  ids,
369
- max_tokens=256,
370
  temperature=0.7,
371
  top_k=50,
372
  ):
@@ -374,28 +642,101 @@ def respond(message, history):
374
  if token == ASST_END:
375
  break
376
 
377
- piece = enc.decode([token])
378
 
379
- partial += piece
380
 
381
  yield partial
382
 
383
  # ============================================================================
384
- # UI
385
  # ============================================================================
386
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
387
  demo = gr.ChatInterface(
388
  fn=respond,
389
- type="messages",
390
- title="Treevik-2 (2.5B)",
391
- description="PlusB Intelligence Lab",
392
- chatbot=gr.Chatbot(height=600),
 
 
 
 
 
 
 
 
 
 
 
 
 
393
  textbox=gr.Textbox(
394
- placeholder="Ask Treevik anything..."
 
 
395
  ),
 
 
 
 
 
 
 
 
 
 
396
  )
397
 
398
  demo.launch(
399
  server_name="0.0.0.0",
400
  server_port=7860,
401
- )
 
1
  """
2
+ Treevik-2 2.5B Instruct
3
+ Modern Gradio UI + HF Spaces GPU Support
 
4
  """
5
 
6
  import gc
 
16
  from huggingface_hub import hf_hub_download
17
 
18
  # ============================================================================
19
+ # DEVICE SETUP
20
  # ============================================================================
21
 
 
 
22
  torch.set_grad_enabled(False)
23
 
24
  if torch.cuda.is_available():
25
+ device = "cuda"
26
+ dtype = torch.float16
27
  torch.backends.cuda.matmul.allow_tf32 = True
28
  else:
29
+ device = "cpu"
30
+ dtype = torch.float32
31
 
32
+ print(f"Using device: {device}")
33
+ print(f"Using dtype: {dtype}")
34
 
35
  # ============================================================================
36
  # DOWNLOAD MODEL
37
  # ============================================================================
38
 
39
+ REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct"
40
+
41
  print(f"Downloading model from {REPO_ID}...")
42
 
43
  meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
 
68
  self.n_head = config.n_head
69
  self.n_kv_head = config.n_kv_head
70
  self.n_embd = config.n_embd
71
+
72
  self.head_dim = self.n_embd // self.n_head
73
 
74
+ self.c_q = nn.Linear(
75
+ self.n_embd,
76
+ self.n_head * self.head_dim,
77
+ bias=False,
78
+ )
79
 
80
+ self.c_k = nn.Linear(
81
+ self.n_embd,
82
+ self.n_kv_head * self.head_dim,
83
+ bias=False,
84
+ )
85
+
86
+ self.c_v = nn.Linear(
87
+ self.n_embd,
88
+ self.n_kv_head * self.head_dim,
89
+ bias=False,
90
+ )
91
+
92
+ self.c_proj = nn.Linear(
93
+ self.n_embd,
94
+ self.n_embd,
95
+ bias=False,
96
+ )
97
 
98
  self.ve_gate = (
99
  nn.Linear(32, self.n_kv_head, bias=False)
 
101
  else None
102
  )
103
 
104
+ def forward(
105
+ self,
106
+ x,
107
+ ve,
108
+ cos_sin,
109
+ window_size,
110
+ kv_cache=None,
111
+ position_id=0,
112
+ ):
113
+
114
  B, T, C = x.size()
115
 
116
+ q = self.c_q(x).view(
117
+ B, T, self.n_head, self.head_dim
118
+ )
119
+
120
+ k = self.c_k(x).view(
121
+ B, T, self.n_kv_head, self.head_dim
122
+ )
123
+
124
+ v = self.c_v(x).view(
125
+ B, T, self.n_kv_head, self.head_dim
126
+ )
127
 
128
  if ve is not None:
129
+ ve = ve.view(
130
+ B,
131
+ T,
132
+ self.n_kv_head,
133
+ self.head_dim,
134
+ )
135
+
136
  v = v + (
137
+ 2 * torch.sigmoid(
138
+ self.ve_gate(x[..., :32])
139
+ )
140
  ).unsqueeze(-1) * ve
141
 
142
  cos, sin = cos_sin
 
148
  k = norm(k)
149
 
150
  if kv_cache is not None:
151
+
152
  if position_id == 0:
153
  kv_cache["k"] = k
154
  kv_cache["v"] = v
155
+
156
  else:
157
+ kv_cache["k"] = torch.cat(
158
+ [kv_cache["k"], k],
159
+ dim=1,
160
+ )
161
+
162
+ kv_cache["v"] = torch.cat(
163
+ [kv_cache["v"], v],
164
+ dim=1,
165
+ )
166
 
167
  k = kv_cache["k"]
168
  v = kv_cache["v"]
 
170
  nr = self.n_head // self.n_kv_head
171
 
172
  if nr > 1:
173
+
174
  k = (
175
  k.unsqueeze(3)
176
+ .expand(
177
+ B,
178
+ k.size(1),
179
+ self.n_kv_head,
180
+ nr,
181
+ self.head_dim,
182
+ )
183
+ .reshape(
184
+ B,
185
+ k.size(1),
186
+ self.n_head,
187
+ self.head_dim,
188
+ )
189
  )
190
 
191
  v = (
192
  v.unsqueeze(3)
193
+ .expand(
194
+ B,
195
+ v.size(1),
196
+ self.n_kv_head,
197
+ nr,
198
+ self.head_dim,
199
+ )
200
+ .reshape(
201
+ B,
202
+ v.size(1),
203
+ self.n_head,
204
+ self.head_dim,
205
+ )
206
  )
207
 
208
  q = q.transpose(1, 2)
 
216
  is_causal=(T > 1),
217
  )
218
 
219
+ y = (
220
+ y.transpose(1, 2)
221
+ .contiguous()
222
+ .view(B, T, -1)
223
  )
224
 
225
+ return self.c_proj(y)
226
+
227
  class MLP(nn.Module):
228
  def __init__(self, config):
229
  super().__init__()
230
 
231
+ self.c_fc = nn.Linear(
232
+ config.n_embd,
233
+ 4 * config.n_embd,
234
+ bias=False,
235
+ )
236
+
237
+ self.c_proj = nn.Linear(
238
+ 4 * config.n_embd,
239
+ config.n_embd,
240
+ bias=False,
241
+ )
242
 
243
  def forward(self, x):
244
+ return self.c_proj(
245
+ F.relu(self.c_fc(x)).square()
246
+ )
247
 
248
  class Block(nn.Module):
249
  def __init__(self, config, li):
 
252
  self.attn = CausalSelfAttention(config, li)
253
  self.mlp = MLP(config)
254
 
255
+ def forward(
256
+ self,
257
+ x,
258
+ ve,
259
+ cs,
260
+ ws,
261
+ kv_cache=None,
262
+ position_id=0,
263
+ ):
264
+
265
+ x = x + self.attn(
266
+ norm(x),
267
+ ve,
268
+ cs,
269
+ ws,
270
+ kv_cache,
271
+ position_id,
272
+ )
273
+
274
  x = x + self.mlp(norm(x))
275
+
276
  return x
277
 
278
  @dataclass
 
291
 
292
  self.config = config
293
 
294
+ pat = config.window_pattern.upper()
295
+
296
+ lw = config.sequence_len
297
+ sw = config.sequence_len // 2
298
+
299
+ self.window_sizes = [
300
+ (
301
+ (sw, 0)
302
+ if c == "S"
303
+ else (lw, 0)
304
+ )
305
+ for c in (pat * config.n_layer)[:config.n_layer]
306
+ ]
307
+
308
+ self.window_sizes[-1] = (lw, 0)
309
+
310
  pad = ((config.vocab_size + 63) // 64) * 64
311
 
312
  self.transformer = nn.ModuleDict({
313
+ "wte": nn.Embedding(
314
+ pad,
315
+ config.n_embd,
316
+ ),
317
+
318
  "h": nn.ModuleList([
319
+ Block(config, i)
320
+ for i in range(config.n_layer)
321
+ ]),
322
  })
323
 
324
+ self.lm_head = nn.Linear(
325
+ config.n_embd,
326
+ pad,
327
+ bias=False,
328
+ )
329
+
330
+ self.resid_lambdas = nn.Parameter(
331
+ torch.ones(config.n_layer)
332
+ )
333
+
334
+ self.x0_lambdas = nn.Parameter(
335
+ torch.zeros(config.n_layer)
336
+ )
337
 
338
  hd = config.n_embd // config.n_head
339
 
340
+ self.value_embeds = nn.ModuleDict({
341
+ str(i): nn.Embedding(
342
+ pad,
343
+ config.n_kv_head * hd,
344
+ )
345
+ for i in range(config.n_layer)
346
+ if has_ve(i, config.n_layer)
347
+ })
348
+
349
+ self.rotary_seq_len = config.sequence_len * 10
350
+
351
  inv = 1.0 / (
352
  10000 ** (
353
+ torch.arange(
354
+ 0,
355
+ hd,
356
+ 2,
357
+ dtype=torch.float32,
358
+ ) / hd
359
  )
360
  )
361
 
362
  freqs = torch.outer(
363
+ torch.arange(
364
+ self.rotary_seq_len,
365
+ dtype=torch.float32,
366
+ ),
367
+ inv,
368
  )
369
 
370
  self.register_buffer(
 
379
  persistent=False,
380
  )
381
 
382
+ def forward(
383
+ self,
384
+ idx,
385
+ kv_cache=None,
386
+ position_id=0,
387
+ ):
388
+
389
  B, T = idx.size()
390
 
391
  cs = (
392
+ self.cos[:, position_id:position_id + T],
393
+ self.sin[:, position_id:position_id + T],
394
+ )
395
+
396
+ x = norm(
397
+ self.transformer.wte(idx)
398
  )
399
 
400
+ x0 = x
401
+
402
+ for i, b in enumerate(self.transformer.h):
403
+
404
+ x = (
405
+ self.resid_lambdas[i] * x
406
+ + self.x0_lambdas[i] * x0
407
+ )
408
+
409
+ ve = (
410
+ self.value_embeds[str(i)](idx)
411
+ if str(i) in self.value_embeds
412
+ else None
413
+ )
414
 
415
+ layer_cache = (
416
+ kv_cache[i]
417
+ if kv_cache is not None
418
+ else None
419
+ )
420
+
421
+ x = b(
422
+ x,
423
+ ve,
424
+ cs,
425
+ self.window_sizes[i],
426
+ layer_cache,
427
+ position_id,
428
+ )
429
 
430
  logits = self.lm_head(norm(x))
431
+ logits = logits[..., :self.config.vocab_size]
432
+
433
+ return 15 * torch.tanh(logits / 15)
434
 
435
  @torch.inference_mode()
436
+ def generate(
437
+ self,
438
+ tokens,
439
+ max_tokens=512,
440
+ temperature=0.7,
441
+ top_k=50,
442
+ ):
443
 
444
  ids = torch.tensor(
445
  [tokens],
446
  dtype=torch.long,
447
+ device=device,
448
  )
449
 
450
+ kv_cache = [
451
+ {}
452
+ for _ in range(self.config.n_layer)
453
+ ]
454
+
455
+ position_id = 0
456
+
457
  for _ in range(max_tokens):
458
 
459
+ curr_ids = (
460
+ ids
461
+ if position_id == 0
462
+ else ids[:, -1:]
463
+ )
464
+
465
+ logits = self.forward(
466
+ curr_ids,
467
+ kv_cache=kv_cache,
468
+ position_id=position_id,
469
+ )[:, -1, :]
470
 
471
  if top_k > 0:
472
  v, _ = torch.topk(
473
  logits,
474
  min(top_k, logits.size(-1))
475
  )
 
476
 
477
+ logits[
478
+ logits < v[:, [-1]]
479
+ ] = -float("Inf")
480
+
481
+ if temperature > 0:
482
+ nid = torch.multinomial(
483
+ F.softmax(
484
+ logits / temperature,
485
+ dim=-1,
486
+ ),
487
+ 1,
488
+ )
489
+ else:
490
+ nid = torch.argmax(
491
+ logits,
492
+ dim=-1,
493
+ keepdim=True,
494
+ )
495
 
496
+ ids = torch.cat(
497
+ (ids, nid),
498
+ dim=1,
499
+ )
500
 
501
+ position_id += curr_ids.size(1)
502
 
503
  yield nid.item()
504
 
 
522
  ]
523
  })
524
 
525
+ config.window_pattern = mc.get(
526
+ "window_pattern",
527
+ "L",
528
+ )
529
+
530
  model = GPT(config)
531
 
532
  print("Loading weights...")
 
551
  scale_key = key + "_scale"
552
 
553
  if scale_key in raw_state_dict:
554
+
555
  tensor = (
556
  tensor.float()
557
  * raw_state_dict[scale_key].float()
558
  )
559
 
560
+ model_state[key].copy_(
561
+ tensor.to(model_state[key].dtype)
562
+ )
563
 
564
  print("Moving model to device...")
565
 
566
+ model = model.to(
567
+ device=device,
568
+ dtype=dtype,
569
+ )
570
 
571
  model.eval()
572
 
573
  del raw_state_dict
574
  gc.collect()
575
 
576
+ if device == "cuda":
577
  torch.cuda.empty_cache()
578
 
579
  # ============================================================================
 
589
  ASST_START = enc.encode_single_token("<|assistant_start|>")
590
  ASST_END = enc.encode_single_token("<|assistant_end|>")
591
 
592
+ param_count = (
593
+ sum(p.numel() for p in model.parameters())
594
+ / 1e9
595
+ )
596
 
597
  print(
598
+ f"✅ Loaded on {device} "
599
+ f"({param_count:.1f}B params)"
600
  )
601
 
602
  # ============================================================================
603
+ # CHAT FUNCTION
604
  # ============================================================================
605
 
606
  @spaces.GPU
 
611
  return
612
 
613
  ids = [BOS]
614
+
615
+ for user_msg, bot_msg in history:
616
+
617
+ ids += [USER_START]
618
+ ids += enc.encode_ordinary(user_msg)
619
+ ids += [USER_END]
620
+
621
+ ids += [ASST_START]
622
+ ids += enc.encode_ordinary(bot_msg)
623
+ ids += [ASST_END]
624
+
625
  ids += [USER_START]
626
  ids += enc.encode_ordinary(message)
627
  ids += [USER_END]
628
+
629
  ids += [ASST_START]
630
 
631
+ generated = []
632
+
633
  partial = ""
634
 
635
  for token in model.generate(
636
  ids,
637
+ max_tokens=512,
638
  temperature=0.7,
639
  top_k=50,
640
  ):
 
642
  if token == ASST_END:
643
  break
644
 
645
+ generated.append(token)
646
 
647
+ partial = enc.decode(generated)
648
 
649
  yield partial
650
 
651
  # ============================================================================
652
+ # MODERN CHATGPT STYLE UI
653
  # ============================================================================
654
 
655
+ css = """
656
+ body {
657
+ background: #0f172a;
658
+ }
659
+
660
+ .gradio-container {
661
+ max-width: 1100px !important;
662
+ margin: auto;
663
+ }
664
+
665
+ footer {
666
+ display: none !important;
667
+ }
668
+
669
+ .message.user {
670
+ background: #2563eb !important;
671
+ color: white !important;
672
+ border-radius: 18px !important;
673
+ }
674
+
675
+ .message.bot {
676
+ background: #1e293b !important;
677
+ color: #f1f5f9 !important;
678
+ border-radius: 18px !important;
679
+ }
680
+
681
+ textarea {
682
+ border-radius: 18px !important;
683
+ background: #111827 !important;
684
+ color: white !important;
685
+ border: 1px solid #334155 !important;
686
+ }
687
+
688
+ button {
689
+ border-radius: 14px !important;
690
+ }
691
+
692
+ .chatbot {
693
+ border: none !important;
694
+ }
695
+ """
696
+
697
+ theme = gr.themes.Soft(
698
+ primary_hue="blue",
699
+ secondary_hue="slate",
700
+ neutral_hue="slate",
701
+ )
702
+
703
  demo = gr.ChatInterface(
704
  fn=respond,
705
+ type="tuples",
706
+ title="Treevik-2 2.5B",
707
+ description="""
708
+ <div style='text-align:center'>
709
+ <h3>PlusB Intelligence Lab</h3>
710
+ <p>Instruction Tuned 2.5B Parameter Model</p>
711
+ </div>
712
+ """,
713
+ chatbot=gr.Chatbot(
714
+ height=650,
715
+ bubble_full_width=False,
716
+ show_copy_button=True,
717
+ avatar_images=(
718
+ None,
719
+ "https://ui-avatars.com/api/?name=T&background=2563eb&color=fff",
720
+ ),
721
+ ),
722
  textbox=gr.Textbox(
723
+ placeholder="Message Treevik...",
724
+ container=False,
725
+ scale=8,
726
  ),
727
+ submit_btn="Send",
728
+ retry_btn="Retry",
729
+ undo_btn="Undo",
730
+ clear_btn="Clear",
731
+ theme=theme,
732
+ css=css,
733
+ )
734
+
735
+ demo.queue(
736
+ max_size=32
737
  )
738
 
739
  demo.launch(
740
  server_name="0.0.0.0",
741
  server_port=7860,
742
+ )