lahyk commited on
Commit
bcd4bbf
·
verified ·
1 Parent(s): 5b297d8

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +77 -0
app.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import spaces
3
+ import torch
4
+
5
+ from transformers import AutoProcessor, AutoModelForMultimodalLM
6
+
7
+
8
+ MODEL_ID = "google/gemma-4-E4B-it"
9
+
10
+ processor = AutoProcessor.from_pretrained(MODEL_ID)
11
+
12
+ model = AutoModelForMultimodalLM.from_pretrained(
13
+ MODEL_ID,
14
+ dtype="auto",
15
+ device_map="auto",
16
+ )
17
+
18
+
19
+ @spaces.GPU
20
+ def chat(message, history):
21
+ messages = []
22
+
23
+ for user, assistant in history:
24
+ messages.append({
25
+ "role": "user",
26
+ "content": user
27
+ })
28
+ messages.append({
29
+ "role": "assistant",
30
+ "content": assistant
31
+ })
32
+
33
+ messages.append({
34
+ "role": "user",
35
+ "content": message
36
+ })
37
+
38
+ inputs = processor.apply_chat_template(
39
+ messages,
40
+ tokenize=True,
41
+ return_dict=True,
42
+ return_tensors="pt",
43
+ add_generation_prompt=True,
44
+ enable_thinking=False,
45
+ ).to(model.device)
46
+
47
+ input_len = inputs["input_ids"].shape[-1]
48
+
49
+ with torch.inference_mode():
50
+ outputs = model.generate(
51
+ **inputs,
52
+ max_new_tokens=512,
53
+ do_sample=True,
54
+ temperature=0.7,
55
+ top_p=0.95,
56
+ )
57
+
58
+ response = processor.decode(
59
+ outputs[0][input_len:],
60
+ skip_special_tokens=False
61
+ )
62
+
63
+ response = processor.parse_response(response)
64
+
65
+ return response
66
+
67
+
68
+ demo = gr.ChatInterface(
69
+ fn=chat,
70
+ title="Gemma 4 E4B",
71
+ textbox=gr.Textbox(
72
+ placeholder="Напиши что-нибудь...",
73
+ container=True
74
+ ),
75
+ )
76
+
77
+ demo.launch()