Nafiul Huda commited on
Commit
f1db52b
·
verified ·
1 Parent(s): b9caef8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +32 -42
app.py CHANGED
@@ -1,61 +1,51 @@
1
  import os
2
- import sys
3
-
4
- # 1. Map PyTorch's internal CUDA shared objects before loading llama_cpp
5
- try:
6
- import torch
7
- torch_lib_path = os.path.join(os.path.dirname(torch.__file__), "lib")
8
- if torch_lib_path not in os.environ.get("LD_LIBRARY_PATH", ""):
9
- os.environ["LD_LIBRARY_PATH"] = f"{torch_lib_path}:{os.environ.get('LD_LIBRARY_PATH', '')}"
10
- import ctypes
11
- for lib in ["libcudart.so.12", "libcublas.so.12"]:
12
- try:
13
- ctypes.CDLL(os.path.join(torch_lib_path, lib))
14
- except Exception:
15
- pass
16
- except ImportError:
17
- pass
18
-
19
- from llama_cpp import Llama
20
  import gradio as gr
21
  import spaces
22
- from huggingface_hub import hf_hub_download
23
 
24
- # 2. Download and prepare your model locally
25
- print("Downloading model...")
26
- model_path = hf_hub_download(
27
- repo_id="Abiray/MiniCPM5-1B-GGUF",
28
- filename="minicpm5-1b-Q6_K.gguf"
29
- )
30
 
31
- print("Initializing model on GPU paths...")
32
- llm = Llama(
33
- model_path=model_path,
34
- n_ctx=2048,
35
- n_gpu_layers=-1 # Full CUDA offloading
36
- )
37
 
38
- # 3. Claim the ZeroGPU runtime inside the inference execution layer
39
  @spaces.GPU(duration=60)
40
  def generate_response(message, history):
 
 
 
 
 
 
 
 
 
41
  prompt = f"<|user|>\n{message}\n<|assistant|>\n"
 
42
 
43
- output = llm(
44
- prompt,
45
- max_tokens=256,
46
- stop=["<|user|>", "<|assistant|>", "\n\n"],
47
- echo=False
48
- )
 
 
 
 
 
 
 
 
49
 
50
- return output["choices"][0]["text"].strip()
51
 
52
- # 4. Clean Gradio 6 App Layout
53
  with gr.Blocks() as demo:
54
- gr.Markdown("### MiniCPM-1B GGUF ZeroGPU Pipeline")
55
  chatbot = gr.ChatInterface(fn=generate_response)
56
 
57
  if __name__ == "__main__":
58
- # Theme configuration explicitly goes into launch() under Gradio 6
59
  demo.launch(
60
  server_name="0.0.0.0",
61
  server_port=7860,
 
1
  import os
2
+ import torch
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  import gradio as gr
4
  import spaces
5
+ from transformers import AutoModelForCausalLM, AutoTokenizer
6
 
7
+ model_id = "openbmb/MiniCPM5-1B"
 
 
 
 
 
8
 
9
+ print("Loading tokenizer...")
10
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
 
 
 
 
11
 
 
12
  @spaces.GPU(duration=60)
13
  def generate_response(message, history):
14
+ # Load model directly into the allocated GPU memory space
15
+ model = AutoModelForCausalLM.from_pretrained(
16
+ model_id,
17
+ torch_dtype=torch.bfloat16,
18
+ device_map="cuda",
19
+ trust_remote_code=True
20
+ )
21
+
22
+ # Format utilizing the standard ChatML syntax structure for MiniCPM
23
  prompt = f"<|user|>\n{message}\n<|assistant|>\n"
24
+ inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
25
 
26
+ # High-speed native PyTorch generation
27
+ with torch.no_grad():
28
+ outputs = model.generate(
29
+ **inputs,
30
+ max_new_tokens=256,
31
+ do_sample=True,
32
+ temperature=0.7,
33
+ top_p=0.8,
34
+ eos_token_id=tokenizer.eos_token_id
35
+ )
36
+
37
+ # Slice output tokens to extract only the generated assistant text
38
+ generated_tokens = outputs[0][inputs.input_ids.shape[1]:]
39
+ response_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
40
 
41
+ return response_text.strip()
42
 
43
+ # Gradio 6 Minimal Layout
44
  with gr.Blocks() as demo:
45
+ gr.Markdown("### MiniCPM5-1B ZeroGPU Pipeline (Transformers)")
46
  chatbot = gr.ChatInterface(fn=generate_response)
47
 
48
  if __name__ == "__main__":
 
49
  demo.launch(
50
  server_name="0.0.0.0",
51
  server_port=7860,