1adaodoze commited on
Commit
28d21d5
·
verified ·
1 Parent(s): 53e1932

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +28 -7
app.py CHANGED
@@ -5,9 +5,22 @@ from fastapi import FastAPI
5
  from fastapi.responses import StreamingResponse
6
  from pydantic import BaseModel
7
  from huggingface_hub import hf_hub_download
8
- from llama_cpp import Llama
9
  import gradio as gr
10
  import uvicorn
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
  # --- Modelo ---
13
  MODEL_REPO = "bartowski/Dolphin3.0-Llama3.1-8B-GGUF"
@@ -26,6 +39,7 @@ def load_model():
26
  model_path=model_path,
27
  n_ctx=N_CTX,
28
  n_threads=N_THREADS,
 
29
  verbose=False,
30
  )
31
  print("Modelo carregado com sucesso.")
@@ -60,10 +74,15 @@ async def list_models():
60
  ],
61
  }
62
 
63
- @api.post("/v1/chat/completions")
64
- async def chat_completions(req: ChatCompletionRequest):
65
  model = load_model()
 
 
66
 
 
 
 
67
  prompt = ""
68
  for msg in req.messages:
69
  if msg.role == "system":
@@ -74,8 +93,10 @@ async def chat_completions(req: ChatCompletionRequest):
74
  prompt += f"<|im_start|>assistant\n{msg.content}<|im_end|>\n"
75
  prompt += "<|im_start|>assistant\n"
76
 
 
77
  if req.stream:
78
  async def stream_generator():
 
79
  output = model(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stream=True)
80
  for chunk in output:
81
  token = chunk["choices"][0]["text"]
@@ -88,8 +109,8 @@ async def chat_completions(req: ChatCompletionRequest):
88
  yield "data: [DONE]\n\n"
89
  return StreamingResponse(stream_generator(), media_type="text/event-stream")
90
 
91
- output = model(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stream=False)
92
- text = output["choices"][0]["text"]
93
  return {
94
  "id": "chatcmpl-local",
95
  "object": "chat.completion",
@@ -99,7 +120,7 @@ async def chat_completions(req: ChatCompletionRequest):
99
  "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
100
  }
101
 
102
- # --- Gradio (obrigatório para o SDK) ---
103
  def dummy(x):
104
  return x
105
 
@@ -108,7 +129,7 @@ demo = gr.Interface(fn=dummy, inputs="text", outputs="text")
108
  # --- Monta o Gradio dentro do FastAPI ---
109
  app = gr.mount_gradio_app(api, demo, path="/ui")
110
 
111
- # --- Rodar localmente (o Hugging Face ignora este bloco) ---
112
  if __name__ == "__main__":
113
  load_model()
114
  uvicorn.run(app, host="0.0.0.0", port=7860)
 
5
  from fastapi.responses import StreamingResponse
6
  from pydantic import BaseModel
7
  from huggingface_hub import hf_hub_download
 
8
  import gradio as gr
9
  import uvicorn
10
+ import spaces
11
+ import subprocess
12
+ import sys
13
+
14
+ # --- Instalação do llama-cpp-python via wheel (compatível com o ambiente do HF) ---
15
+ # Este comando instala a biblioteca a partir de um pacote binário pré-compilado.
16
+ # É uma alternativa robusta para evitar erros de compilação no ambiente do Space.
17
+ subprocess.check_call([
18
+ sys.executable, "-m", "pip", "install", "--no-cache-dir",
19
+ "https://huggingface.co/Luigi/llama-cpp-python-wheels-hf-spaces-free-cpu/resolve/main/llama_cpp_python-0.3.22-cp310-cp310-linux_x86_64.whl"
20
+ ])
21
+
22
+ # Agora que o wheel foi instalado, podemos importar a biblioteca com segurança.
23
+ from llama_cpp import Llama
24
 
25
  # --- Modelo ---
26
  MODEL_REPO = "bartowski/Dolphin3.0-Llama3.1-8B-GGUF"
 
39
  model_path=model_path,
40
  n_ctx=N_CTX,
41
  n_threads=N_THREADS,
42
+ n_gpu_layers=-1, # Tenta usar a GPU (se disponível)
43
  verbose=False,
44
  )
45
  print("Modelo carregado com sucesso.")
 
74
  ],
75
  }
76
 
77
+ @spaces.GPU # <-- Esta linha garante que a GPU seja alocada para a inferência
78
+ def generate_response(prompt: str, max_tokens: int, temperature: float):
79
  model = load_model()
80
+ output = model(prompt, max_tokens=max_tokens, temperature=temperature, stream=False)
81
+ return output["choices"][0]["text"]
82
 
83
+ @api.post("/v1/chat/completions")
84
+ async def chat_completions(req: ChatCompletionRequest):
85
+ # Formata o prompt para o modelo Dolphin (formato ChatML)
86
  prompt = ""
87
  for msg in req.messages:
88
  if msg.role == "system":
 
93
  prompt += f"<|im_start|>assistant\n{msg.content}<|im_end|>\n"
94
  prompt += "<|im_start|>assistant\n"
95
 
96
+ # --- Modo de streaming ---
97
  if req.stream:
98
  async def stream_generator():
99
+ model = load_model()
100
  output = model(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stream=True)
101
  for chunk in output:
102
  token = chunk["choices"][0]["text"]
 
109
  yield "data: [DONE]\n\n"
110
  return StreamingResponse(stream_generator(), media_type="text/event-stream")
111
 
112
+ # --- Modo normal ---
113
+ text = generate_response(prompt, req.max_tokens, req.temperature)
114
  return {
115
  "id": "chatcmpl-local",
116
  "object": "chat.completion",
 
120
  "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
121
  }
122
 
123
+ # --- Gradio (Interface web + API) ---
124
  def dummy(x):
125
  return x
126
 
 
129
  # --- Monta o Gradio dentro do FastAPI ---
130
  app = gr.mount_gradio_app(api, demo, path="/ui")
131
 
132
+ # --- Rodar (o Hugging Face inicia via `python app.py`) ---
133
  if __name__ == "__main__":
134
  load_model()
135
  uvicorn.run(app, host="0.0.0.0", port=7860)