Update app.py
Browse files
app.py
CHANGED
|
@@ -5,9 +5,22 @@ from fastapi import FastAPI
|
|
| 5 |
from fastapi.responses import StreamingResponse
|
| 6 |
from pydantic import BaseModel
|
| 7 |
from huggingface_hub import hf_hub_download
|
| 8 |
-
from llama_cpp import Llama
|
| 9 |
import gradio as gr
|
| 10 |
import uvicorn
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
# --- Modelo ---
|
| 13 |
MODEL_REPO = "bartowski/Dolphin3.0-Llama3.1-8B-GGUF"
|
|
@@ -26,6 +39,7 @@ def load_model():
|
|
| 26 |
model_path=model_path,
|
| 27 |
n_ctx=N_CTX,
|
| 28 |
n_threads=N_THREADS,
|
|
|
|
| 29 |
verbose=False,
|
| 30 |
)
|
| 31 |
print("Modelo carregado com sucesso.")
|
|
@@ -60,10 +74,15 @@ async def list_models():
|
|
| 60 |
],
|
| 61 |
}
|
| 62 |
|
| 63 |
-
@
|
| 64 |
-
|
| 65 |
model = load_model()
|
|
|
|
|
|
|
| 66 |
|
|
|
|
|
|
|
|
|
|
| 67 |
prompt = ""
|
| 68 |
for msg in req.messages:
|
| 69 |
if msg.role == "system":
|
|
@@ -74,8 +93,10 @@ async def chat_completions(req: ChatCompletionRequest):
|
|
| 74 |
prompt += f"<|im_start|>assistant\n{msg.content}<|im_end|>\n"
|
| 75 |
prompt += "<|im_start|>assistant\n"
|
| 76 |
|
|
|
|
| 77 |
if req.stream:
|
| 78 |
async def stream_generator():
|
|
|
|
| 79 |
output = model(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stream=True)
|
| 80 |
for chunk in output:
|
| 81 |
token = chunk["choices"][0]["text"]
|
|
@@ -88,8 +109,8 @@ async def chat_completions(req: ChatCompletionRequest):
|
|
| 88 |
yield "data: [DONE]\n\n"
|
| 89 |
return StreamingResponse(stream_generator(), media_type="text/event-stream")
|
| 90 |
|
| 91 |
-
|
| 92 |
-
text =
|
| 93 |
return {
|
| 94 |
"id": "chatcmpl-local",
|
| 95 |
"object": "chat.completion",
|
|
@@ -99,7 +120,7 @@ async def chat_completions(req: ChatCompletionRequest):
|
|
| 99 |
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
| 100 |
}
|
| 101 |
|
| 102 |
-
# --- Gradio (
|
| 103 |
def dummy(x):
|
| 104 |
return x
|
| 105 |
|
|
@@ -108,7 +129,7 @@ demo = gr.Interface(fn=dummy, inputs="text", outputs="text")
|
|
| 108 |
# --- Monta o Gradio dentro do FastAPI ---
|
| 109 |
app = gr.mount_gradio_app(api, demo, path="/ui")
|
| 110 |
|
| 111 |
-
# --- Rodar
|
| 112 |
if __name__ == "__main__":
|
| 113 |
load_model()
|
| 114 |
uvicorn.run(app, host="0.0.0.0", port=7860)
|
|
|
|
| 5 |
from fastapi.responses import StreamingResponse
|
| 6 |
from pydantic import BaseModel
|
| 7 |
from huggingface_hub import hf_hub_download
|
|
|
|
| 8 |
import gradio as gr
|
| 9 |
import uvicorn
|
| 10 |
+
import spaces
|
| 11 |
+
import subprocess
|
| 12 |
+
import sys
|
| 13 |
+
|
| 14 |
+
# --- Instalação do llama-cpp-python via wheel (compatível com o ambiente do HF) ---
|
| 15 |
+
# Este comando instala a biblioteca a partir de um pacote binário pré-compilado.
|
| 16 |
+
# É uma alternativa robusta para evitar erros de compilação no ambiente do Space.
|
| 17 |
+
subprocess.check_call([
|
| 18 |
+
sys.executable, "-m", "pip", "install", "--no-cache-dir",
|
| 19 |
+
"https://huggingface.co/Luigi/llama-cpp-python-wheels-hf-spaces-free-cpu/resolve/main/llama_cpp_python-0.3.22-cp310-cp310-linux_x86_64.whl"
|
| 20 |
+
])
|
| 21 |
+
|
| 22 |
+
# Agora que o wheel foi instalado, podemos importar a biblioteca com segurança.
|
| 23 |
+
from llama_cpp import Llama
|
| 24 |
|
| 25 |
# --- Modelo ---
|
| 26 |
MODEL_REPO = "bartowski/Dolphin3.0-Llama3.1-8B-GGUF"
|
|
|
|
| 39 |
model_path=model_path,
|
| 40 |
n_ctx=N_CTX,
|
| 41 |
n_threads=N_THREADS,
|
| 42 |
+
n_gpu_layers=-1, # Tenta usar a GPU (se disponível)
|
| 43 |
verbose=False,
|
| 44 |
)
|
| 45 |
print("Modelo carregado com sucesso.")
|
|
|
|
| 74 |
],
|
| 75 |
}
|
| 76 |
|
| 77 |
+
@spaces.GPU # <-- Esta linha garante que a GPU seja alocada para a inferência
|
| 78 |
+
def generate_response(prompt: str, max_tokens: int, temperature: float):
|
| 79 |
model = load_model()
|
| 80 |
+
output = model(prompt, max_tokens=max_tokens, temperature=temperature, stream=False)
|
| 81 |
+
return output["choices"][0]["text"]
|
| 82 |
|
| 83 |
+
@api.post("/v1/chat/completions")
|
| 84 |
+
async def chat_completions(req: ChatCompletionRequest):
|
| 85 |
+
# Formata o prompt para o modelo Dolphin (formato ChatML)
|
| 86 |
prompt = ""
|
| 87 |
for msg in req.messages:
|
| 88 |
if msg.role == "system":
|
|
|
|
| 93 |
prompt += f"<|im_start|>assistant\n{msg.content}<|im_end|>\n"
|
| 94 |
prompt += "<|im_start|>assistant\n"
|
| 95 |
|
| 96 |
+
# --- Modo de streaming ---
|
| 97 |
if req.stream:
|
| 98 |
async def stream_generator():
|
| 99 |
+
model = load_model()
|
| 100 |
output = model(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stream=True)
|
| 101 |
for chunk in output:
|
| 102 |
token = chunk["choices"][0]["text"]
|
|
|
|
| 109 |
yield "data: [DONE]\n\n"
|
| 110 |
return StreamingResponse(stream_generator(), media_type="text/event-stream")
|
| 111 |
|
| 112 |
+
# --- Modo normal ---
|
| 113 |
+
text = generate_response(prompt, req.max_tokens, req.temperature)
|
| 114 |
return {
|
| 115 |
"id": "chatcmpl-local",
|
| 116 |
"object": "chat.completion",
|
|
|
|
| 120 |
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
|
| 121 |
}
|
| 122 |
|
| 123 |
+
# --- Gradio (Interface web + API) ---
|
| 124 |
def dummy(x):
|
| 125 |
return x
|
| 126 |
|
|
|
|
| 129 |
# --- Monta o Gradio dentro do FastAPI ---
|
| 130 |
app = gr.mount_gradio_app(api, demo, path="/ui")
|
| 131 |
|
| 132 |
+
# --- Rodar (o Hugging Face inicia via `python app.py`) ---
|
| 133 |
if __name__ == "__main__":
|
| 134 |
load_model()
|
| 135 |
uvicorn.run(app, host="0.0.0.0", port=7860)
|