File size: 5,457 Bytes
66eca51 ecd61a2 66eca51 ecd61a2 66eca51 ecd61a2 66eca51 ecd61a2 66eca51 ecd61a2 66eca51 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | import os
from huggingface_hub import hf_hub_download
from fastapi import FastAPI, HTTPException
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from typing import List, Optional
import uvicorn
# -------------------- Configuration --------------------
MODEL_REPO = "Qwen/Qwen2.5-1.5B-Instruct-GGUF"
MODEL_FILE = "qwen2.5-1.5b-instruct-q4_k_m.gguf"
MODEL_PATH = "/data/model.gguf"
PORT = int(os.environ.get("PORT", 7860))
# -------------------- Download model if not present --------------------
def download_model():
if os.path.exists(MODEL_PATH) and os.path.getsize(MODEL_PATH) > 1_000_000:
print("Model already downloaded.")
return
print(f"Downloading {MODEL_FILE} from {MODEL_REPO} ...")
downloaded_path = hf_hub_download(
repo_id=MODEL_REPO,
filename=MODEL_FILE,
local_dir="/data",
local_dir_use_symlinks=False,
)
# Rename to standard path if needed
if downloaded_path != MODEL_PATH:
os.rename(downloaded_path, MODEL_PATH)
print("Download complete.")
download_model()
from llama_cpp import Llama
print("Loading model...")
llm = Llama(
model_path=MODEL_PATH,
n_ctx=2048,
n_threads=2, # HF free tier = 2 vCPUs
verbose=False,
)
print("Model ready.")
# -------------------- FastAPI setup --------------------
app = FastAPI(title="Eudora Orchestrator")
# Embedded HTML UI
html_content = '''
<!DOCTYPE html>
<html>
<head>
<title>Eudora Orchestrator Tester</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<style>
body { font-family: Arial; max-width: 700px; margin: 2rem auto; padding: 0 1rem; background: #f5f5f5; }
textarea, input, button { width: 100%; padding: 10px; margin: 5px 0; font-size: 16px; box-sizing: border-box; }
#response { background: white; padding: 15px; border-radius: 8px; white-space: pre-wrap; margin-top: 10px; min-height: 100px; }
</style>
</head>
<body>
<h2>🧠 Eudora Orchestrator – Chat UI</h2>
<div>
<label>System prompt (optional):</label>
<textarea id="systemPrompt" rows="4">You are an intent router for a voice assistant called Eudora.
Your only job is to decide which tools to call. Output only tool calls using the TOOL: format.
Available tools: get_route, find_restaurants, fuel_cost.</textarea>
</div>
<div>
<label>User message:</label>
<input id="userInput" type="text" value="I want breakfast near my college on the least polluted route" />
</div>
<button onclick="sendMessage()">Send</button>
<h3>Response:</h3>
<div id="response">Waiting...</div>
<script>
async function sendMessage() {
const systemPrompt = document.getElementById('systemPrompt').value.trim();
const userMessage = document.getElementById('userInput').value.trim();
const responseDiv = document.getElementById('response');
responseDiv.innerText = "Thinking...";
let messages = [];
if (systemPrompt) {
messages.push({ role: "system", content: systemPrompt });
}
messages.push({ role: "user", content: userMessage });
try {
const res = await fetch('/v1/chat/completions', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
messages: messages,
temperature: 0.7,
max_tokens: 256
})
});
const data = await res.json();
if (data.choices && data.choices.length > 0) {
responseDiv.innerText = data.choices[0].message.content;
} else {
responseDiv.innerText = 'Error: ' + JSON.stringify(data);
}
} catch (err) {
responseDiv.innerText = 'Network error: ' + err.message;
}
}
document.getElementById('userInput').addEventListener('keypress', function(e) {
if (e.key === 'Enter') sendMessage();
});
</script>
</body>
</html>
'''
class Message(BaseModel):
role: str
content: str
class ChatCompletionRequest(BaseModel):
messages: List[Message]
temperature: Optional[float] = 0.7
max_tokens: Optional[int] = 256
class ChatCompletionResponse(BaseModel):
choices: List[dict]
@app.get("/")
async def root():
return HTMLResponse(content=html_content)
@app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
async def chat_completions(request: ChatCompletionRequest):
msgs = [{"role": m.role, "content": m.content} for m in request.messages]
try:
output = llm.create_chat_completion(
messages=msgs,
temperature=request.temperature,
max_tokens=request.max_tokens,
)
assistant_msg = output["choices"][0]["message"]["content"]
return ChatCompletionResponse(
choices=[{"message": {"role": "assistant", "content": assistant_msg}}]
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.get("/health")
async def health():
return {"status": "ok", "model_loaded": True}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=PORT) |