| import os |
| from huggingface_hub import hf_hub_download |
| from fastapi import FastAPI, HTTPException |
| from fastapi.responses import HTMLResponse |
| from pydantic import BaseModel |
| from typing import List, Optional |
| import uvicorn |
|
|
| |
| MODEL_REPO = "Qwen/Qwen2.5-1.5B-Instruct-GGUF" |
| MODEL_FILE = "qwen2.5-1.5b-instruct-q4_k_m.gguf" |
| MODEL_PATH = "/data/model.gguf" |
| PORT = int(os.environ.get("PORT", 7860)) |
|
|
| |
| def download_model(): |
| if os.path.exists(MODEL_PATH) and os.path.getsize(MODEL_PATH) > 1_000_000: |
| print("Model already downloaded.") |
| return |
| print(f"Downloading {MODEL_FILE} from {MODEL_REPO} ...") |
| downloaded_path = hf_hub_download( |
| repo_id=MODEL_REPO, |
| filename=MODEL_FILE, |
| local_dir="/data", |
| local_dir_use_symlinks=False, |
| ) |
| |
| if downloaded_path != MODEL_PATH: |
| os.rename(downloaded_path, MODEL_PATH) |
| print("Download complete.") |
|
|
| download_model() |
| from llama_cpp import Llama |
|
|
| print("Loading model...") |
| llm = Llama( |
| model_path=MODEL_PATH, |
| n_ctx=2048, |
| n_threads=2, |
| verbose=False, |
| ) |
| print("Model ready.") |
|
|
| |
| app = FastAPI(title="Eudora Orchestrator") |
|
|
| |
| html_content = ''' |
| <!DOCTYPE html> |
| <html> |
| <head> |
| <title>Eudora Orchestrator Tester</title> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <style> |
| body { font-family: Arial; max-width: 700px; margin: 2rem auto; padding: 0 1rem; background: #f5f5f5; } |
| textarea, input, button { width: 100%; padding: 10px; margin: 5px 0; font-size: 16px; box-sizing: border-box; } |
| #response { background: white; padding: 15px; border-radius: 8px; white-space: pre-wrap; margin-top: 10px; min-height: 100px; } |
| </style> |
| </head> |
| <body> |
| <h2>🧠 Eudora Orchestrator – Chat UI</h2> |
| <div> |
| <label>System prompt (optional):</label> |
| <textarea id="systemPrompt" rows="4">You are an intent router for a voice assistant called Eudora. |
| Your only job is to decide which tools to call. Output only tool calls using the TOOL: format. |
| Available tools: get_route, find_restaurants, fuel_cost.</textarea> |
| </div> |
| <div> |
| <label>User message:</label> |
| <input id="userInput" type="text" value="I want breakfast near my college on the least polluted route" /> |
| </div> |
| <button onclick="sendMessage()">Send</button> |
| <h3>Response:</h3> |
| <div id="response">Waiting...</div> |
| |
| <script> |
| async function sendMessage() { |
| const systemPrompt = document.getElementById('systemPrompt').value.trim(); |
| const userMessage = document.getElementById('userInput').value.trim(); |
| const responseDiv = document.getElementById('response'); |
| responseDiv.innerText = "Thinking..."; |
| |
| let messages = []; |
| if (systemPrompt) { |
| messages.push({ role: "system", content: systemPrompt }); |
| } |
| messages.push({ role: "user", content: userMessage }); |
| |
| try { |
| const res = await fetch('/v1/chat/completions', { |
| method: 'POST', |
| headers: { 'Content-Type': 'application/json' }, |
| body: JSON.stringify({ |
| messages: messages, |
| temperature: 0.7, |
| max_tokens: 256 |
| }) |
| }); |
| const data = await res.json(); |
| if (data.choices && data.choices.length > 0) { |
| responseDiv.innerText = data.choices[0].message.content; |
| } else { |
| responseDiv.innerText = 'Error: ' + JSON.stringify(data); |
| } |
| } catch (err) { |
| responseDiv.innerText = 'Network error: ' + err.message; |
| } |
| } |
| |
| document.getElementById('userInput').addEventListener('keypress', function(e) { |
| if (e.key === 'Enter') sendMessage(); |
| }); |
| </script> |
| </body> |
| </html> |
| ''' |
|
|
| class Message(BaseModel): |
| role: str |
| content: str |
|
|
| class ChatCompletionRequest(BaseModel): |
| messages: List[Message] |
| temperature: Optional[float] = 0.7 |
| max_tokens: Optional[int] = 256 |
|
|
| class ChatCompletionResponse(BaseModel): |
| choices: List[dict] |
|
|
| @app.get("/") |
| async def root(): |
| return HTMLResponse(content=html_content) |
|
|
| @app.post("/v1/chat/completions", response_model=ChatCompletionResponse) |
| async def chat_completions(request: ChatCompletionRequest): |
| msgs = [{"role": m.role, "content": m.content} for m in request.messages] |
| try: |
| output = llm.create_chat_completion( |
| messages=msgs, |
| temperature=request.temperature, |
| max_tokens=request.max_tokens, |
| ) |
| assistant_msg = output["choices"][0]["message"]["content"] |
| return ChatCompletionResponse( |
| choices=[{"message": {"role": "assistant", "content": assistant_msg}}] |
| ) |
| except Exception as e: |
| raise HTTPException(status_code=500, detail=str(e)) |
|
|
| @app.get("/health") |
| async def health(): |
| return {"status": "ok", "model_loaded": True} |
|
|
| if __name__ == "__main__": |
| uvicorn.run(app, host="0.0.0.0", port=PORT) |
