Spaces:
Running
Running
Route chat to the Fuugi worker, keep OpenRouter as fallback
Browse files- .env.example +4 -1
- app/config.py +8 -0
- app/main.py +36 -14
- app/worker.py +95 -0
- tests/test_api.py +57 -0
.env.example
CHANGED
|
@@ -41,6 +41,9 @@ MEMORY_MAX_ITEMS=50
|
|
| 41 |
MEMORY_MAX_ITEM_CHARACTERS=500
|
| 42 |
MEMORY_MAX_CONTEXT_CHARACTERS=4000
|
| 43 |
|
| 44 |
-
#
|
|
|
|
|
|
|
|
|
|
| 45 |
TRUST_PROXY_HEADERS=false
|
| 46 |
TRUSTED_PROXY_NETWORKS=127.0.0.1/32,::1/128
|
|
|
|
| 41 |
MEMORY_MAX_ITEM_CHARACTERS=500
|
| 42 |
MEMORY_MAX_CONTEXT_CHARACTERS=4000
|
| 43 |
|
| 44 |
+
# Server-side route to the private Fuugi worker. Never put the token in a public variable.
|
| 45 |
+
FUUGI_WORKER_URL=https://2btainment-fuugi-worker.hf.space
|
| 46 |
+
FUUGI_WORKER_TOKEN=replace-with-the-worker-secret
|
| 47 |
+
FUUGI_WORKER_TIMEOUT_SECONDS=45
|
| 48 |
TRUST_PROXY_HEADERS=false
|
| 49 |
TRUSTED_PROXY_NETWORKS=127.0.0.1/32,::1/128
|
app/config.py
CHANGED
|
@@ -119,6 +119,9 @@ class Settings:
|
|
| 119 |
memory_max_items: int = 50
|
| 120 |
memory_max_item_characters: int = 500
|
| 121 |
memory_max_context_characters: int = 4_000
|
|
|
|
|
|
|
|
|
|
| 122 |
|
| 123 |
@classmethod
|
| 124 |
def from_env(cls) -> Settings:
|
|
@@ -188,6 +191,11 @@ class Settings:
|
|
| 188 |
memory_max_context_characters=_integer(
|
| 189 |
"MEMORY_MAX_CONTEXT_CHARACTERS", 4_000, 100, 10_000
|
| 190 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
)
|
| 192 |
if settings.memory_enabled:
|
| 193 |
if len(settings.memory_session_pepper) < 32:
|
|
|
|
| 119 |
memory_max_items: int = 50
|
| 120 |
memory_max_item_characters: int = 500
|
| 121 |
memory_max_context_characters: int = 4_000
|
| 122 |
+
fuugi_worker_url: str = ""
|
| 123 |
+
fuugi_worker_token: str = field(default="", repr=False)
|
| 124 |
+
fuugi_worker_timeout_seconds: float = 45.0
|
| 125 |
|
| 126 |
@classmethod
|
| 127 |
def from_env(cls) -> Settings:
|
|
|
|
| 191 |
memory_max_context_characters=_integer(
|
| 192 |
"MEMORY_MAX_CONTEXT_CHARACTERS", 4_000, 100, 10_000
|
| 193 |
),
|
| 194 |
+
fuugi_worker_url=os.getenv("FUUGI_WORKER_URL", "").strip(),
|
| 195 |
+
fuugi_worker_token=os.getenv("FUUGI_WORKER_TOKEN", "").strip(),
|
| 196 |
+
fuugi_worker_timeout_seconds=_number(
|
| 197 |
+
"FUUGI_WORKER_TIMEOUT_SECONDS", 45.0, 5.0, 90.0
|
| 198 |
+
),
|
| 199 |
)
|
| 200 |
if settings.memory_enabled:
|
| 201 |
if len(settings.memory_session_pepper) < 32:
|
app/main.py
CHANGED
|
@@ -30,6 +30,7 @@ from app.models import (
|
|
| 30 |
MemoryListResponse,
|
| 31 |
)
|
| 32 |
from app.openrouter import OpenRouterClient, OpenRouterNotConfigured, OpenRouterUnavailable
|
|
|
|
| 33 |
from app.research import ResearchStore, ResearchWriteError, classify_research
|
| 34 |
from app.security import (
|
| 35 |
BodySizeLimitMiddleware,
|
|
@@ -44,6 +45,7 @@ logger = logging.getLogger(__name__)
|
|
| 44 |
def create_app(
|
| 45 |
settings: Settings | None = None,
|
| 46 |
transport: httpx.AsyncBaseTransport | None = None,
|
|
|
|
| 47 |
) -> FastAPI:
|
| 48 |
active_settings = settings or Settings.from_env()
|
| 49 |
memory_store = (
|
|
@@ -70,10 +72,13 @@ def create_app(
|
|
| 70 |
@asynccontextmanager
|
| 71 |
async def lifespan(application: FastAPI) -> AsyncIterator[None]:
|
| 72 |
client = OpenRouterClient(active_settings, transport=transport)
|
|
|
|
| 73 |
application.state.openrouter = client
|
|
|
|
| 74 |
try:
|
| 75 |
yield
|
| 76 |
finally:
|
|
|
|
| 77 |
await client.close()
|
| 78 |
|
| 79 |
application = FastAPI(
|
|
@@ -121,12 +126,16 @@ def create_app(
|
|
| 121 |
|
| 122 |
@application.get("/health")
|
| 123 |
async def health() -> dict[str, str | bool]:
|
|
|
|
|
|
|
|
|
|
| 124 |
return {
|
| 125 |
"status": "ok",
|
| 126 |
"service": "fuugi-api",
|
| 127 |
-
"source": "openrouter",
|
| 128 |
-
"model": active_settings.primary_model,
|
| 129 |
"upstream_configured": bool(active_settings.openrouter_api_key),
|
|
|
|
| 130 |
}
|
| 131 |
|
| 132 |
def memory_access(request: Request) -> tuple[MemoryStore, str]:
|
|
@@ -283,17 +292,30 @@ def create_app(
|
|
| 283 |
except ValueError as error:
|
| 284 |
raise HTTPException(status_code=400, detail="Invalid request.") from error
|
| 285 |
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
|
| 298 |
research: ResearchStore | None = request.app.state.research
|
| 299 |
if research is not None:
|
|
@@ -316,7 +338,7 @@ def create_app(
|
|
| 316 |
|
| 317 |
return FuugiResponse(
|
| 318 |
content=completion.content,
|
| 319 |
-
source=
|
| 320 |
model=completion.model,
|
| 321 |
)
|
| 322 |
|
|
|
|
| 30 |
MemoryListResponse,
|
| 31 |
)
|
| 32 |
from app.openrouter import OpenRouterClient, OpenRouterNotConfigured, OpenRouterUnavailable
|
| 33 |
+
from app.worker import WorkerClient, WorkerNotConfigured, WorkerUnavailable
|
| 34 |
from app.research import ResearchStore, ResearchWriteError, classify_research
|
| 35 |
from app.security import (
|
| 36 |
BodySizeLimitMiddleware,
|
|
|
|
| 45 |
def create_app(
|
| 46 |
settings: Settings | None = None,
|
| 47 |
transport: httpx.AsyncBaseTransport | None = None,
|
| 48 |
+
worker_transport: httpx.AsyncBaseTransport | None = None,
|
| 49 |
) -> FastAPI:
|
| 50 |
active_settings = settings or Settings.from_env()
|
| 51 |
memory_store = (
|
|
|
|
| 72 |
@asynccontextmanager
|
| 73 |
async def lifespan(application: FastAPI) -> AsyncIterator[None]:
|
| 74 |
client = OpenRouterClient(active_settings, transport=transport)
|
| 75 |
+
worker = WorkerClient(active_settings, transport=worker_transport)
|
| 76 |
application.state.openrouter = client
|
| 77 |
+
application.state.worker = worker
|
| 78 |
try:
|
| 79 |
yield
|
| 80 |
finally:
|
| 81 |
+
await worker.close()
|
| 82 |
await client.close()
|
| 83 |
|
| 84 |
application = FastAPI(
|
|
|
|
| 126 |
|
| 127 |
@application.get("/health")
|
| 128 |
async def health() -> dict[str, str | bool]:
|
| 129 |
+
worker_on = bool(
|
| 130 |
+
active_settings.fuugi_worker_url and active_settings.fuugi_worker_token
|
| 131 |
+
)
|
| 132 |
return {
|
| 133 |
"status": "ok",
|
| 134 |
"service": "fuugi-api",
|
| 135 |
+
"source": "fuugi-worker" if worker_on else "openrouter",
|
| 136 |
+
"model": "fuugi-worker" if worker_on else active_settings.primary_model,
|
| 137 |
"upstream_configured": bool(active_settings.openrouter_api_key),
|
| 138 |
+
"worker_configured": worker_on,
|
| 139 |
}
|
| 140 |
|
| 141 |
def memory_access(request: Request) -> tuple[MemoryStore, str]:
|
|
|
|
| 292 |
except ValueError as error:
|
| 293 |
raise HTTPException(status_code=400, detail="Invalid request.") from error
|
| 294 |
|
| 295 |
+
worker: WorkerClient = request.app.state.worker
|
| 296 |
+
completion = None
|
| 297 |
+
source = "openrouter"
|
| 298 |
+
if worker.configured:
|
| 299 |
+
try:
|
| 300 |
+
completion = await worker.complete(messages)
|
| 301 |
+
source = "fuugi-worker"
|
| 302 |
+
except WorkerNotConfigured:
|
| 303 |
+
completion = None
|
| 304 |
+
except WorkerUnavailable:
|
| 305 |
+
logger.warning("Fuugi worker unavailable; trying OpenRouter fallback.")
|
| 306 |
+
if completion is None:
|
| 307 |
+
client: OpenRouterClient = request.app.state.openrouter
|
| 308 |
+
try:
|
| 309 |
+
completion = await client.complete(messages)
|
| 310 |
+
source = "openrouter"
|
| 311 |
+
except OpenRouterNotConfigured as error:
|
| 312 |
+
raise HTTPException(
|
| 313 |
+
status_code=503, detail="Service temporarily unavailable."
|
| 314 |
+
) from error
|
| 315 |
+
except OpenRouterUnavailable as error:
|
| 316 |
+
raise HTTPException(
|
| 317 |
+
status_code=502, detail="Upstream service temporarily unavailable."
|
| 318 |
+
) from error
|
| 319 |
|
| 320 |
research: ResearchStore | None = request.app.state.research
|
| 321 |
if research is not None:
|
|
|
|
| 338 |
|
| 339 |
return FuugiResponse(
|
| 340 |
content=completion.content,
|
| 341 |
+
source=source,
|
| 342 |
model=completion.model,
|
| 343 |
)
|
| 344 |
|
app/worker.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from urllib.parse import urlsplit
|
| 4 |
+
|
| 5 |
+
import httpx
|
| 6 |
+
|
| 7 |
+
from app.config import Settings
|
| 8 |
+
from app.openrouter import Completion
|
| 9 |
+
|
| 10 |
+
WORKER_HOST = "2btainment-fuugi-worker.hf.space"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class WorkerNotConfigured(Exception):
|
| 14 |
+
pass
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class WorkerUnavailable(Exception):
|
| 18 |
+
pass
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def worker_endpoint(url: str) -> str:
|
| 22 |
+
parsed = urlsplit(url.strip())
|
| 23 |
+
if (
|
| 24 |
+
parsed.scheme != "https"
|
| 25 |
+
or parsed.hostname != WORKER_HOST
|
| 26 |
+
or parsed.port not in {None, 443}
|
| 27 |
+
or parsed.username
|
| 28 |
+
or parsed.password
|
| 29 |
+
or parsed.query
|
| 30 |
+
or parsed.fragment
|
| 31 |
+
or parsed.path not in {"", "/"}
|
| 32 |
+
):
|
| 33 |
+
raise WorkerNotConfigured
|
| 34 |
+
return f"https://{WORKER_HOST}/v1/chat/completions"
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class WorkerClient:
|
| 38 |
+
def __init__(
|
| 39 |
+
self,
|
| 40 |
+
settings: Settings,
|
| 41 |
+
transport: httpx.AsyncBaseTransport | None = None,
|
| 42 |
+
) -> None:
|
| 43 |
+
self.settings = settings
|
| 44 |
+
self._endpoint = ""
|
| 45 |
+
if settings.fuugi_worker_url and settings.fuugi_worker_token:
|
| 46 |
+
try:
|
| 47 |
+
self._endpoint = worker_endpoint(settings.fuugi_worker_url)
|
| 48 |
+
except WorkerNotConfigured:
|
| 49 |
+
self._endpoint = ""
|
| 50 |
+
timeout = httpx.Timeout(settings.fuugi_worker_timeout_seconds)
|
| 51 |
+
self._client = httpx.AsyncClient(
|
| 52 |
+
transport=transport,
|
| 53 |
+
timeout=timeout,
|
| 54 |
+
follow_redirects=False,
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
@property
|
| 58 |
+
def configured(self) -> bool:
|
| 59 |
+
return bool(self._endpoint and self.settings.fuugi_worker_token)
|
| 60 |
+
|
| 61 |
+
async def close(self) -> None:
|
| 62 |
+
await self._client.aclose()
|
| 63 |
+
|
| 64 |
+
async def complete(self, messages: list[dict[str, str]]) -> Completion:
|
| 65 |
+
if not self.configured:
|
| 66 |
+
raise WorkerNotConfigured
|
| 67 |
+
headers = {
|
| 68 |
+
"Authorization": f"Bearer {self.settings.fuugi_worker_token}",
|
| 69 |
+
"Content-Type": "application/json",
|
| 70 |
+
}
|
| 71 |
+
payload = {
|
| 72 |
+
"messages": messages,
|
| 73 |
+
"temperature": 0.3,
|
| 74 |
+
"max_tokens": min(self.settings.max_output_tokens, 220),
|
| 75 |
+
}
|
| 76 |
+
try:
|
| 77 |
+
response = await self._client.post(self._endpoint, headers=headers, json=payload)
|
| 78 |
+
except httpx.HTTPError as error:
|
| 79 |
+
raise WorkerUnavailable from error
|
| 80 |
+
if response.status_code != 200:
|
| 81 |
+
raise WorkerUnavailable
|
| 82 |
+
try:
|
| 83 |
+
data = response.json()
|
| 84 |
+
content = data["choices"][0]["message"]["content"]
|
| 85 |
+
except (KeyError, IndexError, TypeError, ValueError) as error:
|
| 86 |
+
raise WorkerUnavailable from error
|
| 87 |
+
if not isinstance(content, str) or not content.strip():
|
| 88 |
+
raise WorkerUnavailable
|
| 89 |
+
model = data.get("model")
|
| 90 |
+
if not isinstance(model, str) or not model.strip():
|
| 91 |
+
model = "fuugi-worker"
|
| 92 |
+
return Completion(
|
| 93 |
+
content=content.strip()[: self.settings.max_output_characters],
|
| 94 |
+
model=model.strip()[:200],
|
| 95 |
+
)
|
tests/test_api.py
CHANGED
|
@@ -303,3 +303,60 @@ def test_system_role_in_history_is_rejected() -> None:
|
|
| 303 |
|
| 304 |
assert response.status_code == 400
|
| 305 |
assert response.json() == {"detail": "Invalid request."}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
|
| 304 |
assert response.status_code == 400
|
| 305 |
assert response.json() == {"detail": "Invalid request."}
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
WORKER_URL = "https://2btainment-fuugi-worker.hf.space"
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def worker_transport(status: int = 200, content: str = "Roster: vier Acts.") -> httpx.MockTransport:
|
| 312 |
+
async def handler(request: httpx.Request) -> httpx.Response:
|
| 313 |
+
assert request.url.host == "2btainment-fuugi-worker.hf.space"
|
| 314 |
+
assert request.headers["authorization"] == "Bearer worker-test-token"
|
| 315 |
+
if status != 200:
|
| 316 |
+
return httpx.Response(status, json={"detail": "down"})
|
| 317 |
+
return httpx.Response(
|
| 318 |
+
200,
|
| 319 |
+
json={
|
| 320 |
+
"model": "qwen2.5-3b-instruct-q4_k_m.gguf",
|
| 321 |
+
"choices": [{"message": {"content": content}}],
|
| 322 |
+
},
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
return httpx.MockTransport(handler)
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def test_worker_is_preferred_when_configured() -> None:
|
| 329 |
+
app = create_app(
|
| 330 |
+
settings(
|
| 331 |
+
fuugi_worker_url=WORKER_URL,
|
| 332 |
+
fuugi_worker_token="worker-test-token",
|
| 333 |
+
),
|
| 334 |
+
transport=success_transport(),
|
| 335 |
+
worker_transport=worker_transport(),
|
| 336 |
+
)
|
| 337 |
+
with TestClient(app) as client:
|
| 338 |
+
health = client.get("/health")
|
| 339 |
+
response = client.post("/api/fuugi", json={"message": "Wer ist im Roster?"})
|
| 340 |
+
|
| 341 |
+
assert health.json()["worker_configured"] is True
|
| 342 |
+
assert response.status_code == 200
|
| 343 |
+
body = response.json()
|
| 344 |
+
assert body["source"] == "fuugi-worker"
|
| 345 |
+
assert body["model"] == "qwen2.5-3b-instruct-q4_k_m.gguf"
|
| 346 |
+
assert "worker-test-token" not in response.text
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def test_worker_failure_falls_back_to_openrouter() -> None:
|
| 350 |
+
app = create_app(
|
| 351 |
+
settings(
|
| 352 |
+
fuugi_worker_url=WORKER_URL,
|
| 353 |
+
fuugi_worker_token="worker-test-token",
|
| 354 |
+
),
|
| 355 |
+
transport=success_transport(),
|
| 356 |
+
worker_transport=worker_transport(status=503),
|
| 357 |
+
)
|
| 358 |
+
with TestClient(app) as client:
|
| 359 |
+
response = client.post("/api/fuugi", json={"message": "Hallo"})
|
| 360 |
+
|
| 361 |
+
assert response.status_code == 200
|
| 362 |
+
assert response.json()["source"] == "openrouter"
|