2Btainment commited on
Commit
2b17a70
·
verified ·
1 Parent(s): 53b749c

Route chat to the Fuugi worker, keep OpenRouter as fallback

Browse files
Files changed (5) hide show
  1. .env.example +4 -1
  2. app/config.py +8 -0
  3. app/main.py +36 -14
  4. app/worker.py +95 -0
  5. tests/test_api.py +57 -0
.env.example CHANGED
@@ -41,6 +41,9 @@ MEMORY_MAX_ITEMS=50
41
  MEMORY_MAX_ITEM_CHARACTERS=500
42
  MEMORY_MAX_CONTEXT_CHARACTERS=4000
43
 
44
- # Disabled by default. Enable only behind known proxies and list their exact networks.
 
 
 
45
  TRUST_PROXY_HEADERS=false
46
  TRUSTED_PROXY_NETWORKS=127.0.0.1/32,::1/128
 
41
  MEMORY_MAX_ITEM_CHARACTERS=500
42
  MEMORY_MAX_CONTEXT_CHARACTERS=4000
43
 
44
+ # Server-side route to the private Fuugi worker. Never put the token in a public variable.
45
+ FUUGI_WORKER_URL=https://2btainment-fuugi-worker.hf.space
46
+ FUUGI_WORKER_TOKEN=replace-with-the-worker-secret
47
+ FUUGI_WORKER_TIMEOUT_SECONDS=45
48
  TRUST_PROXY_HEADERS=false
49
  TRUSTED_PROXY_NETWORKS=127.0.0.1/32,::1/128
app/config.py CHANGED
@@ -119,6 +119,9 @@ class Settings:
119
  memory_max_items: int = 50
120
  memory_max_item_characters: int = 500
121
  memory_max_context_characters: int = 4_000
 
 
 
122
 
123
  @classmethod
124
  def from_env(cls) -> Settings:
@@ -188,6 +191,11 @@ class Settings:
188
  memory_max_context_characters=_integer(
189
  "MEMORY_MAX_CONTEXT_CHARACTERS", 4_000, 100, 10_000
190
  ),
 
 
 
 
 
191
  )
192
  if settings.memory_enabled:
193
  if len(settings.memory_session_pepper) < 32:
 
119
  memory_max_items: int = 50
120
  memory_max_item_characters: int = 500
121
  memory_max_context_characters: int = 4_000
122
+ fuugi_worker_url: str = ""
123
+ fuugi_worker_token: str = field(default="", repr=False)
124
+ fuugi_worker_timeout_seconds: float = 45.0
125
 
126
  @classmethod
127
  def from_env(cls) -> Settings:
 
191
  memory_max_context_characters=_integer(
192
  "MEMORY_MAX_CONTEXT_CHARACTERS", 4_000, 100, 10_000
193
  ),
194
+ fuugi_worker_url=os.getenv("FUUGI_WORKER_URL", "").strip(),
195
+ fuugi_worker_token=os.getenv("FUUGI_WORKER_TOKEN", "").strip(),
196
+ fuugi_worker_timeout_seconds=_number(
197
+ "FUUGI_WORKER_TIMEOUT_SECONDS", 45.0, 5.0, 90.0
198
+ ),
199
  )
200
  if settings.memory_enabled:
201
  if len(settings.memory_session_pepper) < 32:
app/main.py CHANGED
@@ -30,6 +30,7 @@ from app.models import (
30
  MemoryListResponse,
31
  )
32
  from app.openrouter import OpenRouterClient, OpenRouterNotConfigured, OpenRouterUnavailable
 
33
  from app.research import ResearchStore, ResearchWriteError, classify_research
34
  from app.security import (
35
  BodySizeLimitMiddleware,
@@ -44,6 +45,7 @@ logger = logging.getLogger(__name__)
44
  def create_app(
45
  settings: Settings | None = None,
46
  transport: httpx.AsyncBaseTransport | None = None,
 
47
  ) -> FastAPI:
48
  active_settings = settings or Settings.from_env()
49
  memory_store = (
@@ -70,10 +72,13 @@ def create_app(
70
  @asynccontextmanager
71
  async def lifespan(application: FastAPI) -> AsyncIterator[None]:
72
  client = OpenRouterClient(active_settings, transport=transport)
 
73
  application.state.openrouter = client
 
74
  try:
75
  yield
76
  finally:
 
77
  await client.close()
78
 
79
  application = FastAPI(
@@ -121,12 +126,16 @@ def create_app(
121
 
122
  @application.get("/health")
123
  async def health() -> dict[str, str | bool]:
 
 
 
124
  return {
125
  "status": "ok",
126
  "service": "fuugi-api",
127
- "source": "openrouter",
128
- "model": active_settings.primary_model,
129
  "upstream_configured": bool(active_settings.openrouter_api_key),
 
130
  }
131
 
132
  def memory_access(request: Request) -> tuple[MemoryStore, str]:
@@ -283,17 +292,30 @@ def create_app(
283
  except ValueError as error:
284
  raise HTTPException(status_code=400, detail="Invalid request.") from error
285
 
286
- client: OpenRouterClient = request.app.state.openrouter
287
- try:
288
- completion = await client.complete(messages)
289
- except OpenRouterNotConfigured as error:
290
- raise HTTPException(
291
- status_code=503, detail="Service temporarily unavailable."
292
- ) from error
293
- except OpenRouterUnavailable as error:
294
- raise HTTPException(
295
- status_code=502, detail="Upstream service temporarily unavailable."
296
- ) from error
 
 
 
 
 
 
 
 
 
 
 
 
 
297
 
298
  research: ResearchStore | None = request.app.state.research
299
  if research is not None:
@@ -316,7 +338,7 @@ def create_app(
316
 
317
  return FuugiResponse(
318
  content=completion.content,
319
- source="openrouter",
320
  model=completion.model,
321
  )
322
 
 
30
  MemoryListResponse,
31
  )
32
  from app.openrouter import OpenRouterClient, OpenRouterNotConfigured, OpenRouterUnavailable
33
+ from app.worker import WorkerClient, WorkerNotConfigured, WorkerUnavailable
34
  from app.research import ResearchStore, ResearchWriteError, classify_research
35
  from app.security import (
36
  BodySizeLimitMiddleware,
 
45
  def create_app(
46
  settings: Settings | None = None,
47
  transport: httpx.AsyncBaseTransport | None = None,
48
+ worker_transport: httpx.AsyncBaseTransport | None = None,
49
  ) -> FastAPI:
50
  active_settings = settings or Settings.from_env()
51
  memory_store = (
 
72
  @asynccontextmanager
73
  async def lifespan(application: FastAPI) -> AsyncIterator[None]:
74
  client = OpenRouterClient(active_settings, transport=transport)
75
+ worker = WorkerClient(active_settings, transport=worker_transport)
76
  application.state.openrouter = client
77
+ application.state.worker = worker
78
  try:
79
  yield
80
  finally:
81
+ await worker.close()
82
  await client.close()
83
 
84
  application = FastAPI(
 
126
 
127
  @application.get("/health")
128
  async def health() -> dict[str, str | bool]:
129
+ worker_on = bool(
130
+ active_settings.fuugi_worker_url and active_settings.fuugi_worker_token
131
+ )
132
  return {
133
  "status": "ok",
134
  "service": "fuugi-api",
135
+ "source": "fuugi-worker" if worker_on else "openrouter",
136
+ "model": "fuugi-worker" if worker_on else active_settings.primary_model,
137
  "upstream_configured": bool(active_settings.openrouter_api_key),
138
+ "worker_configured": worker_on,
139
  }
140
 
141
  def memory_access(request: Request) -> tuple[MemoryStore, str]:
 
292
  except ValueError as error:
293
  raise HTTPException(status_code=400, detail="Invalid request.") from error
294
 
295
+ worker: WorkerClient = request.app.state.worker
296
+ completion = None
297
+ source = "openrouter"
298
+ if worker.configured:
299
+ try:
300
+ completion = await worker.complete(messages)
301
+ source = "fuugi-worker"
302
+ except WorkerNotConfigured:
303
+ completion = None
304
+ except WorkerUnavailable:
305
+ logger.warning("Fuugi worker unavailable; trying OpenRouter fallback.")
306
+ if completion is None:
307
+ client: OpenRouterClient = request.app.state.openrouter
308
+ try:
309
+ completion = await client.complete(messages)
310
+ source = "openrouter"
311
+ except OpenRouterNotConfigured as error:
312
+ raise HTTPException(
313
+ status_code=503, detail="Service temporarily unavailable."
314
+ ) from error
315
+ except OpenRouterUnavailable as error:
316
+ raise HTTPException(
317
+ status_code=502, detail="Upstream service temporarily unavailable."
318
+ ) from error
319
 
320
  research: ResearchStore | None = request.app.state.research
321
  if research is not None:
 
338
 
339
  return FuugiResponse(
340
  content=completion.content,
341
+ source=source,
342
  model=completion.model,
343
  )
344
 
app/worker.py ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from urllib.parse import urlsplit
4
+
5
+ import httpx
6
+
7
+ from app.config import Settings
8
+ from app.openrouter import Completion
9
+
10
+ WORKER_HOST = "2btainment-fuugi-worker.hf.space"
11
+
12
+
13
+ class WorkerNotConfigured(Exception):
14
+ pass
15
+
16
+
17
+ class WorkerUnavailable(Exception):
18
+ pass
19
+
20
+
21
+ def worker_endpoint(url: str) -> str:
22
+ parsed = urlsplit(url.strip())
23
+ if (
24
+ parsed.scheme != "https"
25
+ or parsed.hostname != WORKER_HOST
26
+ or parsed.port not in {None, 443}
27
+ or parsed.username
28
+ or parsed.password
29
+ or parsed.query
30
+ or parsed.fragment
31
+ or parsed.path not in {"", "/"}
32
+ ):
33
+ raise WorkerNotConfigured
34
+ return f"https://{WORKER_HOST}/v1/chat/completions"
35
+
36
+
37
+ class WorkerClient:
38
+ def __init__(
39
+ self,
40
+ settings: Settings,
41
+ transport: httpx.AsyncBaseTransport | None = None,
42
+ ) -> None:
43
+ self.settings = settings
44
+ self._endpoint = ""
45
+ if settings.fuugi_worker_url and settings.fuugi_worker_token:
46
+ try:
47
+ self._endpoint = worker_endpoint(settings.fuugi_worker_url)
48
+ except WorkerNotConfigured:
49
+ self._endpoint = ""
50
+ timeout = httpx.Timeout(settings.fuugi_worker_timeout_seconds)
51
+ self._client = httpx.AsyncClient(
52
+ transport=transport,
53
+ timeout=timeout,
54
+ follow_redirects=False,
55
+ )
56
+
57
+ @property
58
+ def configured(self) -> bool:
59
+ return bool(self._endpoint and self.settings.fuugi_worker_token)
60
+
61
+ async def close(self) -> None:
62
+ await self._client.aclose()
63
+
64
+ async def complete(self, messages: list[dict[str, str]]) -> Completion:
65
+ if not self.configured:
66
+ raise WorkerNotConfigured
67
+ headers = {
68
+ "Authorization": f"Bearer {self.settings.fuugi_worker_token}",
69
+ "Content-Type": "application/json",
70
+ }
71
+ payload = {
72
+ "messages": messages,
73
+ "temperature": 0.3,
74
+ "max_tokens": min(self.settings.max_output_tokens, 220),
75
+ }
76
+ try:
77
+ response = await self._client.post(self._endpoint, headers=headers, json=payload)
78
+ except httpx.HTTPError as error:
79
+ raise WorkerUnavailable from error
80
+ if response.status_code != 200:
81
+ raise WorkerUnavailable
82
+ try:
83
+ data = response.json()
84
+ content = data["choices"][0]["message"]["content"]
85
+ except (KeyError, IndexError, TypeError, ValueError) as error:
86
+ raise WorkerUnavailable from error
87
+ if not isinstance(content, str) or not content.strip():
88
+ raise WorkerUnavailable
89
+ model = data.get("model")
90
+ if not isinstance(model, str) or not model.strip():
91
+ model = "fuugi-worker"
92
+ return Completion(
93
+ content=content.strip()[: self.settings.max_output_characters],
94
+ model=model.strip()[:200],
95
+ )
tests/test_api.py CHANGED
@@ -303,3 +303,60 @@ def test_system_role_in_history_is_rejected() -> None:
303
 
304
  assert response.status_code == 400
305
  assert response.json() == {"detail": "Invalid request."}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
 
304
  assert response.status_code == 400
305
  assert response.json() == {"detail": "Invalid request."}
306
+
307
+
308
+ WORKER_URL = "https://2btainment-fuugi-worker.hf.space"
309
+
310
+
311
+ def worker_transport(status: int = 200, content: str = "Roster: vier Acts.") -> httpx.MockTransport:
312
+ async def handler(request: httpx.Request) -> httpx.Response:
313
+ assert request.url.host == "2btainment-fuugi-worker.hf.space"
314
+ assert request.headers["authorization"] == "Bearer worker-test-token"
315
+ if status != 200:
316
+ return httpx.Response(status, json={"detail": "down"})
317
+ return httpx.Response(
318
+ 200,
319
+ json={
320
+ "model": "qwen2.5-3b-instruct-q4_k_m.gguf",
321
+ "choices": [{"message": {"content": content}}],
322
+ },
323
+ )
324
+
325
+ return httpx.MockTransport(handler)
326
+
327
+
328
+ def test_worker_is_preferred_when_configured() -> None:
329
+ app = create_app(
330
+ settings(
331
+ fuugi_worker_url=WORKER_URL,
332
+ fuugi_worker_token="worker-test-token",
333
+ ),
334
+ transport=success_transport(),
335
+ worker_transport=worker_transport(),
336
+ )
337
+ with TestClient(app) as client:
338
+ health = client.get("/health")
339
+ response = client.post("/api/fuugi", json={"message": "Wer ist im Roster?"})
340
+
341
+ assert health.json()["worker_configured"] is True
342
+ assert response.status_code == 200
343
+ body = response.json()
344
+ assert body["source"] == "fuugi-worker"
345
+ assert body["model"] == "qwen2.5-3b-instruct-q4_k_m.gguf"
346
+ assert "worker-test-token" not in response.text
347
+
348
+
349
+ def test_worker_failure_falls_back_to_openrouter() -> None:
350
+ app = create_app(
351
+ settings(
352
+ fuugi_worker_url=WORKER_URL,
353
+ fuugi_worker_token="worker-test-token",
354
+ ),
355
+ transport=success_transport(),
356
+ worker_transport=worker_transport(status=503),
357
+ )
358
+ with TestClient(app) as client:
359
+ response = client.post("/api/fuugi", json={"message": "Hallo"})
360
+
361
+ assert response.status_code == 200
362
+ assert response.json()["source"] == "openrouter"