Spaces:
Sleeping
Sleeping
File size: 3,056 Bytes
a1e4405 775cbe6 9e95393 775cbe6 a1e4405 775cbe6 a1e4405 9e95393 775cbe6 9e95393 775cbe6 9e95393 a1e4405 775cbe6 a1e4405 775cbe6 a1e4405 9e95393 775cbe6 9e95393 a1e4405 775cbe6 a1e4405 775cbe6 a1e4405 775cbe6 a1e4405 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | from __future__ import annotations
import json
from collections.abc import Sequence
import httpx
HF_MODEL_4B = "llm-jp/llm-jp-3-3.7b-instruct3"
HF_MODEL_8B = "llm-jp/llm-jp-4-8b-instruct"
HF_ALLOWED_MODELS = frozenset({HF_MODEL_4B, HF_MODEL_8B})
HF_CHAT_API_CONSTRUCTIVE = "http://127.0.0.1:8081/v1/chat/completions"
HF_CHAT_API_ADVERSARIAL = "http://127.0.0.1:8082/v1/chat/completions"
HF_CHAT_API_EVIDENCE = "http://127.0.0.1:8083/v1/chat/completions"
HF_CHAT_API = HF_CHAT_API_CONSTRUCTIVE # compatibility alias only
def _default_local_url(model_id: str) -> str:
if model_id == HF_MODEL_4B:
return HF_CHAT_API_CONSTRUCTIVE
if model_id == HF_MODEL_8B:
return HF_CHAT_API_EVIDENCE
raise ValueError(f"model_drift:{model_id}")
class HFChatClient:
"""Role-scoped OpenAI-compatible client backed only by local llama.cpp.
Production rejects non-loopback endpoints so HF Inference Providers cannot
be used accidentally and cannot consume inference credits.
"""
def __init__(
self,
*,
token: str = "",
model_id: str,
api_url: str | None = None,
timeout_seconds: float = 600.0,
client: httpx.AsyncClient | None = None,
):
if model_id not in HF_ALLOWED_MODELS:
raise ValueError(f"model_drift:{model_id}")
resolved_url = (api_url or _default_local_url(model_id)).strip()
if not resolved_url.startswith(("http://127.0.0.1", "http://localhost")):
raise ValueError("remote_inference_endpoint_forbidden")
self.model_id = model_id
self.api_url = resolved_url
self._owned = client is None
self._client = client or httpx.AsyncClient(timeout=timeout_seconds)
async def chat_json(
self,
messages: Sequence[dict[str, str]],
*,
max_tokens: int = 900,
) -> dict[str, object]:
payload = {
"model": self.model_id,
"messages": list(messages),
"response_format": {"type": "json_object"},
"stream": False,
"temperature": 0.1,
"top_p": 0.9,
"max_tokens": max_tokens,
}
response = await self._client.post(self.api_url, json=payload)
response.raise_for_status()
body = response.json()
choices = body.get("choices") or []
if not choices:
raise RuntimeError("model_empty_choices")
content = choices[0].get("message", {}).get("content")
if not isinstance(content, str) or not content.strip():
raise RuntimeError("model_empty_content")
text = content.strip()
first = text.find("{")
last = text.rfind("}")
if first >= 0 and last >= first:
text = text[first : last + 1]
decoded = json.loads(text)
if not isinstance(decoded, dict):
raise RuntimeError("model_non_object_json")
return decoded
async def aclose(self) -> None:
if self._owned:
await self._client.aclose()
|