Instructions to use patdev/k3-a40-bootstrap with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use patdev/k3-a40-bootstrap with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf patdev/k3-a40-bootstrap:BF16 # Run inference directly in the terminal: llama cli -hf patdev/k3-a40-bootstrap:BF16
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf patdev/k3-a40-bootstrap:BF16 # Run inference directly in the terminal: llama cli -hf patdev/k3-a40-bootstrap:BF16
Use pre-built binary
# Download pre-built binary from: # https://github.com/ggerganov/llama.cpp/releases # Start a local OpenAI-compatible server with a web UI: ./llama-server -hf patdev/k3-a40-bootstrap:BF16 # Run inference directly in the terminal: ./llama-cli -hf patdev/k3-a40-bootstrap:BF16
Build from source code
git clone https://github.com/ggerganov/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-server llama-cli # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama-server -hf patdev/k3-a40-bootstrap:BF16 # Run inference directly in the terminal: ./build/bin/llama-cli -hf patdev/k3-a40-bootstrap:BF16
Use Docker
docker model run hf.co/patdev/k3-a40-bootstrap:BF16
- LM Studio
- Jan
- Ollama
How to use patdev/k3-a40-bootstrap with Ollama:
ollama run hf.co/patdev/k3-a40-bootstrap:BF16
- Unsloth Desktop
- Docker Model Runner
How to use patdev/k3-a40-bootstrap with Docker Model Runner:
docker model run hf.co/patdev/k3-a40-bootstrap:BF16
- Lemonade
How to use patdev/k3-a40-bootstrap with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull patdev/k3-a40-bootstrap:BF16
Run and chat with the model
lemonade run user.k3-a40-bootstrap-BF16
List all available models
lemonade list
- Atomic Chat
Upload anthropic_proxy.py with huggingface_hub
Browse files- anthropic_proxy.py +47 -6
anthropic_proxy.py
CHANGED
|
@@ -25,6 +25,18 @@ UPSTREAM = os.environ.get("VL_UPSTREAM", "http://127.0.0.1:8080")
|
|
| 25 |
MODEL = os.environ.get("VL_SERVED_NAME", "qwen")
|
| 26 |
TIMEOUT = float(os.environ.get("VL_TIMEOUT", "1800"))
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
app = FastAPI(title="anthropic-bridge")
|
| 29 |
_client = httpx.AsyncClient(base_url=UPSTREAM, timeout=TIMEOUT)
|
| 30 |
|
|
@@ -311,17 +323,46 @@ async def health():
|
|
| 311 |
|
| 312 |
|
| 313 |
@app.get("/v1/models")
|
| 314 |
-
async def models():
|
| 315 |
-
"""
|
| 316 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
try:
|
| 318 |
r = await _client.get("/v1/models", timeout=10)
|
| 319 |
if r.status_code == 200:
|
| 320 |
-
|
| 321 |
except Exception:
|
| 322 |
pass
|
| 323 |
-
|
| 324 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
|
| 326 |
# ------------------------------------------------------------------ passe-plat
|
| 327 |
# Le pod n'expose qu'un port. Le pont le prend (c'est l'API que Claude Code
|
|
|
|
| 25 |
MODEL = os.environ.get("VL_SERVED_NAME", "qwen")
|
| 26 |
TIMEOUT = float(os.environ.get("VL_TIMEOUT", "1800"))
|
| 27 |
|
| 28 |
+
# Claude Code refuse tout identifiant de modele qui ne commence pas par
|
| 29 |
+
# "claude-" : il valide le nom avant d'emettre la requete. On expose donc des
|
| 30 |
+
# alias conformes, et on ignore le nom recu pour router vers l'unique modele
|
| 31 |
+
# reellement charge -- le client choisit une etiquette, pas un moteur.
|
| 32 |
+
ALIASES = [
|
| 33 |
+
"claude-kimi-k3",
|
| 34 |
+
"claude-kimi-k3-linear",
|
| 35 |
+
"claude-qwen3-coder",
|
| 36 |
+
"claude-sonnet-4-5", # alias de compatibilite : certains clients
|
| 37 |
+
"claude-3-5-haiku", # codent en dur un modele "rapide" et un "lent"
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
app = FastAPI(title="anthropic-bridge")
|
| 41 |
_client = httpx.AsyncClient(base_url=UPSTREAM, timeout=TIMEOUT)
|
| 42 |
|
|
|
|
| 323 |
|
| 324 |
|
| 325 |
@app.get("/v1/models")
|
| 326 |
+
async def models(request: Request):
|
| 327 |
+
"""Deux protocoles sur la meme route.
|
| 328 |
+
|
| 329 |
+
Un client Anthropic attend `{"data":[{"type":"model","id":...}]}` avec des
|
| 330 |
+
identifiants en "claude-*". Un client OpenAI ou un banc attend la reponse
|
| 331 |
+
de vLLM, dont il lit `max_model_len`. On distingue sur l'en-tete
|
| 332 |
+
`anthropic-version`, et on renvoie a chacun ce qu'il sait lire.
|
| 333 |
+
"""
|
| 334 |
+
if "anthropic-version" not in request.headers:
|
| 335 |
+
try:
|
| 336 |
+
r = await _client.get("/v1/models", timeout=10)
|
| 337 |
+
if r.status_code == 200:
|
| 338 |
+
return JSONResponse(r.json())
|
| 339 |
+
except Exception:
|
| 340 |
+
pass
|
| 341 |
+
|
| 342 |
+
ctx = None
|
| 343 |
try:
|
| 344 |
r = await _client.get("/v1/models", timeout=10)
|
| 345 |
if r.status_code == 200:
|
| 346 |
+
ctx = (r.json().get("data") or [{}])[0].get("max_model_len")
|
| 347 |
except Exception:
|
| 348 |
pass
|
| 349 |
+
|
| 350 |
+
data = [{
|
| 351 |
+
"type": "model",
|
| 352 |
+
"id": a,
|
| 353 |
+
"display_name": f"{a} ({MODEL})",
|
| 354 |
+
"created_at": "2026-01-01T00:00:00Z",
|
| 355 |
+
**({"context_window": ctx} if ctx else {}),
|
| 356 |
+
} for a in ALIASES]
|
| 357 |
+
return JSONResponse({"data": data, "has_more": False,
|
| 358 |
+
"first_id": data[0]["id"], "last_id": data[-1]["id"]})
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
@app.get("/v1/models/{model_id}")
|
| 362 |
+
async def model_detail(model_id: str):
|
| 363 |
+
return JSONResponse({"type": "model", "id": model_id,
|
| 364 |
+
"display_name": f"{model_id} ({MODEL})",
|
| 365 |
+
"created_at": "2026-01-01T00:00:00Z"})
|
| 366 |
|
| 367 |
# ------------------------------------------------------------------ passe-plat
|
| 368 |
# Le pod n'expose qu'un port. Le pont le prend (c'est l'API que Claude Code
|