23v commited on
Commit
918e18a
·
verified ·
1 Parent(s): 4117f63

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +185 -0
app.py ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+ import json
3
+ import os
4
+ import subprocess
5
+ import tarfile
6
+ import time
7
+ import urllib.request
8
+ import zipfile
9
+
10
+ from fastapi import Request
11
+ from fastapi.responses import StreamingResponse
12
+ from gradio import Server
13
+ import httpx
14
+
15
+ app = Server()
16
+
17
+ try:
18
+ import spaces
19
+ HAS_SPACES = True
20
+ except ImportError:
21
+ HAS_SPACES = False
22
+
23
+ @spaces.GPU if HAS_SPACES else lambda f: f
24
+ def whygpu():
25
+ pass
26
+
27
+ def build_latest_llama_server():
28
+ binary_path = os.path.abspath("llama.cpp/build/bin/llama-server")
29
+ if os.path.exists(binary_path):
30
+ return binary_path
31
+
32
+ print("Building latest llama.cpp from master source...")
33
+
34
+ if not os.path.exists("llama.cpp"):
35
+ subprocess.run(["git", "clone", "https://github.com/unslothai/llama.cpp.git"], check=True)
36
+ else:
37
+ subprocess.run(["git", "pull"], cwd="llama.cpp", check=True)
38
+
39
+ subprocess.run(["git", "fetch", "origin", "pull/144/head:mtp"], cwd="llama.cpp", check=True)
40
+ subprocess.run(["git", "checkout", "mtp"], cwd="llama.cpp", check=True)
41
+
42
+ env = os.environ.copy()
43
+ if "/usr/local/cuda/bin" not in env.get("PATH", ""):
44
+ env["PATH"] = f"/usr/local/cuda/bin:{env.get('PATH', '')}"
45
+
46
+ cmake_cmd = [
47
+ "cmake",
48
+ "-B",
49
+ "build",
50
+ "-DCMAKE_BUILD_TYPE=Release",
51
+ "-DGGML_NATIVE=ON",
52
+ "-DGGML_OPENMP=ON",
53
+ "-DGGML_AVX512=ON",
54
+ ]
55
+
56
+ subprocess.run(cmake_cmd, cwd="llama.cpp", env=env, check=True)
57
+ subprocess.run(
58
+ ["cmake", "--build", "build", "--config", "Release", "-j", str(os.cpu_count() or 4), "--target", "llama-server"],
59
+ cwd="llama.cpp",
60
+ env=env,
61
+ check=True
62
+ )
63
+
64
+ if not os.path.exists(binary_path):
65
+ raise RuntimeError("Failed to build llama-server binary!")
66
+
67
+ return binary_path
68
+
69
+ def apathy_exe():
70
+ model_path = hf_hub_download(
71
+ repo_id="Qwen/Qwen3.8-Flash-Next",
72
+ filename="q.gguf",
73
+ )
74
+
75
+ def start_llama_server():
76
+ binary_path = build_latest_llama_server()
77
+
78
+ from huggingface_hub import hf_hub_download
79
+
80
+ #model_path = hf_hub_download(
81
+ # repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
82
+ # filename="UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
83
+ #)
84
+ model_path = None
85
+ for i in range(1,34):
86
+ mp = hf_hub_download(
87
+ repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
88
+ filename="Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64/Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64-{}-of-00033.gguf".format(str(i).zfill(5))
89
+ )
90
+ if not model_path:
91
+ model_path = mp
92
+
93
+ mmproj_path = hf_hub_download(
94
+ repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
95
+ filename="mmproj-Qwen3.8-Flash-Next-F16.gguf",
96
+ )
97
+
98
+ #draft_path = hf_hub_download(
99
+ # repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
100
+ # filename="MTP/mtp-Qwen3.8-Flash-Next-Q4_K_M.gguf",
101
+ #)
102
+
103
+
104
+ binary_dir = os.path.dirname(os.path.abspath(binary_path))
105
+ env = os.environ.copy()
106
+ env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
107
+
108
+ cmd = [
109
+ binary_path,
110
+ "-m", model_path,
111
+ "-mm", mmproj_path,
112
+ #"-md", draft_path,
113
+ "--port", "8000",
114
+ "--host", "127.0.0.1",
115
+ "-t", "16",
116
+ "-tb", "16",
117
+ "-fa", "on",
118
+ "--parallel", "1",
119
+ "--load-mode", "dio",
120
+ "--cache-type-k", "q4_0",
121
+ "--cache-type-v", "q4_0",
122
+ "--jinja",
123
+ "--temp", "1.0",
124
+ "--top-p", "0.95",
125
+ "--top-k", "20",
126
+ "--min-p", "0.0",
127
+ "--presence-penalty", "0.0",
128
+ "--repeat-penalty", "1.0",
129
+ ]
130
+
131
+ process = subprocess.Popen(cmd, env=env)
132
+ time.sleep(5)
133
+ return process
134
+
135
+ @app.middleware("http")
136
+ async def proxy_middleware(request: Request, call_next):
137
+ path = request.url.path
138
+ if path.startswith("/gradio_api"):
139
+ return await call_next(request)
140
+
141
+ url = f"http://127.0.0.1:8000{path}"
142
+
143
+ headers = {
144
+ k: v for k, v in request.headers.items()
145
+ if k.lower() not in ("host", "content-length", "accept-encoding")
146
+ }
147
+ body = await request.body()
148
+
149
+ client = httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=120.0))
150
+ req = client.build_request(
151
+ method=request.method,
152
+ url=url,
153
+ headers=headers,
154
+ params=request.query_params,
155
+ content=body,
156
+ )
157
+
158
+ try:
159
+ response = await client.send(req, stream=True)
160
+ except Exception as e:
161
+ await client.aclose()
162
+ return StreamingResponse(iter([f"Proxy Error: {e}".encode()]), status_code=502)
163
+
164
+ async def stream_and_close():
165
+ try:
166
+ async for chunk in response.aiter_raw():
167
+ yield chunk
168
+ finally:
169
+ await response.aclose()
170
+ await client.aclose()
171
+
172
+ res_headers = {
173
+ k: v for k, v in response.headers.items()
174
+ if k.lower() not in ("content-length", "transfer-encoding", "connection")
175
+ }
176
+
177
+ return StreamingResponse(
178
+ stream_and_close(),
179
+ status_code=response.status_code,
180
+ headers=res_headers
181
+ )
182
+
183
+ if __name__ == "__main__":
184
+ start_llama_server()
185
+ app.launch(show_error=True)