Spaces:
Sleeping
Sleeping
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import threading | |
| import time | |
| import urllib.error | |
| import urllib.request | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| MODEL_REVISION = "a5995439030829f0c6ef1980f69192660df86f38" | |
| MODEL_REPO = "CodeMasterCody3D/taardis-27b-full-ternary" | |
| MODEL_NAME = "TAARDIS-27B-Full-Ternary-V2-1.75bit.gguf" | |
| ADAPTER_NAME = "TAARDIS-27B-Doctors-V3.lora.gguf" | |
| EXPECTED_SIZES = {MODEL_NAME: 5904497568, ADAPTER_NAME: 322730176} | |
| MODEL_DIR = Path(os.environ.get("TAARDIS_MODEL_DIR", "/tmp/taardis-models")) | |
| RUNTIME = os.environ.get("TAARDIS_RUNTIME", "/opt/llama/bin/llama-server") | |
| RUNTIME_URL = "http://127.0.0.1:8080" | |
| state = {"status": "starting", "message": "Preparing the public TAARDIS model."} | |
| generation_lock = threading.Lock() | |
| runtime_process = None | |
| PAGE = r"""<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>TAARDIS research demo | Internectual Labs</title><style> | |
| *{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 85% 5%,#12364c,transparent 45%),#080d16;color:#edf5ff;font:16px/1.6 system-ui,sans-serif}main{max-width:900px;margin:auto;padding:40px 24px}a{color:#71d8ff}header{display:flex;align-items:center;justify-content:space-between;gap:20px;border-bottom:1px solid #294054;padding-bottom:20px}.label{font-size:12px;letter-spacing:.17em;color:#71d8ff;text-transform:uppercase}h1{font-size:clamp(38px,7vw,70px);line-height:1.1;letter-spacing:-.05em;margin:30px 0 10px}p{color:#adbed2}.panel{border:1px solid #294054;border-radius:16px;padding:24px;background:#101d2b99;margin-top:24px}#status{color:#e7c981}textarea{width:100%;min-height:130px;resize:vertical;background:#080f19;color:#fff;border:1px solid #425b75;border-radius:10px;padding:14px;font:inherit}button{background:#6ed6ff;color:#08141f;border:0;border-radius:8px;padding:12px 20px;font:700 16px system-ui;cursor:pointer}button:disabled{opacity:.45;cursor:wait}.actions{display:flex;align-items:center;gap:18px;margin-top:14px;flex-wrap:wrap}#answer{white-space:pre-wrap;overflow-wrap:anywhere;min-height:80px}small{color:#adbed2}details{margin-top:20px}summary{cursor:pointer}label{display:block;margin-bottom:10px;font-weight:650}a:focus-visible,button:focus-visible,textarea:focus-visible{outline:3px solid #efcb7e;outline-offset:4px}</style></head><body><main><header><strong>INTERNECTUAL LABS</strong><a href="https://huggingface.co/spaces/CodeMasterCody3D/internectual-labs" target="_blank" rel="noopener">Explore the lab ↗</a></header><div class="label" style="margin-top:30px">Experimental model / public research demo</div><h1>Meet TAARDIS.</h1><p>A compact representation of a 27-billion-parameter model. Try a short prompt using the public V2 weights and Doctors V3 corrections.</p><div class="panel"><strong>Runtime status</strong><div id="status" role="status">Checking model availability…</div><small>Free CPU hosting. Startup downloads about 6.2 GB and loads a custom runtime; responses may be slow.</small></div><form class="panel" id="chat"><label for="prompt">What would you like to ask?</label><textarea id="prompt" maxlength="6000" placeholder="Explain why the sky is blue in two sentences." required></textarea><div class="actions"><button id="send" disabled>Generate response</button><small>One request at a time · 4K context · up to 128 output tokens</small></div></form><section class="panel" aria-labelledby="answer-title"><strong id="answer-title">Response</strong><div id="answer" aria-live="polite">Your response will appear here.</div><small id="timing"></small></section><details><summary>Model and limitations</summary><p>This is a research model. Aggressive quantization can reduce quality. Responses can be incomplete or incorrect. Do not use this demo for consequential decisions or submit private information.</p><p>The interface does not retain prompts in an application database. Inputs are sent to the Space server for inference; hosting infrastructure may process operational logs.</p><p><a href="https://huggingface.co/CodeMasterCody3D/taardis-27b-full-ternary" target="_blank" rel="noopener">Model card and Apache 2.0 attribution ↗</a> · <a href="https://github.com/CodeMasterCody3D/taardis-llama.cpp" target="_blank" rel="noopener">Required public runtime ↗</a></p><p>This CPU demo has no GPU grant yet. GPU support would improve interactive use; performance is measured only after an actual successful run.</p></details></main><script> | |
| let busy=false;const send=document.getElementById('send');async function check(){try{const r=await fetch('/api/status');const s=await r.json();document.getElementById('status').textContent=s.message;send.disabled=busy||s.status!=='ready';}catch(e){document.getElementById('status').textContent='The Space is waking up. Please try again shortly.';send.disabled=true;}}check();setInterval(check,5000);document.getElementById('chat').addEventListener('submit',async e=>{e.preventDefault();if(busy)return;busy=true;send.disabled=true;document.getElementById('answer').textContent='Generating on the CPU. Please keep this page open…';document.getElementById('timing').textContent='';try{const r=await fetch('/api/chat',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({prompt:document.getElementById('prompt').value})});if(!r.ok){const d=await r.json();throw Error(d.error||'The request could not be completed.');}const reader=r.body.getReader();const decoder=new TextDecoder();let buffer='',text='';while(true){const result=await reader.read();if(result.done)break;buffer+=decoder.decode(result.value,{stream:true});let split;while((split=buffer.indexOf('\n'))>=0){const line=buffer.slice(0,split);buffer=buffer.slice(split+1);if(!line.startsWith('data: '))continue;const d=JSON.parse(line.slice(6));if(d.error)throw Error(d.error);if(d.text){text+=d.text;document.getElementById('answer').textContent=text;}if(d.done){if(!text)document.getElementById('answer').textContent='The model reached the token limit before a visible answer. Try a simpler prompt.';document.getElementById('timing').textContent='Completed in '+d.elapsed_seconds+' seconds.';}}}}catch(e){document.getElementById('answer').textContent=e.message;}finally{busy=false;check();}}); | |
| </script></body></html>""" | |
| def download(filename, remote_path): | |
| dest = MODEL_DIR / filename | |
| if dest.exists() and dest.stat().st_size == EXPECTED_SIZES[filename]: | |
| return dest | |
| state.update(status="loading", message=f"Downloading public model files: {filename}.") | |
| url = f"https://huggingface.co/{MODEL_REPO}/resolve/{MODEL_REVISION}/{remote_path}" | |
| temp = dest.with_suffix(dest.suffix + ".part") | |
| with urllib.request.urlopen(url, timeout=600) as response, temp.open("wb") as output: | |
| while chunk := response.read(8 * 1024 * 1024): | |
| output.write(chunk) | |
| if temp.stat().st_size != EXPECTED_SIZES[filename]: | |
| raise ValueError("Model download was incomplete.") | |
| with temp.open("rb") as downloaded: | |
| if downloaded.read(4) != b"GGUF": | |
| raise ValueError("Model download is not a GGUF file.") | |
| temp.replace(dest) | |
| return dest | |
| def startup(): | |
| global runtime_process | |
| try: | |
| MODEL_DIR.mkdir(parents=True, exist_ok=True) | |
| model = download(MODEL_NAME, MODEL_NAME) | |
| adapter = download(ADAPTER_NAME, f"doctors/{ADAPTER_NAME}") | |
| state.update(status="loading", message="Loading TAARDIS V2 and Doctors V3 into the CPU runtime.") | |
| threads = str(max(1, min(4, int(os.environ.get("TAARDIS_THREADS", "2"))))) | |
| runtime_process = subprocess.Popen([ | |
| RUNTIME, "-m", str(model), "--lora", str(adapter), | |
| "--host", "127.0.0.1", "--port", "8080", "-c", "4096", | |
| "-b", "64", "-ub", "64", "-np", "1", "-ngl", "0", | |
| "-t", threads, "-tb", threads, "--no-warmup", | |
| ]) | |
| while runtime_process.poll() is None: | |
| try: | |
| with urllib.request.urlopen(RUNTIME_URL + "/health", timeout=3) as response: | |
| if response.status == 200: | |
| state.update(status="ready", message="TAARDIS is ready. CPU responses may take several minutes.") | |
| break | |
| except (urllib.error.URLError, TimeoutError): | |
| pass | |
| time.sleep(2) | |
| runtime_process.wait() | |
| state.update(status="error", message="The model runtime stopped. Please check the Space runtime logs.") | |
| except Exception as exc: | |
| print(f"Runtime setup failed: {exc}", flush=True) | |
| state.update(status="error", message="The model could not start. The owner can inspect Space runtime logs.") | |
| class Handler(BaseHTTPRequestHandler): | |
| def log_message(self, *_): | |
| pass | |
| def respond(self, status, body, content_type="application/json"): | |
| data = body.encode() if isinstance(body, str) else json.dumps(body).encode() | |
| self.send_response(status) | |
| self.send_header("Content-Type", content_type + "; charset=utf-8") | |
| self.send_header("Content-Length", str(len(data))) | |
| self.send_header("Cache-Control", "no-store") | |
| self.send_header("X-Content-Type-Options", "nosniff") | |
| self.end_headers() | |
| self.wfile.write(data) | |
| def do_GET(self): | |
| if self.path in ("/", "/index.html"): | |
| self.respond(200, PAGE, "text/html") | |
| elif self.path == "/api/status": | |
| self.respond(200, state) | |
| else: | |
| self.respond(404, {"error": "Not found."}) | |
| def do_POST(self): | |
| if self.path != "/api/chat": | |
| self.respond(404, {"error": "Not found."}) | |
| return | |
| if state["status"] != "ready": | |
| self.respond(503, {"error": state["message"]}) | |
| return | |
| if not generation_lock.acquire(blocking=False): | |
| self.respond(429, {"error": "Another request is running. Please try again after it finishes."}) | |
| return | |
| try: | |
| length = int(self.headers.get("Content-Length", "0")) | |
| if not 0 < length <= 30000: | |
| raise ValueError("Invalid request size.") | |
| data = json.loads(self.rfile.read(length)) | |
| prompt = data.get("prompt", "") | |
| if not isinstance(prompt, str) or not 1 <= len(prompt.strip()) <= 6000: | |
| raise ValueError("Enter a prompt of 1 to 6000 characters.") | |
| payload = { | |
| "messages": [{"role": "user", "content": prompt}], | |
| "max_tokens": 128, "temperature": 0.6, "top_p": 0.95, | |
| "top_k": 20, "repeat_penalty": 1.3, "stream": True, | |
| "chat_template_kwargs": {"enable_thinking": False}, | |
| } | |
| start = time.monotonic() | |
| request = urllib.request.Request(RUNTIME_URL + "/v1/chat/completions", data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) | |
| with urllib.request.urlopen(request, timeout=900) as response: | |
| self.send_response(200) | |
| self.send_header("Content-Type", "text/event-stream; charset=utf-8") | |
| self.send_header("Cache-Control", "no-store") | |
| self.send_header("X-Accel-Buffering", "no") | |
| self.end_headers() | |
| for line in response: | |
| if not line.startswith(b"data: "): | |
| continue | |
| data = line[6:].strip() | |
| if data == b"[DONE]": | |
| break | |
| chunk = json.loads(data) | |
| choices = chunk.get("choices", []) | |
| delta = choices[0].get("delta", {}) if choices else {} | |
| text = delta.get("content") or delta.get("reasoning_content") or "" | |
| if text: | |
| self.wfile.write(("data: " + json.dumps({"text": text}) + "\n\n").encode()) | |
| self.wfile.flush() | |
| self.wfile.write(("data: " + json.dumps({"done": True, "elapsed_seconds": round(time.monotonic() - start, 1)}) + "\n\n").encode()) | |
| self.wfile.flush() | |
| except (ValueError, json.JSONDecodeError) as exc: | |
| self.respond(400, {"error": str(exc)}) | |
| except Exception as exc: | |
| print(f"Generation failed: {type(exc).__name__}", flush=True) | |
| self.respond(502, {"error": "The model could not finish this request. Try a shorter prompt or wait for the runtime to recover."}) | |
| finally: | |
| generation_lock.release() | |
| if __name__ == "__main__": | |
| threading.Thread(target=startup, daemon=True).start() | |
| ThreadingHTTPServer(("0.0.0.0", int(os.environ.get("PORT", "7860"))), Handler).serve_forever() | |