Add CPU Qwen3.8 inference job

app.py

@@ -107,6 +107,19 @@ API_RESOURCES = {
         },
         "output": "WAV audio file",
     },
+    "qwen38_inference": {
+        "name": "Qwen3.8-27B text generation",
+        "description": "Generate text with Qwen3.8-27B using the Space CPU; no GPU allocation.",
+        "endpoint": "/respite/text/qwen38",
+        "method": "POST",
+        "input": {
+            "prompt": "string",
+            "max_new_tokens": "integer (1-256, default 64)",
+            "temperature": "number (0-2, default 0.7)",
+            "thinking": "boolean (default false)",
+        },
+        "output": "JSON containing generated text and CPU timing",
+    },
     "tinyllama_inference": {
         "name": "TinyLlama text generation",
         "description": "Generate text with TinyLlama 1.1B using the Space CPU; no GPU allocation.",
@@ -488,7 +501,7 @@ def _run_cpu_diagnose(model_filter):
     return result
 
 
-def _run_benchmark_job(job_id, kind, model_filter=None, force=False):
+def _run_benchmark_job(job_id, kind, model_filter=None, force=False, request=None):
     try:
         if kind == "smoke":
             with _BENCH_LOCK:
@@ -502,6 +515,16 @@ def _run_benchmark_job(job_id, kind, model_filter=None, force=False):
             with _BENCH_LOCK:
                 result = _run_cpu_process_probe(model_filter)
             payload = {"status": "complete", "result": result}
+        elif kind == "qwen38":
+            request = request or _BENCH_JOBS.get(job_id, {}).get("request", {})
+            with _BENCH_LOCK:
+                result = _generate_qwen38(
+                    model_filter,
+                    request.get("max_new_tokens", 64),
+                    request.get("temperature", 0.7),
+                    request.get("thinking", False),
+                )
+            payload = {"status": "complete", "result": result}
         else:
             if force:
                 global _BENCH_RESULTS
@@ -522,7 +545,7 @@ def _run_benchmark_job(job_id, kind, model_filter=None, force=False):
             _BENCH_JOBS[job_id] = {"status": "error", "error": str(exc)}
 
 
-def _start_benchmark_job(kind, model_filter=None, force=False):
+def _start_benchmark_job(kind, model_filter=None, force=False, request=None):
     job_id = f"bench-{int(time.time() * 1000)}-{threading.get_ident()}"
     with _BENCH_JOBS_LOCK:
         _BENCH_JOBS[job_id] = {
@@ -530,10 +553,11 @@ def _start_benchmark_job(kind, model_filter=None, force=False):
             "kind": kind,
             "model": model_filter,
             "started_at": time.time(),
+            "request": request or {},
         }
     threading.Thread(
         target=_run_benchmark_job,
-        args=(job_id, kind, model_filter, force),
+        args=(job_id, kind, model_filter, force, request or {}),
         daemon=True,
     ).start()
     return job_id
@@ -544,6 +568,57 @@ _TINYLLAMA = None
 _TINYLLAMA_TOKENIZER = None
 _TINYLLAMA_LOCK = threading.Lock()
 _TINYLLAMA_REPO = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+_QWEN38 = None
+_QWEN38_PROCESSOR = None
+_QWEN38_LOCK = threading.Lock()
+_QWEN38_REPO = "Qwen/Qwen3.8-27B"
+
+
+def _load_qwen38():
+    global _QWEN38, _QWEN38_PROCESSOR
+    with _QWEN38_LOCK:
+        if _QWEN38 is not None:
+            return _QWEN38_PROCESSOR, _QWEN38
+        from transformers import AutoModelForImageTextToText, AutoProcessor
+        try:
+            torch.set_num_threads(min(_parse_cpu_count() or 1, 16))
+        except RuntimeError:
+            pass
+        processor = AutoProcessor.from_pretrained(_QWEN38_REPO)
+        model = AutoModelForImageTextToText.from_pretrained(
+            _QWEN38_REPO,
+            torch_dtype=torch.bfloat16,
+            low_cpu_mem_usage=True,
+            device_map="cpu",
+        )
+        model.eval()
+        _QWEN38_PROCESSOR = processor
+        _QWEN38 = model
+        return processor, model
+
+
+def _generate_qwen38(prompt, max_new_tokens=64, temperature=0.7, thinking=False):
+    processor, model = _load_qwen38()
+    max_new_tokens = max(1, min(256, int(max_new_tokens)))
+    temperature = max(0.0, min(2.0, float(temperature)))
+    content = prompt if thinking else prompt + "/no_think"
+    messages = [{"role": "user", "content": content}]
+    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    inputs = processor(text=[text], return_tensors="pt")
+    inputs = {key: value.to("cpu") if hasattr(value, "to") else value for key, value in inputs.items()}
+    t0 = time.time()
+    with torch.inference_mode():
+        output = model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            do_sample=temperature > 0,
+            temperature=max(temperature, 0.01),
+        )
+    elapsed = max(time.time() - t0, 0.001)
+    input_tokens = int(inputs["input_ids"].shape[-1])
+    generated = output[0, input_tokens:]
+    answer = processor.tokenizer.decode(generated, skip_special_tokens=True)
+    return {"text": answer, "prompt_tokens": input_tokens, "completion_tokens": int(generated.shape[-1]), "generation_tokens_per_sec": round(float(generated.shape[-1]) / elapsed, 2), "inference_time_sec": round(elapsed, 3), "device": "cpu", "dtype": "bfloat16", "thinking": bool(thinking)}
 
 
 def _load_tinyllama():
@@ -664,6 +739,30 @@ def _patched_create_app(blocks, **kwargs):
         try: return JSONResponse(_search_web(q, backend=backend, max_results=max_results, extract_top=extract_top, region=region))
         except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
 
+    @fa_app.post("/respite/text/qwen38")
+    def _qwen38(payload: dict = Body(...)):
+        prompt = str(payload.get("prompt", ""))
+        if not prompt.strip():
+            return JSONResponse({"error": "Missing 'prompt'"}, status_code=400)
+        job = _start_benchmark_job(
+            "qwen38",
+            prompt,
+            False,
+            request={
+                "max_new_tokens": payload.get("max_new_tokens", 64),
+                "temperature": payload.get("temperature", 0.7),
+                "thinking": payload.get("thinking", False),
+            },
+        )
+        return JSONResponse({"status": "accepted", "job_id": job, "poll": "/respite/text/qwen38?job_id=" + job}, status_code=202)
+
+    @fa_app.get("/respite/text/qwen38")
+    def _qwen38_status(job_id: str = ""):
+        if not job_id:
+            return JSONResponse({"error": "Missing 'job_id'"}, status_code=400)
+        with _BENCH_JOBS_LOCK:
+            return JSONResponse(_BENCH_JOBS.get(job_id, {"status": "not_found"}))
+
     @fa_app.post("/respite/text/generate")
     def _tinyllama(payload: dict = Body(...)):
         prompt = str(payload.get("prompt", ""))