Update app.py

app.py

@@ -1,10 +1,14 @@
+import os
+import json
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import StreamingResponse, JSONResponse
 import gradio as gr
 import spaces
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-from threading import Thread
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 
-# 1. DUMMY GPU FUNCTION: satisfies the Hugging Face startup checker 
+# 1. DUMMY GPU FUNCTION: Satisfies the Hugging Face ZeroGPU checks 
 @spaces.GPU(duration=5)
 def dummy_gpu():
     print("ZeroGPU validation satisfied.")
@@ -13,69 +17,105 @@ def dummy_gpu():
 # Automatically execute it once right away
 dummy_gpu()
 
-# 2. LOAD NATIVE PYTORCH MODEL
-print("Loading model via native PyTorch & Transformers...")
-model_id = "Qwen/Qwen2.5-1.5B-Instruct"
+# 2. DOWNLOAD LFM 2.5 8B A1B GGUF MODEL
+print("Downloading LFM2.5-8B-A1B-GGUF model...")
+model_path = hf_hub_download(
+    repo_id="LiquidAI/LFM2.5-8B-A1B-GGUF", 
+    filename="LFM2.5-8B-A1B-Q4_K_M.gguf"
+)
 
-# Using standard safetensors natively supported by HF.
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    torch_dtype=torch.float32, # Safe for standard CPU execution
-    device_map="cpu",          # Force pure CPU execution
-    low_cpu_mem_usage=True
+# 3. LOAD MODEL DIRECTLY IN PYTHON (CPU ONLY)
+print("Loading model into memory via llama-cpp-python...")
+llm = Llama(
+    model_path=model_path,
+    n_ctx=8192,           # Safe context limit for RAM
+    n_threads=2,          # Optimal for standard HF CPU Space
+    chat_format="chatml", # Native format used by LFM2.5
+    verbose=False     
 )
 print("Model loaded successfully!")
 
-# 3. CHAT FUNCTION WITH STREAMING
-def chat_with_hf(message, history):
-    # Format the conversation for the model
+# 4. BUILD FASTAPI APP FOR NATIVE OPENAI-COMPATIBLE ENDPOINTS
+app = FastAPI(title="LFM2.5-8B API Server")
+
+@app.post("/v1/chat/completions")
+async def chat_api(request: Request):
+    """
+    OpenAI-compatible API Endpoint!
+    You can send JSON payloads here exactly as you would to the OpenAI API.
+    """
+    try:
+        body = await request.json()
+        messages = body.get("messages", [])
+        stream = body.get("stream", False)
+        temperature = body.get("temperature", 0.7)
+        max_tokens = body.get("max_tokens", 1024)
+        
+        if stream:
+            def stream_generator():
+                for chunk in llm.create_chat_completion(
+                    messages=messages, 
+                    stream=True, 
+                    temperature=temperature,
+                    max_tokens=max_tokens
+                ):
+                    yield f"data: {json.dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+            return StreamingResponse(stream_generator(), media_type="text/event-stream")
+        else:
+            response = llm.create_chat_completion(
+                messages=messages, 
+                stream=False, 
+                temperature=temperature,
+                max_tokens=max_tokens
+            )
+            return JSONResponse(content=response)
+    except Exception as e:
+        return JSONResponse(status_code=500, content={"error": str(e)})
+
+# 5. BUILD GRADIO UI (Fallback Web Interface)
+def chat_with_llama(message, history):
     messages = [{"role": "system", "content": "You are a helpful AI assistant."}]
+    
     for user_msg, assistant_msg in history:
         messages.append({"role": "user", "content": user_msg})
         messages.append({"role": "assistant", "content": assistant_msg})
         
     messages.append({"role": "user", "content": message})
     
-    # Convert chat to model tokens
-    input_ids = tokenizer.apply_chat_template(
-        messages, 
-        add_generation_prompt=True, 
-        return_tensors="pt"
-    ).to("cpu")
-    
-    # Streamer to yield words one by one to Gradio
-    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
-    
-    # Generation arguments
-    generate_kwargs = dict(
-        input_ids=input_ids,
-        streamer=streamer,
-        max_new_tokens=512,
+    stream = llm.create_chat_completion(
+        messages=messages,
+        stream=True,
         temperature=0.7,
-        top_p=0.9,
-        do_sample=True
+        max_tokens=1024
     )
     
-    # Start generation in a background thread so the streamer can yield in real-time
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-    
-    # Yield the text back to Gradio UI
     partial_response = ""
-    for new_text in streamer:
-        partial_response += new_text
-        yield partial_response
+    for chunk in stream:
+        if "choices" in chunk and len(chunk["choices"]) > 0:
+            delta = chunk["choices"][0].get("delta", {})
+            if "content" in delta:
+                partial_response += delta["content"]
+                yield partial_response
 
-# 4. LAUNCH GRADIO APP
 with gr.Blocks() as demo:
-    gr.Markdown("# Native CPU PyTorch Execution on ZeroGPU Space")
-    gr.Markdown("Running **Qwen2.5-1.5B** entirely via standard Hugging Face `transformers` and `torch`. No C++, no compilations, no GGUF.")
+    gr.Markdown("# 🚀 LiquidAI LFM2.5-8B-A1B (CPU & API Enabled)")
+    gr.Markdown("""
+    Running **LFM2.5-8B-A1B** natively via `llama-cpp-python` entirely on CPU! ZeroGPU is bypassed safely.
+    
+    ### 🔌 Developer API Available!
+    This Space also silently hosts a real API endpoint. You can query it using standard Python code without using the Web UI.
+    **Endpoint:** `POST /v1/chat/completions` (Supports Streaming & Non-Streaming)
+    """)
     
     gr.ChatInterface(
-        fn=chat_with_hf,
+        fn=chat_with_llama,
         examples=["Who are you?", "Write a python script to reverse a string.", "Explain quantum computing."],
     )
 
+# 6. MOUNT GRADIO ON FASTAPI & LAUNCH
+# This runs the API and the Web UI simultaneously on the same port!
+app = gr.mount_gradio_app(app, demo, path="/")
+
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+    uvicorn.run(app, host="0.0.0.0", port=7860)