Create app.py

app.py

@@ -0,0 +1,78 @@
+import os
+import threading
+import time
+import subprocess
+import gradio as gr
+from ollama import Client
+
+# 1. Start Ollama Server in a background thread
+# We don't need to install it anymore; Docker did that!
+def ollama_service_thread():
+    # We use 'serve' to start the background process
+    subprocess.run("ollama serve", shell=True)
+
+threading.Thread(target=ollama_service_thread, daemon=True).start()
+
+# 2. Wait for the server to wake up
+print("Waiting for Ollama to start...")
+time.sleep(10)  # Give it a moment
+
+# 3. Download the model (runs only if not already present)
+# Llama 3.2 3B is small and fast!
+MODEL = "llama3.2:3b"
+print(f"Checking/Pulling model: {MODEL}...")
+subprocess.run(f"ollama pull {MODEL}", shell=True)
+
+# 4. Connect the Client
+client = Client(host='http://127.0.0.1:11434')
+
+# 5. The Chat Logic
+def chat(message, history, temperature, max_new_tokens, top_p, top_k, penalty):
+    # Format history for the model
+    conversation = []
+    for prompt, answer in history:
+        conversation.extend([
+            {"role": "user", "content": prompt},
+            {"role": "assistant", "content": answer},
+        ])
+    conversation.append({"role": "user", "content": message})
+
+    # Call the API
+    response = client.chat(
+        model=MODEL,
+        messages=conversation,
+        stream=True,
+        options={
+            "num_predict": max_new_tokens,
+            "temperature": temperature,
+            "top_p": top_p,
+            "top_k": top_k,
+            "repeat_penalty": penalty,
+        }
+    )
+
+    # Stream the output
+    partial_message = ""
+    for chunk in response:
+        if 'message' in chunk and 'content' in chunk['message']:
+            content = chunk['message']['content']
+            partial_message += content
+            yield partial_message
+
+# 6. The UI
+with gr.Blocks() as demo:
+    gr.HTML("<h1>🦙 Llama 3.2 3B on HF Docker</h1>")
+    
+    gr.ChatInterface(
+        fn=chat,
+        additional_inputs=[
+            gr.Slider(0, 1, 0.7, label="Temperature"),
+            gr.Slider(128, 4096, 1024, label="Max New Tokens"),
+            gr.Slider(0.0, 1.0, 0.9, label="Top P"),
+            gr.Slider(1, 100, 40, label="Top K"),
+            gr.Slider(0.0, 2.0, 1.1, label="Repetition Penalty"),
+        ]
+    )
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)