test app.py fix

app.py

@@ -1,53 +1,129 @@
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
 import gradio as gr
-from ctransformers import AutoModelForCausalLM
-
-# 🔍 Sanity test model: small, public GGUF that is known to work
-HF_REPO = "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
-MODEL_FILE = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"  # this file exists in that repo
-
-print("Loading TinyLlama GGUF model with ctransformers...\n")
-
-llm = AutoModelForCausalLM.from_pretrained(
-    HF_REPO,
-    model_file=MODEL_FILE,
-    model_type="llama",   # TinyLlama is LLaMA-family
-    hf=True,              # load from Hugging Face Hub
-    gpu_layers=0,         # CPU only (free HF Space)
-    threads=2,            # free Spaces usually have 2 vCPUs
-    context_length=2048,
+import os
+
+# ==============================
+# Model loading
+# ==============================
+
+# Get HF token from Space secrets (optional, for private repos)
+hf_token = os.environ.get("HF_TOKEN")
+
+print("Downloading GGUF model from HuggingFace...")
+model_path = hf_hub_download(
+    repo_id="Zedel17/llama_1b_gguf",
+    filename="llama_1b_finetuned-Q4_K_M.gguf",
+    token=hf_token
 )
+print(f"Model downloaded to: {model_path}")
 
+print("Loading GGUF model with llama-cpp-python...")
+llm = Llama(
+    model_path=model_path,
+    n_ctx=2048,
+    n_threads=2,  # HF Spaces typically have 2 vCPUs
+    n_gpu_layers=0
+)
 print("Model loaded successfully!")
 
-def chat(message, history):
-    # Turn Gradio chat history into a simple prompt
-    parts = []
-    for user_msg, bot_msg in history:
-        parts.append(f"User: {user_msg}\nAssistant: {bot_msg}")
-    parts.append(f"User: {message}\nAssistant:")
-    prompt = "\n".join(parts)
+# ==============================
+# System prompt (kept short and stable)
+# ==============================
+
+SYSTEM_PROMPT = (
+    "You are a friendly and helpful AI assistant. "
+    "You respond in clear, concise, and structured English. "
+    "You NEVER ask follow-up questions. "
+    "You NEVER continue the conversation beyond one answer. "
+    "You ONLY answer the user's last message. "
+    "Be precise, helpful, and human-like."
+)
+
+# ==============================
+# Safe one-turn chat function
+# ==============================
 
-    response = llm(
+def chat_fn(message, history):
+    # DO NOT rebuild the conversation. This prevents hallucinated follow-ups.
+    prompt = (
+        SYSTEM_PROMPT
+        + "\n\n"
+        + f"User: {message}\n"
+        + "Assistant:"
+    )
+
+    output = llm(
         prompt,
-        max_new_tokens=128,
+        max_tokens=350,
         temperature=0.7,
-        top_p=0.9,
-        repetition_penalty=1.1,
     )
 
-    return str(response).strip()
-
-demo = gr.ChatInterface(
-    fn=chat,
-    title="Sanity Test: TinyLlama 1.1B Chat",
-    description="If this works, the Space + ctransformers setup is healthy.",
-    examples=[
-        "Hello! Who are you?",
-        "Write a short poem about cats.",
-        "Explain what an LLM is in simple terms.",
-    ],
-    cache_examples=False,
-)
+    reply = output["choices"][0]["text"].strip()
+
+    # Guarantee no extra questions or turns
+    # Occasionally small models hallucinate "User:" — we clean that.
+    reply = reply.split("User:")[0].strip()
+    reply = reply.split("Assistant:")[0].strip()
+
+    return reply
+
+
+# ==============================
+# Gradio UI
+# ==============================
+
+with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue")) as demo:
+
+    gr.Markdown(
+        """
+        # 🤖 **Fine-Tuned Llama-3.2-1B Chat**
+        _Fine-tuned on FineTome-100k dataset by **Zedel17**_
+
+        This is a lightweight Llama-3.2-1B model fine-tuned using PEFT/LoRA
+        to generate clearer, more structured and more human-like responses
+        compared to the base model.
+
+        It uses GGUF quantization (Q4_K_M) and CPU-only inference through llama.cpp.
+        """
+    )
+
+    chatbot = gr.Chatbot(height=500, show_label=False)
+
+    with gr.Row():
+        msg = gr.Textbox(
+            placeholder="Type your message here...",
+            scale=8,
+            show_label=False
+        )
+        send_btn = gr.Button("Send", variant="primary", scale=1)
+
+    clear_btn = gr.Button("🧹 Clear conversation")
+
+    # ==============================
+    # Button logic
+    # ==============================
+
+    def respond(message, chat_history):
+        reply = chat_fn(message, chat_history)
+        chat_history.append((message, reply))
+        return "", chat_history
+
+    send_btn.click(respond, [msg, chatbot], [msg, chatbot])
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+    clear_btn.click(lambda: None, None, chatbot, queue=False)
+
+    gr.Markdown(
+        """
+        ---
+        ### ℹ️ Technical details
+        - **Base Model:** meta-llama/Llama-3.2-1B
+        - **Fine-tuning:** PEFT/LoRA on FineTome-100k dataset
+        - **Format:** GGUF Q4_K_M quantization
+        - **Inference:** llama-cpp-python (CPU only)
+        - **Training:** Supervised Fine-Tuning (SFT) with 100 steps
+        - **Context window:** 2048 tokens
+        """
+    )
 
-if __name__ == "__main__":
-    demo.launch()
+demo.launch()