feat: VidyaBot v1.0 Production with offline Ollama fallback

.dockerignore

@@ -0,0 +1,14 @@
+.git
+.gitignore
+__pycache__
+*.pyc
+.pytest_cache
+*.egg-info
+.env
+htmlcov/
+.venv
+venv/
+data/*.db
+data/textbooks/*
+backend/llm/models/mistral-vidyabot-hf/
+backend/llm/models/llama.cpp/

Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    wget \
+    git \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Ollama
+RUN curl -fsSL https://ollama.com/install.sh | sh
+
+# Copy and install python dependencies
+COPY backend/requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Expose port
+EXPOSE 7860
+
+# Run cross-platform launcher
+CMD ["python", "app.py"]

README.md

@@ -11,7 +11,7 @@ We have successfully earned all **5 merit badges** for the **Build Small 2026**
 * 🦙 **Llama Champion** — Runs via standard llama.cpp runtime embedded in local Ollama instance
 * 🎨 **Off-Brand** — Completely custom Gradio frontend with Indian flag aesthetics and responsive layouts
 * 📓 **Field Notes** — Complete 2,000-word engineering retro published at `docs/field_notes.md`
-* 🎯 **Well-Tuned** — Mistral 7B fine-tuned on 60+ student Q&A pairs via Modal A10G GPU (LoRA/QLoRA)
+* 🎯 **Well-Tuned** — Mistral 7B fine-tuned on 103 student Q&A pairs via Modal A10G GPU (LoRA/QLoRA)
 
 ---
 
@@ -24,7 +24,7 @@ VidyaBot uses **Mistral 7B Instruct fine-tuned on student Q&A pairs** from NCERT
 | Detail | Value |
 |--------|-------|
 | **Base model** | `mistralai/Mistral-7B-Instruct-v0.1` |
-| **Training data** | 60+ hand-crafted + synthetic NCERT Q&A pairs |
+| **Training data** | 103 hand-crafted + synthetic NCERT Q&A pairs |
 | **Method** | QLoRA (4-bit quantization + LoRA adapters) |
 | **LoRA config** | r=8, alpha=16, targets: q/v/k/o_proj |
 | **Hardware** | Modal A10G GPU (24GB VRAM) |
@@ -44,7 +44,7 @@ Base `mistral:latest` is a strong general-purpose model, but fine-tuning on NCER
 ### Fine-Tuning Pipeline
 
 ```
-Student Q&A Data (60 pairs)
+Student Q&A Data (103 pairs)
     ↓
 [Modal A10G GPU]
     QLoRA: Mistral-7B-Instruct + LoRA adapters (r=8)
@@ -133,7 +133,7 @@ Over **200 million** Indian students use textbooks from national and state board
                                     ▼
                     ┌───────────────────────────────┐
                     │  OLLAMA LOCAL INFERENCE (CPU)  │
-                    │  - Model: llama3.2:latest     │
+                    │  - Model: mistral-vidyabot    │
                     │  - Cost: $0.00 | TTFT: <2s    │
                     └───────────────┬───────────────┘
                                     │
@@ -229,7 +229,7 @@ Create a `.env` file in the root directory:
 ```env
 LLM_BACKEND=ollama
 OLLAMA_BASE_URL=http://localhost:11434
-OLLAMA_MODEL=llama3.2:latest
+OLLAMA_MODEL=mistral-vidyabot
 ```
 
 ### 4. Start Unified Application

app.py

@@ -1,6 +1,9 @@
-# app.py — Main entry point for VidyaBot Gradio Edition
+# app.py — Main entry point for VidyaBot Gradio Edition (Cross-platform local and HF Space)
 import sys
 import warnings
+import subprocess
+import time
+import os
 
 # Enforce UTF-8 encoding on standard streams to prevent Windows console encoding crashes
 if hasattr(sys.stdout, 'reconfigure'):
@@ -11,6 +14,36 @@ if hasattr(sys.stderr, 'reconfigure'):
 # Suppress Gradio's warning about Blocks parameters theme/css moving to launch()
 warnings.filterwarnings("ignore", category=UserWarning, module="gradio")
 
+print("🚀 VidyaBot Launch Sequence")
+print("=" * 50)
+
+# Start Ollama daemon on Linux (e.g. Hugging Face Spaces)
+if sys.platform != "win32":
+    print("📦 Starting Ollama daemon...")
+    try:
+        subprocess.Popen(
+            ["ollama", "serve"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        time.sleep(3)
+        
+        # Pull base model in the background as fallback
+        print("📥 Ensuring mistral:latest is available...")
+        subprocess.Popen(
+            ["ollama", "pull", "mistral:latest"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+    except Exception as e:
+        print(f"⚠️ Could not start Ollama: {e}")
+else:
+    print("💻 Windows environment detected. Assumes local Ollama is running.")
+
+print("=" * 50)
+print("🎯 Launching VidyaBot Gradio UI...")
+print("=" * 50)
+
 import gradio as gr
 from backend.main import app as fastapi_app
 from gradio_app import create_demo
@@ -20,4 +53,5 @@ app = gr.mount_gradio_app(fastapi_app, demo, path="/")
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run("app:app", host="0.0.0.0", port=7860, reload=False)
+    port = int(os.environ.get("PORT", 7860))
+    uvicorn.run("app:app", host="0.0.0.0", port=port, reload=False)

modal_convert_gguf.py

@@ -1,38 +1,34 @@
 """
-modal_convert_gguf.py — Download + Convert Fine-Tuned Model to GGUF
+modal_convert_gguf.py — Download GGUF from Modal Volume & Register with Ollama
 
 Run this AFTER modal_finetune.py completes.
 
 Steps:
-1. Download merged model from Modal Volume to local disk
-2. Clone llama.cpp (for the conversion script)
-3. Convert HuggingFace safetensors → GGUF (Q4_K_M quantization)
-4. Place GGUF file where Ollama can read it
+1. Download the quantized GGUF model directly from Modal Volume to local disk
+2. Create/update the Ollama model pointing to the downloaded GGUF file
 
-Usage (from project root, after fine-tuning completes):
+Usage (from project root):
     python modal_convert_gguf.py
-
-Requirements:
-    pip install modal
-    pip install huggingface-hub
-    Git + cmake available (for llama.cpp build)
 """
 
 import subprocess
 import sys
-import os
 from pathlib import Path
 
+# Enforce UTF-8 encoding on standard streams to prevent Windows console encoding crashes
+if hasattr(sys.stdout, 'reconfigure'):
+    sys.stdout.reconfigure(encoding='utf-8')
+if hasattr(sys.stderr, 'reconfigure'):
+    sys.stderr.reconfigure(encoding='utf-8')
+
 
 # ──────────────────────────────────────────────────────────────
 # Configuration
 # ──────────────────────────────────────────────────────────────
 VOLUME_NAME = "vidyabot-model-output"
-REMOTE_MODEL_PATH = "mistral-vidyabot-merged"
+REMOTE_GGUF_FILENAME = "mistral-vidyabot.Q4_K_M.gguf"
 LOCAL_MODELS_DIR = Path("backend/llm/models")
-LOCAL_HF_MODEL_DIR = LOCAL_MODELS_DIR / "mistral-vidyabot-hf"
-LOCAL_GGUF_PATH = LOCAL_MODELS_DIR / "mistral-vidyabot.Q4_K_M.gguf"
-LLAMACPP_DIR = LOCAL_MODELS_DIR / "llama.cpp"
+LOCAL_GGUF_PATH = LOCAL_MODELS_DIR / REMOTE_GGUF_FILENAME
 
 
 def run(cmd: str, cwd: Path = None, check: bool = True) -> subprocess.CompletedProcess:
@@ -49,93 +45,38 @@ def run(cmd: str, cwd: Path = None, check: bool = True) -> subprocess.CompletedP
 
 
 def step1_download_from_modal():
-    """Download the merged HF model from Modal Volume to local disk."""
-    print("\n[1/4] Downloading merged model from Modal Volume...")
-    LOCAL_HF_MODEL_DIR.mkdir(parents=True, exist_ok=True)
+    """Download the GGUF model file from Modal Volume to local disk."""
+    print(f"\n[1/2] Downloading GGUF model '{REMOTE_GGUF_FILENAME}' from Modal Volume '{VOLUME_NAME}'...")
+    LOCAL_MODELS_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Use the Modal CLI in the virtual environment if available
+    python_dir = Path(sys.executable).parent
+    modal_bin = python_dir / "modal.exe" if (python_dir / "modal.exe").exists() else (python_dir / "modal" if (python_dir / "modal").exists() else "modal")
 
-    # Use the Modal CLI to download
     cmd = (
-        f"modal volume get {VOLUME_NAME} "
-        f"{REMOTE_MODEL_PATH} {LOCAL_HF_MODEL_DIR}"
+        f'"{modal_bin}" volume get {VOLUME_NAME} '
+        f"{REMOTE_GGUF_FILENAME} {LOCAL_MODELS_DIR}"
     )
     run(cmd)
-    print(f"  ✅ Model downloaded to: {LOCAL_HF_MODEL_DIR}")
-
-
-def step2_get_llamacpp():
-    """Clone or update llama.cpp for the GGUF conversion script."""
-    print("\n[2/4] Setting up llama.cpp conversion tools...")
-    LOCAL_MODELS_DIR.mkdir(parents=True, exist_ok=True)
-
-    if not LLAMACPP_DIR.exists():
-        run(
-            f"git clone --depth 1 https://github.com/ggerganov/llama.cpp {LLAMACPP_DIR}",
-            cwd=LOCAL_MODELS_DIR
-        )
-        # Install conversion script requirements
-        req_file = LLAMACPP_DIR / "requirements.txt"
-        if req_file.exists():
-            run(f"{sys.executable} -m pip install -r {req_file}", check=False)
-        print(f"  ✅ llama.cpp cloned to: {LLAMACPP_DIR}")
+    
+    # Check if download succeeded
+    if LOCAL_GGUF_PATH.exists():
+        print(f"  ✅ Model downloaded successfully to: {LOCAL_GGUF_PATH}")
     else:
-        print(f"  ✅ llama.cpp already present at: {LLAMACPP_DIR}")
-
-
-def step3_convert_to_gguf():
-    """Convert HuggingFace safetensors model to GGUF format."""
-    print("\n[3/4] Converting HuggingFace model to GGUF (Q4_K_M)...")
-
-    convert_script = LLAMACPP_DIR / "convert_hf_to_gguf.py"
-    if not convert_script.exists():
-        # Older llama.cpp uses convert.py
-        convert_script = LLAMACPP_DIR / "convert.py"
-    if not convert_script.exists():
-        print("  ❌ llama.cpp conversion script not found.")
-        print("     Try: pip install llama-cpp-python and skip this step.")
+        print(f"  ❌ GGUF file not found at {LOCAL_GGUF_PATH} after download.")
         sys.exit(1)
 
-    LOCAL_GGUF_PATH.parent.mkdir(parents=True, exist_ok=True)
-
-    # Step A: Convert to unquantized GGUF first
-    unquantized_gguf = LOCAL_MODELS_DIR / "mistral-vidyabot-f16.gguf"
-    run(
-        f"{sys.executable} {convert_script} "
-        f"{LOCAL_HF_MODEL_DIR} "
-        f"--outfile {unquantized_gguf} "
-        f"--outtype f16"
-    )
-    print(f"  ✅ Unquantized GGUF created: {unquantized_gguf}")
-
-    # Step B: Quantize to Q4_K_M (4-bit, ~4GB, optimal for CPU inference)
-    # Try llama-quantize binary if built, otherwise use Python-based quantization
-    quantize_bin = LLAMACPP_DIR / "build" / "bin" / "llama-quantize"
-    if not quantize_bin.exists():
-        quantize_bin = LLAMACPP_DIR / "quantize"  # Older path
-
-    if quantize_bin.exists():
-        run(f"{quantize_bin} {unquantized_gguf} {LOCAL_GGUF_PATH} Q4_K_M")
-        # Clean up unquantized file
-        unquantized_gguf.unlink(missing_ok=True)
-        print(f"  ✅ Quantized to Q4_K_M: {LOCAL_GGUF_PATH}")
-    else:
-        # Skip quantization — use f16 directly (larger but works)
-        print("  ⚠️  llama-quantize binary not found — using f16 GGUF (larger, ~13GB)")
-        print("     To build quantize: cd backend/llm/models/llama.cpp && cmake -B build && cmake --build build -t llama-quantize")
-        LOCAL_GGUF_PATH = unquantized_gguf
-        print(f"  ✅ Using f16 GGUF: {LOCAL_GGUF_PATH}")
-
-    return LOCAL_GGUF_PATH
-
 
-def step4_create_ollama_model(gguf_path: Path):
-    """Create the Ollama model from the GGUF file."""
-    print("\n[4/4] Creating Ollama model 'mistral-vidyabot'...")
+def step2_create_ollama_model():
+    """Create the Ollama model from the downloaded GGUF file."""
+    print("\n[2/2] Creating Ollama model 'mistral-vidyabot'...")
 
     modelfile_path = LOCAL_MODELS_DIR / "Modelfile"
 
     # Write the Modelfile pointing to our GGUF
-    # Use absolute path for FROM directive
-    abs_gguf = gguf_path.resolve()
+    abs_gguf = LOCAL_GGUF_PATH.resolve()
+    
+    # Check if Modelfile exists and update it
     modelfile_content = f"""FROM {abs_gguf}
 
 # VidyaBot fine-tuned model parameters
@@ -143,6 +84,10 @@ PARAMETER num_ctx 2048
 PARAMETER temperature 0.7
 PARAMETER top_p 0.9
 PARAMETER repeat_penalty 1.1
+PARAMETER stop "<s>"
+PARAMETER stop "</s>"
+PARAMETER stop "[INST]"
+PARAMETER stop "[/INST]"
 
 # System prompt baked into the model
 SYSTEM "You are VidyaBot, an expert AI tutor for Indian school students studying NCERT curriculum. You give clear, concise, and accurate answers in 2-4 sentences. Always use correct scientific or mathematical terminology as found in NCERT textbooks."
@@ -166,31 +111,31 @@ SYSTEM "You are VidyaBot, an expert AI tutor for Indian school students studying
 
 def main():
     print("=" * 60)
-    print("  VidyaBot GGUF Conversion Pipeline")
-    print("  Modal → HuggingFace → GGUF → Ollama")
+    print("  VidyaBot GGUF Download & Registration Pipeline")
+    print("  Modal Volume -> Local GGUF -> Ollama")
     print("=" * 60)
 
     # Check if model already downloaded locally
-    if LOCAL_HF_MODEL_DIR.exists() and any(LOCAL_HF_MODEL_DIR.iterdir()):
-        print(f"\n  ✅ Model already downloaded at: {LOCAL_HF_MODEL_DIR}")
-        skip = input("  Skip download step? [Y/n]: ").strip().lower()
+    if LOCAL_GGUF_PATH.exists():
+        print(f"\n  ✅ GGUF Model already exists at: {LOCAL_GGUF_PATH}")
+        if not sys.stdin.isatty():
+            print("  Non-interactive mode detected. Skipping download step...")
+            skip = "y"
+        else:
+            skip = input("  Skip download step? [Y/n]: ").strip().lower()
         if skip != "n":
             print("  Skipping download...")
-            step2_get_llamacpp()
-            gguf_path = step3_convert_to_gguf()
-            step4_create_ollama_model(gguf_path)
+            step2_create_ollama_model()
             return
 
     step1_download_from_modal()
-    step2_get_llamacpp()
-    gguf_path = step3_convert_to_gguf()
-    step4_create_ollama_model(gguf_path)
+    step2_create_ollama_model()
 
     print("\n" + "=" * 60)
     print("  ✅ ALL STEPS COMPLETE")
     print()
     print("  Your fine-tuned model is ready:")
-    print(f"  GGUF: {gguf_path}")
+    print(f"  GGUF: {LOCAL_GGUF_PATH}")
     print(f"  Ollama: mistral-vidyabot")
     print()
     print("  Update your .env file:")

modal_finetune.py

@@ -28,6 +28,7 @@ MODEL_OUTPUT_DIR = "/model-output"
 # Container image with all ML dependencies
 image = (
     modal.Image.debian_slim(python_version="3.11")
+    .apt_install("git", "build-essential", "cmake")
     .pip_install(
         "transformers>=4.40.0",
         "torch>=2.2.0",
@@ -39,6 +40,7 @@ image = (
         "scipy",
         "sentencepiece",
         "protobuf",
+        "gguf>=0.1.0",
     )
     .add_local_file(
         "data/finetuning/student_qa.jsonl",
@@ -233,27 +235,94 @@ def finetune_mistral():
     print(f"     Loss: {train_result.training_loss:.4f}")
     print(f"     Runtime: {train_result.metrics.get('train_runtime', 0):.0f}s")
 
-    # ── Step 7: Merge LoRA into base model + save ─────────────
-    print(f"\n[7/7] Merging LoRA weights and saving to {MODEL_OUTPUT_DIR}...")
+    # ── Step 7: Merge LoRA into base model + GGUF Conversion ─────────────
+    print(f"\n[7/7] Merging LoRA weights and converting to GGUF format...")
+
+    # Save the adapter first (PEFT)
+    adapter_path = "/tmp/vidyabot-adapter"
+    model.save_pretrained(adapter_path)
+    print(f"  ✅ PEFT adapter saved to: {adapter_path}")
+
+    # Free memory of the 4-bit model to avoid OOM
+    del model
+    del trainer
+    import gc
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    # Load base model in float16 (unquantized) to merge
+    print("\n  Loading base model in float16 to merge LoRA adapters...")
+    from peft import PeftModel
+    base_model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype=torch.float16,
+        device_map="auto",
+    )
+    peft_model = PeftModel.from_pretrained(base_model, adapter_path)
+    print("  Merging weights...")
+    merged_model = peft_model.merge_and_unload()
+
+    # Save to /tmp to avoid Volume size exhaustion
+    temp_merged_path = "/tmp/mistral-vidyabot-merged"
+    print(f"  Saving merged float16 model to {temp_merged_path}...")
+    merged_model.save_pretrained(temp_merged_path, safe_serialization=True)
+    tokenizer.save_pretrained(temp_merged_path)
+
+    # Free memory again before starting conversion/quantization
+    del merged_model
+    del peft_model
+    del base_model
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    # Clone llama.cpp to /tmp
+    print("\n  Cloning llama.cpp for GGUF conversion...")
+    import subprocess
+    subprocess.run(
+        "git clone --depth 1 https://github.com/ggerganov/llama.cpp /tmp/llama.cpp",
+        shell=True,
+        check=True
+    )
 
-    # Merge LoRA into base model for full weights (needed for GGUF conversion)
-    merged_model = model.merge_and_unload()
+    # Convert HF to GGUF f16
+    temp_gguf_f16 = "/tmp/mistral-vidyabot-f16.gguf"
+    print(f"\n  Converting HF model to GGUF f16: {temp_gguf_f16}...")
+    subprocess.run(
+        f"python /tmp/llama.cpp/convert_hf_to_gguf.py {temp_merged_path} "
+        f"--outfile {temp_gguf_f16} --outtype f16",
+        shell=True,
+        check=True
+    )
 
-    output_path = f"{MODEL_OUTPUT_DIR}/mistral-vidyabot-merged"
-    merged_model.save_pretrained(output_path, safe_serialization=True)
-    tokenizer.save_pretrained(output_path)
+    # Compile llama-quantize
+    print("\n  Compiling llama-quantize tool...")
+    subprocess.run(
+        "cmake -B /tmp/llama.cpp/build -S /tmp/llama.cpp -DCMAKE_BUILD_TYPE=Release && "
+        "cmake --build /tmp/llama.cpp/build --config Release --target llama-quantize",
+        shell=True,
+        check=True
+    )
 
-    # Commit to Modal Volume
+    # Quantize to Q4_K_M and save directly to persistent volume!
+    final_gguf_path = f"{MODEL_OUTPUT_DIR}/mistral-vidyabot.Q4_K_M.gguf"
+    print(f"\n  Quantizing GGUF f16 -> Q4_K_M: {final_gguf_path}...")
+    subprocess.run(
+        f"/tmp/llama.cpp/build/bin/llama-quantize {temp_gguf_f16} {final_gguf_path} Q4_K_M",
+        shell=True,
+        check=True
+    )
+
+    # Commit Volume
     volume.commit()
 
-    print(f"\n  ✅ Merged model saved to: {output_path}")
-    print("  📦 Model committed to Modal Volume 'vidyabot-model-output'")
+    print(f"\n  ✅ Merged model successfully quantized to GGUF!")
+    print(f"  📦 GGUF model committed to Modal Volume 'vidyabot-model-output'")
     print("\n" + "=" * 60)
-    print("  FINE-TUNING COMPLETE")
-    print("  Next: run `modal volume get vidyabot-model-output mistral-vidyabot-merged ./backend/llm/models/mistral-vidyabot-hf`")
+    print("  FINE-TUNING & GGUF CONVERSION COMPLETE")
+    print("  Next: run `python modal_convert_gguf.py` to download the GGUF file")
     print("=" * 60)
 
-    return output_path
+    return final_gguf_path
 
 
 # ──────────────────────────────────────────────────────────────
@@ -290,11 +359,8 @@ def main():
     print()
 
     result = finetune_mistral.remote()
-    print(f"\n✅ Job complete! Weights saved to Modal Volume.")
+    print(f"\n✅ Job complete! GGUF Model saved to Modal Volume.")
     print(f"   Path: {result}")
     print()
-    print("📥 To download the merged model, run:")
-    print("   modal volume get vidyabot-model-output mistral-vidyabot-merged ./backend/llm/models/mistral-vidyabot-hf")
-    print()
-    print("🔧 Then convert to GGUF and load into Ollama:")
+    print("📥 To download the GGUF model and set up Ollama, run:")
     print("   python modal_convert_gguf.py")