hugging face implementation CPU/gguf

.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

Dockerfile

@@ -0,0 +1,50 @@
+# # 1. Start with an OS that already has CUDA and PyTorch installed
+# FROM pytorch/pytorch:2.1.2-cuda12.1-cudnn8-runtime
+
+# # 2. Set the working directory inside the container
+# WORKDIR /app
+
+# # 3. Copy your requirements file and install dependencies
+# # (You will need to create a requirements.txt with fastapi, uvicorn, transformers, bitsandbytes, faiss-cpu, sentence-transformers, etc.)
+# COPY requirements.txt .
+# RUN pip install --no-cache-dir -r requirements.txt
+
+# # 4. Copy your actual code and knowledge base into the container
+# COPY . .
+
+# # 5. Open port 8000 so the outside world can talk to the API
+# EXPOSE 8000
+
+# # 6. Start the FastAPI server when the container boots
+# CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
+
+
+
+
+# 1. Use a lightweight Python base
+FROM python:3.11-slim
+
+# 2. Install system dependencies for building llama-cpp on a CPU
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    cmake \
+    python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# 3. Set up a non-root user (Strict Hugging Face Security Requirement)
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+# 4. Install your Python dependencies
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# 5. Copy your actual code and knowledge base into the container
+COPY --chown=user . .
+
+# 6. Run the app using Python so it triggers the dynamic port logic in app.py
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,55 @@
+from fastapi import FastAPI
+from fastapi.responses import FileResponse
+from fastapi.middleware.cors import CORSMiddleware
+
+import uvicorn
+import os
+
+# Import our custom modules
+from schemas import BotRequest
+from llm_cpu import load_llm, generate_answer
+from rag import initialize_vector_store, retrieve_context
+
+app = FastAPI(title="AskJulia-Modular-v1")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# 1. Initialize Systems on Startup
+llm = load_llm()
+faiss_index = initialize_vector_store()
+print("--- Server Ready ---")
+
+# 2. Serve the HTML Interface
+@app.get("/")
+async def get_ui():
+    # Instead of a massive string in Python, we just serve the raw HTML file
+    return FileResponse("index.html")
+
+# 3. Handle the Ask Julia API
+@app.post("/explain")
+async def explain(request: BotRequest):
+    clean_query = request.user_query.lower().replace("@bot", "").strip()
+    
+    # Step A: Find relevant notes from the database
+    retrieved_context = retrieve_context(faiss_index, clean_query)
+    
+    # Step B: Ask the LLM to generate the final answer
+    reply = generate_answer(llm, retrieved_context, clean_query)
+    
+    return {"reply": reply}
+
+if __name__ == "__main__":
+    # uvicorn.run(app, host="0.0.0.0", port=8000)
+
+    # Hugging Face provides the PORT environment variable.
+    # If it's not found (like when you run it locally), it defaults to 7860.
+    port = int(os.environ.get("PORT", 7860))
+    
+    # "0.0.0.0" is critical. It tells the server to accept connections 
+    # from outside the Docker container (which is how Hugging Face routes traffic).
+    uvicorn.run(app, host="0.0.0.0", port=port)

faiss_index.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b75585cf78a809f3d975954231d8b2aaef18958eebb03ac1c80430098cdc27a
+size 6189

faiss_metadata.json

@@ -0,0 +1 @@
+["# Internal Notes: Flow Matching Vectors\nIn our specific implementation for diffusion.fyi, we define the probability path $p_t(x)$ as a linear interpolation between noise $p_0$ and data $p_1$.", "The \"Ask Julia\" velocity field $v_t(x)$ is specifically designed to minimize the \"curviness\" of the flow. We use a unique constant called the **Julia-Constant ($J_c$)** set at $0.42$ for all our Euler integrations to ensure stability in high-dimensions.", "# Integration Techniques for Flow\nWhile standard ODE solvers work, we recommend the **Hyper-Step Method**. This method uses a variable step size where $dt$ is inversely proportional to the local gradient of the vector field.", "Warning: If the velocity $v_t$ exceeds a magnitude of $10.0$, the Hyper-Step will automatically halve the integration time to prevent divergence."]

index.html

@@ -0,0 +1,249 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Ask Julia Assistant</title>
+    
+    <script>
+        window.MathJax = { tex: { inlineMath: [['$', '$'], ['\\(', '\\)']] } };
+    </script>
+    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
+    <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
+    <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+    
+    <style>
+        :root { --sidebar-width: 450px; }
+        body { font-family: 'Inter', sans-serif; background: #0f172a; color: #f8fafc; display: flex; height: 100vh; margin: 0; overflow: hidden; }
+        
+        #content-area { 
+            flex: 1; padding: 40px; overflow-y: auto; 
+            transition: padding-right 0.3s ease; 
+        }
+        body.sidebar-active #content-area {
+            padding-right: calc(var(--sidebar-width) + 40px);
+        }
+        
+        #sidebar { 
+            position: fixed; right: 0; top: 0; height: 100%;
+            width: var(--sidebar-width); background: #1e293b; 
+            display: flex; flex-direction: column; 
+            box-shadow: -5px 0 15px rgba(0,0,0,0.3);
+            transform: translateX(100%); transition: transform 0.3s ease;
+            z-index: 100; padding: 20px; box-sizing: border-box;
+        }
+        #sidebar.open { transform: translateX(0); }
+
+        #sidebar-tab {
+            position: absolute; left: -40px; top: 50%; transform: translateY(-50%);
+            width: 40px; height: 70px; background: #1e293b; color: #38bdf8;
+            border: none; border-radius: 8px 0 0 8px; cursor: pointer;
+            box-shadow: -5px 0 10px rgba(0,0,0,0.1); display: flex;
+            align-items: center; justify-content: center; font-size: 1.2rem;
+            border-left: 1px solid #334155; border-top: 1px solid #334155; border-bottom: 1px solid #334155;
+            padding: 0; transition: background 0.2s;
+        }
+        #sidebar-tab:hover { background: #334155; }
+
+        h2 { color: #38bdf8; }
+        .research-text { line-height: 1.8; font-size: 1.1rem; color: #cbd5e1; }
+        
+        #chat-output { flex: 1; overflow-y: auto; margin-bottom: 10px; font-size: 0.95rem; line-height: 1.6; border-bottom: 1px solid #334155; padding-bottom: 10px; }
+        
+        .ai-msg { background: #0f172a; padding: 15px; border-radius: 8px; margin-top: 10px; border-left: 4px solid #38bdf8; }
+        
+        ::selection { background: #38bdf8; color: #0f172a; }
+        .cursor-blink { animation: blink 1s step-end infinite; color: #38bdf8; }
+        @keyframes blink { 50% { opacity: 0; } }
+
+        /* FOOLPROOF FLEXBOX INPUT LAYOUT */
+        .input-container {
+            display: flex;
+            align-items: flex-end; /* Keeps button at the bottom when typing long messages */
+            gap: 10px;
+            margin-top: 10px;
+            width: 100%;
+        }
+
+        #chat-input { 
+            flex: 1; 
+            background: #0f172a; border: 1px solid #334155; color: white; 
+            padding: 12px; 
+            border-radius: 8px; box-sizing: border-box; outline: none; 
+            resize: none; overflow-y: hidden; min-height: 45px; max-height: 150px; 
+            font-family: inherit; font-size: 0.95rem;
+        }
+        #chat-input:focus { border-color: #38bdf8; }
+
+        #send-btn {
+            flex-shrink: 0; /* Prevents button from shrinking */
+            background: #38bdf8;
+            color: #0f172a;
+            border: none;
+            border-radius: 8px;
+            width: 45px;
+            height: 45px; /* Exactly matches the minimum height of the text area */
+            cursor: pointer;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            transition: transform 0.1s, background 0.2s, opacity 0.2s;
+        }
+
+        #send-btn:hover { background: #7dd3fc; }
+        #send-btn:active { transform: scale(0.9); }
+
+        /* Mobile Adjustment */
+        @media (max-width: 600px) {
+            :root { --sidebar-width: 100%; }
+            #sidebar-tab { width: 50px; height: 50px; left: -50px; }
+        }
+    </style>
+</head>
+<body>
+    <div id="content-area">
+        <h2>Research Material</h2>
+        <div class="research-text">
+            <p>In Flow Matching, the transition is defined by the ODE: $dx = v_t(x)dt$. 
+            This vector field $v_t$ defines the velocity at each point in space-time to push samples from $p_0$ to $p_1$.</p>
+            
+            <p>The forward diffusion process adds noise $q(x_t | x_{t-1}) = \mathcal{N}(x_t; \sqrt{1-\beta_t}x_{t-1}, \beta_t I)$. 
+            As $t$ increases, the original structure of the image is lost to Gaussian randomness.</p>
+            
+            <p>Copy any text above and paste it into the sidebar with your question!</p>
+        </div>
+    </div>
+
+    <div id="sidebar">
+        <button id="sidebar-tab" onclick="toggleSidebar()" title="Toggle Ask Julia">
+            <span id="tab-icon">◀</span>
+        </button>
+
+        <h3>Ask Julia</h3>
+        <div id="chat-output"> </div>
+        
+        <div class="input-container">
+            <textarea id="chat-input" rows="1" placeholder="@bot explain..."></textarea>
+            <button id="send-btn" onclick="askBot()" title="Send">
+                <svg viewBox="0 0 24 24" width="20" height="20" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round"><line x1="22" y1="2" x2="11" y2="13"></line><polygon points="22 2 15 22 11 13 2 9 22 2"></polygon></svg>
+            </button>
+        </div>
+        
+    </div>
+
+<script>
+        const sidebar = document.getElementById('sidebar');
+        const output = document.getElementById('chat-output');
+        const input = document.getElementById('chat-input');
+        const sendBtn = document.getElementById('send-btn');
+        const tabIcon = document.getElementById('tab-icon');
+
+        function toggleSidebar() {
+            sidebar.classList.toggle('open');
+            document.body.classList.toggle('sidebar-active');
+            
+            if (sidebar.classList.contains('open')) {
+                tabIcon.innerText = "▶"; 
+            } else {
+                tabIcon.innerText = "◀"; 
+            }
+        }
+
+        input.addEventListener('input', function() {
+            this.style.height = 'auto'; 
+            this.style.height = this.scrollHeight + 'px'; 
+            this.style.overflowY = this.scrollHeight > 150 ? 'auto' : 'hidden';
+        });
+
+        function renderTextWithMath(text) {
+            const mathBlocks = [];
+            const mathRegex = /(\\$\\$[\\s\\S]*?\\$\\$|\\$[^\\$]*?\\$|\\\\\\[[\\s\\S]*?\\\\\\]|\\\\\\([\\s\\S]*?\\\\\\))/g;
+            
+            let placeholderText = text.replace(mathRegex, (match) => {
+                mathBlocks.push(match);
+                return `@@MATH_BLOCK_${mathBlocks.length - 1}@@`;
+            });
+            
+            let html = marked.parse(placeholderText);
+            
+            mathBlocks.forEach((math, index) => {
+                html = html.replace(`@@MATH_BLOCK_${index}@@`, math);
+            });
+            
+            return html;
+        }
+
+        function typeARResponse(text, element) {
+            return new Promise((resolve) => {
+                let i = 0;
+                let currentText = "";
+                const speed = 10; 
+                const chunkSize = 2; 
+
+                const interval = setInterval(() => {
+                    currentText += text.substring(i, i + chunkSize);
+                    i += chunkSize;
+                    
+                    element.innerHTML = renderTextWithMath(currentText) + '<span class="cursor-blink">▌</span>';
+                    output.scrollTop = output.scrollHeight; 
+                    
+                    if (i >= text.length) {
+                        clearInterval(interval);
+                        element.innerHTML = renderTextWithMath(text);
+                        resolve();
+                    }
+                }, speed);
+            });
+        }
+
+        async function askBot() {
+            const query = input.value.trim();
+            if(!query.includes('@bot')) return;
+
+            // Disable UI while waiting for response
+            sendBtn.style.opacity = "0.5";
+            sendBtn.disabled = true;
+            input.disabled = true;
+
+            output.innerHTML += `<div><b>You:</b> ${query}</div><div class="ai-msg" id="last-response">Thinking...</div>`;
+            
+            input.value = "";
+            input.style.height = 'auto'; 
+            output.scrollTop = output.scrollHeight;
+
+            try {
+                const res = await fetch('/explain', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ page_text: "Context provided in query.", user_query: query })
+                });
+                const data = await res.json();
+                
+                const lastResponse = document.getElementById('last-response');
+                lastResponse.removeAttribute('id'); 
+                
+                await typeARResponse(data.reply, lastResponse);
+                
+                if (window.MathJax) MathJax.typesetPromise([lastResponse]);
+                
+            } catch (err) {
+                document.getElementById('last-response').innerHTML = "<span style='color: #f87171;'>Error: Could not connect to backend.</span>";
+            }
+            
+            // Re-enable UI
+            sendBtn.style.opacity = "1";
+            sendBtn.disabled = false;
+            input.disabled = false;
+            input.focus();
+            output.scrollTop = output.scrollHeight;
+        }
+
+        input.addEventListener('keydown', (e) => {
+            if(e.key === 'Enter' && !e.shiftKey) {
+                e.preventDefault(); 
+                askBot();
+            }
+        });
+    </script>
+</body>
+</html>

knowledge_base/flow_matching_logic.md

@@ -0,0 +1,4 @@
+# Internal Notes: Flow Matching Vectors
+In our specific implementation for diffusion.fyi, we define the probability path $p_t(x)$ as a linear interpolation between noise $p_0$ and data $p_1$. 
+
+The "Ask Julia" velocity field $v_t(x)$ is specifically designed to minimize the "curviness" of the flow. We use a unique constant called the **Julia-Constant ($J_c$)** set at $0.42$ for all our Euler integrations to ensure stability in high-dimensions.

knowledge_base/integration_methods.md

@@ -0,0 +1,4 @@
+# Integration Techniques for Flow
+While standard ODE solvers work, we recommend the **Hyper-Step Method**. This method uses a variable step size where $dt$ is inversely proportional to the local gradient of the vector field.
+
+Warning: If the velocity $v_t$ exceeds a magnitude of $10.0$, the Hyper-Step will automatically halve the integration time to prevent divergence.

llm_cpu.py

@@ -0,0 +1,58 @@
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+import os
+
+def load_llm():
+    print("--- Downloading/Loading Qwen2.5 1.5B GGUF (4-bit Quantized) ---")
+    
+    # This downloads the heavily compressed CPU version of Qwen
+    # It caches it locally, so it only downloads once!
+    model_path = hf_hub_download(
+        repo_id="Qwen/Qwen2.5-1.5B-Instruct-GGUF",
+        filename="qwen2.5-1.5b-instruct-q4_k_m.gguf"
+    )
+    
+    # Load into standard CPU RAM
+    # n_threads should match your physical CPU cores (e.g., 4 or 8)
+    llm = Llama(
+        model_path=model_path,
+        n_ctx=2048,       # How much text it can remember at once
+        n_threads=4,      # Number of CPU cores to use
+        verbose=False     # Hides the messy C++ logs
+    )
+    print("--- CPU Model Loaded Successfully ---")
+    return llm
+
+def generate_answer(llm, retrieved_context, query):
+    # Qwen-specific ChatML format for the CPU engine
+    # This guarantees the model stays in 'Julia Assistant' mode
+    prompt = (
+        f"<|im_start|>system\n"
+        f"You are the 'Ask Julia' assistant for diffusion.fyi. "
+        f"Your tone is encouraging and academic. "
+        f"You MUST use LaTeX for math (e.g., $x$ or $$y=x$$). "
+        f"CRITICAL INSTRUCTION: You may ONLY answer based on the provided CONTEXT. "
+        f"If the exact name or concept asked by the user is NOT in the CONTEXT, you MUST reply with EXACTLY: 'I don't have that specific information in my notes.' "
+        f"DO NOT guess, DO NOT talk about other constants instead, and DO NOT hallucinate.<|im_end|>\n"
+        f"<|im_start|>user\n"
+        f"CONTEXT:\n{retrieved_context}\n\n"
+        f"USER QUESTION: {query}<|im_end|>\n"
+        f"<|im_start|>assistant\n"
+    )
+
+    # Generation parameters tuned for CPU
+    output = llm(
+        prompt,
+        max_tokens=2048,
+        temperature=0.1,
+        stop=["<|im_end|>", "<|im_start|>"], 
+        echo=False # Prevents the prompt from being repeated in the answer
+    )
+    
+    response = output['choices'][0]['text'].strip()
+    
+    # Delimiter Cleanup (Safety net for the frontend)
+    response = response.replace('\\(', '$').replace('\\)', '$')
+    response = response.replace('\\[', '$$').replace('\\]', '$$')
+    
+    return response

main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from ask-julia-backend!")
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,16 @@
+fastapi
+uvicorn
+pydantic
+
+faiss-cpu
+sentence-transformers
+numpy
+
+# CPU Related -- helped for quantisation
+#llama-cpp-python
+#huggingface-hub
+
+# GPU Related
+transformers
+accelerate
+bitsandbytes