Upload 12 files

.gitignore

@@ -0,0 +1,2 @@
+# OpenMemory - IDE/Assistant specific rules
+.windsurf\rules\openmemory.md

Dockerfile

@@ -0,0 +1,78 @@
+# Ornith-1.0-9B on a Hugging Face Space (Docker SDK) — OpenAI-compatible API.
+#
+# This is the CPU build: it runs on the FREE tier (2 vCPU / 16 GB, no GPU).
+# A 9B model on CPU is usable, not fast (~5-10 tok/s). For real speed, use GPU
+# hardware — see the notes at the bottom of this file.
+#
+# Speed/robustness choices:
+#   * llama.cpp OpenAI-compatible server (prebuilt CPU wheel -> fast build).
+#   * Model GGUF baked into the image  -> instant cold start (no re-download,
+#     which matters because free Spaces have only ephemeral disk).
+#   * --chat_format chatml             -> applies a proper template, which
+#     kills the repetition loop AND enforces <|im_end|> as a stop (no more
+#     duplicated answers).
+
+FROM python:3.11-slim
+
+# curl/ca-certs only; the prebuilt wheel means we need NO compiler toolchain.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Hugging Face Spaces run the container as a non-root user with UID 1000.
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+# --- Python deps: prebuilt CPU wheel installs in seconds (no source build) ---
+RUN pip install --no-cache-dir --user "llama-cpp-python[server]" huggingface_hub
+
+# --- Model: which GGUF to serve ---------------------------------------------
+# Q4_K_M = good quality/size. For a bit more CPU speed swap to ornith-1.0-9b-Q4_0.gguf
+# (simpler kernels, faster on CPU) or ornith-1.0-9b-Q3_K_M.gguf (smaller, faster).
+# Confirm exact filenames on the repo's "Files" tab before changing.
+ENV MODEL_REPO=deepreinforce-ai/Ornith-1.0-9B-GGUF \
+    MODEL_FILE=ornith-1.0-9b-Q4_K_M.gguf \
+    MODEL_DIR=/home/user/models
+
+# Bake the weights into the image so the Space starts instantly.
+RUN python -c "from huggingface_hub import hf_hub_download; \
+    hf_hub_download(repo_id='${MODEL_REPO}', filename='${MODEL_FILE}', local_dir='${MODEL_DIR}')"
+
+# --- Serving params ----------------------------------------------------------
+# Free tier = 2 vCPU, so 2 threads. n_ctx kept modest to save CPU RAM.
+ENV N_CTX=8192 \
+    N_THREADS=2 \
+    OMP_NUM_THREADS=2
+
+# HF routes external traffic to this single port.
+EXPOSE 7860
+
+# OpenAI-compatible API at http://<space>.hf.space/v1  (chat/completions, models).
+CMD python -m llama_cpp.server \
+        --model ${MODEL_DIR}/${MODEL_FILE} \
+        --host 0.0.0.0 --port 7860 \
+        --n_ctx ${N_CTX} \
+        --n_threads ${N_THREADS} \
+        --n_batch 512 \
+        --chat_format chatml
+
+# -----------------------------------------------------------------------------
+# WANT IT ACTUALLY FAST?  Free HF has no GPU. Two options:
+#
+# 1) HF GPU hardware (paid, e.g. "T4 small"): change the base image to a CUDA
+#    one and install the CUDA wheel, then set --n_gpu_layers -1:
+#
+#        FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04
+#        ... install python3 ...
+#        RUN pip install "llama-cpp-python[server]" \
+#              --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124
+#        CMD python3 -m llama_cpp.server --model ... --n_gpu_layers -1 \
+#              --host 0.0.0.0 --port 7860 --n_ctx 16384 --chat_format chatml
+#
+#    On a T4 this jumps to ~40-70 tok/s.
+#
+# 2) ZeroGPU Space (free-ish, needs PRO): that path uses the Gradio SDK +
+#    @spaces.GPU with transformers/vLLM — NOT this llama.cpp Dockerfile.
+# -----------------------------------------------------------------------------

README_SPACE.md

@@ -0,0 +1,61 @@
+---
+title: Ornith 1.0 9B API
+emoji: 🦅
+colorFrom: indigo
+colorTo: purple
+sdk: docker
+app_port: 7860
+pinned: false
+license: mit
+---
+
+# Ornith-1.0-9B — OpenAI-compatible API (llama.cpp)
+
+Serves `deepreinforce-ai/Ornith-1.0-9B` (Q4_K_M GGUF) as an **OpenAI-compatible
+REST API** via llama.cpp's built-in server.
+
+> **Note:** This is the CPU build for the free tier (2 vCPU, no GPU). A 9B model
+> on CPU is usable but slow (~5-10 tok/s). See the Dockerfile footer for the GPU
+> variant if you need real speed.
+
+## Endpoints
+
+- `GET  /v1/models`
+- `POST /v1/chat/completions`  (streaming supported)
+- `POST /v1/completions`
+
+Base URL: `https://<your-username>-<space-name>.hf.space/v1`
+
+## Use it
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="https://<your-username>-<space-name>.hf.space/v1",
+    api_key="not-needed",           # llama.cpp server ignores it by default
+)
+
+resp = client.chat.completions.create(
+    model="ornith",
+    messages=[{"role": "user", "content": "Write a Python LRU cache with a docstring."}],
+    temperature=0.6, top_p=0.95,
+    stream=True,
+)
+for chunk in resp:
+    print(chunk.choices[0].delta.content or "", end="")
+```
+
+Or with curl:
+
+```bash
+curl https://<your-username>-<space-name>.hf.space/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{"model":"ornith","messages":[{"role":"user","content":"hello"}],"temperature":0.6}'
+```
+
+## Deploy
+
+1. Create a new Space → **Docker** (blank template).
+2. Add this `Dockerfile` and rename this file to `README.md` in the Space repo.
+3. Push. First build takes a few minutes (it bakes the ~5.5 GB model into the image).

hf-space-rag/.gitignore

@@ -0,0 +1,5 @@
+# Baked into the Docker image at build time — do not commit.
+models/
+.cache/
+__pycache__/
+*.pyc

hf-space-rag/Dockerfile

@@ -0,0 +1,52 @@
+# CPU + FREE-tier RAG Space (Docker SDK).
+#   Embeddings : BAAI/bge-small-en-v1.5  (fastembed / ONNX, no torch)
+#   Vector DB  : FAISS (in-memory)
+#   LLM        : Qwen2.5-1.5B-Instruct   (llama.cpp GGUF, fast on CPU)
+#   API        : OpenAI-compatible  ->  /v1/chat/completions  (+ web UI at /)
+#
+# Everything is CPU-only. No GPU, no paid hardware required.
+# Models are baked into the image so the Space cold-starts instantly.
+
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl ca-certificates && rm -rf /var/lib/apt/lists/*
+
+# Hugging Face Spaces run the container as non-root UID 1000.
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR /home/user/app
+
+# --- Python deps (prebuilt wheels -> fast, no build toolchain) ---
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir --user -r requirements.txt
+
+# --- Model config ------------------------------------------------------------
+ENV LLM_REPO=Qwen/Qwen2.5-1.5B-Instruct-GGUF \
+    LLM_FILE=qwen2.5-1.5b-instruct-q4_k_m.gguf \
+    MODEL_DIR=/home/user/models \
+    EMBED_MODEL=BAAI/bge-small-en-v1.5 \
+    FASTEMBED_CACHE=/home/user/.cache/fastembed \
+    HF_HOME=/home/user/.cache/huggingface
+
+# Bake the LLM (~1 GB) into the image -> instant cold start.
+RUN python -c "from huggingface_hub import hf_hub_download; \
+    hf_hub_download(repo_id='${LLM_REPO}', filename='${LLM_FILE}', local_dir='${MODEL_DIR}')"
+
+# Bake the embedding model (ONNX ~130 MB) into the image too.
+RUN python -c "import os; from fastembed import TextEmbedding; \
+    TextEmbedding(os.environ['EMBED_MODEL'], cache_dir=os.environ['FASTEMBED_CACHE'])"
+
+# --- App ---------------------------------------------------------------------
+COPY --chown=user . .
+
+# Runtime knobs (free tier = 2 vCPU).
+ENV N_CTX=8192 \
+    N_THREADS=2 \
+    TOP_K=4 \
+    DOCS_DIR=documents
+
+EXPOSE 7860
+CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

hf-space-rag/README.md

@@ -0,0 +1,67 @@
+---
+title: CPU RAG Space
+emoji: 🦅
+colorFrom: indigo
+colorTo: purple
+sdk: docker
+app_port: 7860
+pinned: false
+license: mit
+---
+
+# CPU RAG Space — Qwen2.5-1.5B + FAISS (free tier, no GPU)
+
+A self-contained Retrieval-Augmented Generation service that runs **entirely on
+CPU** and fits the Hugging Face **free tier** (2 vCPU / 16 GB).
+
+| Component | What | Why |
+|-----------|------|-----|
+| Embeddings | `BAAI/bge-small-en-v1.5` via `fastembed` (ONNX) | fast on CPU, no PyTorch, ~130 MB |
+| Vector DB | **FAISS** (in-memory) | tiny, instant search |
+| LLM | `Qwen2.5-1.5B-Instruct` GGUF Q4_K_M via llama.cpp | fast on CPU (~12–20 tok/s), ~1 GB |
+| API | OpenAI-compatible `/v1/chat/completions` + web UI | drop-in for any client |
+
+Total footprint ≈ 1.5–2 GB RAM. Retrieval adds ~20 ms; the LLM is the only
+real latency.
+
+## Deploy (drag & drop)
+
+1. Create a new Space → **Docker** (blank template).
+2. Drag **all files in this folder** into the Space repo (keep the structure —
+   `documents/` included).
+3. Push. First build takes a few minutes (it bakes the ~1 GB LLM and the
+   embedder into the image so cold starts are instant).
+
+## Use it
+
+Web UI: open the Space URL. Upload `.txt`/`.md` files and ask questions.
+
+API (OpenAI-compatible):
+
+```python
+from openai import OpenAI
+client = OpenAI(base_url="https://<user>-<space>.hf.space/v1", api_key="x")
+r = client.chat.completions.create(
+    model="cpu-rag",
+    messages=[{"role": "user", "content": "How does retrieval work here?"}],
+)
+print(r.choices[0].message.content)
+```
+
+Extra endpoints: `POST /ingest` (upload a doc), `GET /stats`.
+
+## Add your own knowledge
+
+- Put `.txt`/`.md` files in `documents/` before pushing (indexed at startup), or
+- Upload them at runtime via the UI / `POST /ingest`.
+
+> Note: the free tier has ephemeral storage, so runtime-uploaded docs are lost
+> on restart. For a permanent corpus, commit files into `documents/`.
+
+## Swap the model
+
+Change these in the `Dockerfile` (confirm exact filenames on the repo's *Files*
+tab):
+
+- Faster / smaller: `Qwen/Qwen2.5-0.5B-Instruct-GGUF`
+- Coding-focused: `Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF`

hf-space-rag/app.py

@@ -0,0 +1,208 @@
+"""
+CPU RAG Space — bge-small (fastembed) + FAISS + Qwen2.5-1.5B (llama.cpp),
+served as an OpenAI-compatible API with a small web UI.
+
+Everything runs on CPU and fits the Hugging Face free tier (2 vCPU / 16 GB).
+"""
+
+import glob
+import json
+import os
+
+import faiss
+import numpy as np
+from fastapi import FastAPI, File, UploadFile
+from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
+from fastembed import TextEmbedding
+from llama_cpp import Llama
+from pydantic import BaseModel
+
+# --------------------------------------------------------------------------- #
+# Config (all overridable via Space "Variables")
+# --------------------------------------------------------------------------- #
+MODEL_DIR = os.environ.get("MODEL_DIR", "models")
+LLM_FILE = os.environ.get("LLM_FILE", "qwen2.5-1.5b-instruct-q4_k_m.gguf")
+LLM_PATH = os.path.join(MODEL_DIR, LLM_FILE)
+EMBED_MODEL = os.environ.get("EMBED_MODEL", "BAAI/bge-small-en-v1.5")
+FASTEMBED_CACHE = os.environ.get("FASTEMBED_CACHE")
+N_CTX = int(os.environ.get("N_CTX", "8192"))
+N_THREADS = int(os.environ.get("N_THREADS", str(os.cpu_count() or 2)))
+TOP_K = int(os.environ.get("TOP_K", "4"))
+DOCS_DIR = os.environ.get("DOCS_DIR", "documents")
+
+CHUNK_SIZE = 800        # characters per chunk
+CHUNK_OVERLAP = 120
+
+RAG_SYSTEM = (
+    "You are a helpful assistant. Answer the user's question using ONLY the "
+    "context below. If the answer is not in the context, say you don't know. "
+    "Cite the source of each fact in square brackets like [filename].\n\n"
+    "Context:\n{context}"
+)
+
+# --------------------------------------------------------------------------- #
+# Lazily-initialised singletons
+# --------------------------------------------------------------------------- #
+app = FastAPI(title="CPU RAG Space")
+
+_embedder = None
+_llm = None
+_index = None            # faiss.IndexFlatIP
+_chunks = []             # list[{"text": str, "source": str}]
+
+
+def embedder():
+    global _embedder
+    if _embedder is None:
+        _embedder = TextEmbedding(EMBED_MODEL, cache_dir=FASTEMBED_CACHE)
+    return _embedder
+
+
+def embed(texts):
+    vecs = np.array(list(embedder().embed(list(texts))), dtype="float32")
+    faiss.normalize_L2(vecs)          # cosine similarity via inner product
+    return vecs
+
+
+def llm():
+    global _llm
+    if _llm is None:
+        _llm = Llama(model_path=LLM_PATH, n_ctx=N_CTX, n_threads=N_THREADS,
+                     n_batch=512, verbose=False)
+    return _llm
+
+
+# --------------------------------------------------------------------------- #
+# Indexing / retrieval
+# --------------------------------------------------------------------------- #
+def chunk_text(text, source):
+    out, i, n = [], 0, len(text)
+    step = max(CHUNK_SIZE - CHUNK_OVERLAP, 1)
+    while i < n:
+        piece = text[i:i + CHUNK_SIZE].strip()
+        if piece:
+            out.append({"text": piece, "source": source})
+        i += step
+    return out
+
+
+def add_chunks(new_chunks):
+    global _index, _chunks
+    if not new_chunks:
+        return 0
+    vecs = embed([c["text"] for c in new_chunks])
+    if _index is None:
+        _index = faiss.IndexFlatIP(vecs.shape[1])
+    _index.add(vecs)
+    _chunks.extend(new_chunks)
+    return len(new_chunks)
+
+
+def build_index():
+    patterns = ("*.txt", "*.md")
+    files = []
+    for p in patterns:
+        files += glob.glob(os.path.join(DOCS_DIR, "**", p), recursive=True)
+    all_chunks = []
+    for f in files:
+        try:
+            with open(f, encoding="utf-8") as fh:
+                all_chunks += chunk_text(fh.read(), os.path.basename(f))
+        except Exception as exc:
+            print(f"[rag] skip {f}: {exc}")
+    add_chunks(all_chunks)
+
+
+def retrieve(query, k=TOP_K):
+    if _index is None or _index.ntotal == 0:
+        return []
+    scores, ids = _index.search(embed([query]), min(k, _index.ntotal))
+    hits = []
+    for score, idx in zip(scores[0], ids[0]):
+        if idx < 0:
+            continue
+        c = _chunks[idx]
+        hits.append({"text": c["text"], "source": c["source"], "score": float(score)})
+    return hits
+
+
+# --------------------------------------------------------------------------- #
+# Startup
+# --------------------------------------------------------------------------- #
+@app.on_event("startup")
+def _startup():
+    print("[rag] loading embedder + llm ...")
+    embedder()
+    llm()
+    build_index()
+    print(f"[rag] ready. indexed_chunks={len(_chunks)}")
+
+
+# --------------------------------------------------------------------------- #
+# OpenAI-compatible chat endpoint (with RAG)
+# --------------------------------------------------------------------------- #
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+
+
+class ChatRequest(BaseModel):
+    model: str = "cpu-rag"
+    messages: list[ChatMessage]
+    temperature: float = 0.3
+    top_p: float = 0.9
+    max_tokens: int = 512
+    stream: bool = False
+    use_rag: bool = True
+
+
+def _augment(req: ChatRequest):
+    msgs = [m.model_dump() for m in req.messages]
+    users = [m for m in msgs if m["role"] == "user"]
+    query = users[-1]["content"] if users else ""
+    ctxs = retrieve(query) if req.use_rag else []
+    if ctxs:
+        context = "\n\n".join(f"[{c['source']}] {c['text']}" for c in ctxs)
+        system = {"role": "system", "content": RAG_SYSTEM.format(context=context)}
+        msgs = [system] + [m for m in msgs if m["role"] != "system"]
+    return msgs, ctxs
+
+
+@app.post("/v1/chat/completions")
+def chat_completions(req: ChatRequest):
+    messages, ctxs = _augment(req)
+    params = dict(messages=messages, temperature=req.temperature, top_p=req.top_p,
+                  max_tokens=req.max_tokens, stop=["<|im_end|>", "<|endoftext|>"])
+
+    if req.stream:
+        def gen():
+            for chunk in llm().create_chat_completion(**params, stream=True):
+                yield f"data: {json.dumps(chunk)}\n\n"
+            yield "data: [DONE]\n\n"
+        return StreamingResponse(gen(), media_type="text/event-stream")
+
+    resp = llm().create_chat_completion(**params)
+    resp["sources"] = [{"source": c["source"], "score": round(c["score"], 3)} for c in ctxs]
+    return JSONResponse(resp)
+
+
+# --------------------------------------------------------------------------- #
+# Ingest / stats / UI
+# --------------------------------------------------------------------------- #
+@app.post("/ingest")
+async def ingest(file: UploadFile = File(...)):
+    text = (await file.read()).decode("utf-8", "ignore")
+    added = add_chunks(chunk_text(text, file.filename))
+    return {"file": file.filename, "added_chunks": added, "total_chunks": len(_chunks)}
+
+
+@app.get("/stats")
+def stats():
+    return {"indexed_chunks": len(_chunks), "embed_model": EMBED_MODEL,
+            "llm": LLM_FILE, "n_ctx": N_CTX, "threads": N_THREADS, "top_k": TOP_K}
+
+
+@app.get("/", response_class=HTMLResponse)
+def home():
+    with open(os.path.join(os.path.dirname(__file__), "index.html"), encoding="utf-8") as f:
+        return f.read()

hf-space-rag/documents/sample.txt

@@ -0,0 +1,25 @@
+CPU RAG Space — sample knowledge document.
+
+This Space is a fully CPU, free-tier Retrieval-Augmented Generation service.
+
+Architecture:
+- Embedding model: BAAI/bge-small-en-v1.5, run via fastembed (ONNX). It turns
+  text into 384-dimensional vectors and needs no GPU or PyTorch.
+- Vector store: FAISS (IndexFlatIP) holds the document vectors in memory and
+  returns the most similar chunks for a query using cosine similarity.
+- Language model: Qwen2.5-1.5B-Instruct in GGUF Q4_K_M form, served by
+  llama.cpp. It reads the retrieved chunks and writes a grounded answer.
+
+How retrieval works:
+1. Your question is embedded into a vector.
+2. FAISS finds the top-K most similar document chunks (default K = 4).
+3. Those chunks are inserted into the model's system prompt as context.
+4. The model answers using only that context and cites the source file.
+
+Why a small model is fine here:
+RAG moves knowledge out of the model's weights and into the retriever, so the
+model only needs to read and summarise the provided context rather than
+memorise facts. That makes a fast 1.5B model a good fit for CPU serving.
+
+Replace this file with your own .txt or .md documents, or upload files at
+runtime through the web UI, and the Space will answer questions about them.

hf-space-rag/index.html

@@ -0,0 +1,97 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>CPU RAG · Qwen2.5-1.5B + FAISS</title>
+<style>
+  :root { color-scheme: light dark; --bg:#0f1220; --panel:#1a1e33; --acc:#7c8cff; --mut:#8b90a8; }
+  * { box-sizing: border-box; }
+  body { margin:0; font-family: system-ui, sans-serif; background:var(--bg); color:#e7e9f3; }
+  header { padding:14px 18px; border-bottom:1px solid #2a2f4a; display:flex; gap:12px; align-items:center; }
+  header h1 { font-size:15px; margin:0; font-weight:600; }
+  header .tag { font-size:11px; color:var(--mut); background:var(--panel); padding:3px 8px; border-radius:20px; }
+  main { max-width:820px; margin:0 auto; padding:18px; }
+  #chat { display:flex; flex-direction:column; gap:12px; min-height:50vh; }
+  .msg { padding:11px 14px; border-radius:12px; max-width:88%; white-space:pre-wrap; line-height:1.45; font-size:14px; }
+  .user { align-self:flex-end; background:var(--acc); color:#fff; }
+  .bot  { align-self:flex-start; background:var(--panel); }
+  .src  { align-self:flex-start; font-size:11px; color:var(--mut); margin-top:-6px; }
+  form { display:flex; gap:8px; margin-top:16px; }
+  textarea { flex:1; resize:none; padding:11px; border-radius:10px; border:1px solid #2a2f4a; background:#12152a; color:#e7e9f3; font-size:14px; }
+  button { padding:0 16px; border:0; border-radius:10px; background:var(--acc); color:#fff; font-weight:600; cursor:pointer; }
+  button:disabled { opacity:.5; cursor:default; }
+  .bar { display:flex; gap:10px; align-items:center; margin-bottom:14px; font-size:12px; color:var(--mut); }
+  .bar input[type=file] { font-size:12px; color:var(--mut); }
+</style>
+</head>
+<body>
+<header>
+  <h1>🦅 CPU RAG</h1>
+  <span class="tag">Qwen2.5-1.5B · bge-small · FAISS</span>
+  <span class="tag" id="stat">…</span>
+</header>
+<main>
+  <div class="bar">
+    <label>Add a document (.txt/.md):</label>
+    <input type="file" id="file" accept=".txt,.md" />
+    <button id="up" type="button">Upload</button>
+    <span id="upmsg"></span>
+  </div>
+  <div id="chat"></div>
+  <form id="form">
+    <textarea id="q" rows="2" placeholder="Ask about your documents…"></textarea>
+    <button id="send" type="submit">Send</button>
+  </form>
+</main>
+<script>
+const chat = document.getElementById('chat');
+const stat = document.getElementById('stat');
+
+function add(cls, text) {
+  const d = document.createElement('div');
+  d.className = 'msg ' + cls; d.textContent = text;
+  chat.appendChild(d); chat.scrollIntoView({block:'end'}); return d;
+}
+async function refreshStats(){
+  try { const s = await (await fetch('/stats')).json();
+        stat.textContent = s.indexed_chunks + ' chunks indexed'; } catch(e){}
+}
+refreshStats();
+
+document.getElementById('up').onclick = async () => {
+  const f = document.getElementById('file').files[0];
+  if (!f) return;
+  document.getElementById('upmsg').textContent = 'uploading…';
+  const fd = new FormData(); fd.append('file', f);
+  const r = await (await fetch('/ingest', {method:'POST', body:fd})).json();
+  document.getElementById('upmsg').textContent = '+' + r.added_chunks + ' chunks';
+  refreshStats();
+};
+
+document.getElementById('form').onsubmit = async (e) => {
+  e.preventDefault();
+  const q = document.getElementById('q').value.trim();
+  if (!q) return;
+  document.getElementById('q').value = '';
+  document.getElementById('send').disabled = true;
+  add('user', q);
+  const thinking = add('bot', '…');
+  try {
+    const r = await fetch('/v1/chat/completions', {
+      method:'POST', headers:{'Content-Type':'application/json'},
+      body: JSON.stringify({ messages:[{role:'user', content:q}], temperature:0.3 })
+    });
+    const data = await r.json();
+    thinking.textContent = data.choices?.[0]?.message?.content ?? '(no answer)';
+    if (data.sources && data.sources.length) {
+      add('src', 'sources: ' + data.sources.map(s => s.source + ' (' + s.score + ')').join(', '));
+    }
+  } catch (err) {
+    thinking.textContent = 'Error: ' + err;
+  }
+  document.getElementById('send').disabled = false;
+};
+</script>
+</body>
+</html>

hf-space-rag/requirements.txt

@@ -0,0 +1,14 @@
+# Prebuilt CPU wheel for llama.cpp -> fast build, no compiler needed.
+--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+llama-cpp-python
+
+# Lightweight CPU embeddings via ONNX (no PyTorch -> small image, fast).
+fastembed
+faiss-cpu
+
+# API + serving
+fastapi
+uvicorn[standard]
+python-multipart
+huggingface_hub
+numpy

ornith_colab.py

@@ -0,0 +1,508 @@
+"""
+ornith_colab.py — run deepreinforce-ai/Ornith-1.0-9B in Google Colab as an AGENT.
+
+Goal: CPU-first, auto-use a T4 GPU if present. Engine: llama.cpp (via
+llama-cpp-python) on a GGUF quant — NOT vLLM (vLLM is slow to spin up and
+wants a big GPU; llama.cpp installs from a prebuilt wheel and runs the SAME
+GGUF on CPU or a 16 GB T4).
+
+Ornith-1.0-9B (model card + release, June 2026): ~9B dense reasoning model
+post-trained on Qwen3.5-9B, MIT license, emits <think>...</think> then the
+answer, and is built for *agentic coding / tool use*. Sampling: temp 0.6,
+top_p 0.95, top_k 20.
+
+What this file gives you
+------------------------
+  chat(msg)                  -> str          (just the answer)
+  generate(msg, history)     -> dict         ({reasoning, answer, raw})
+  stream_chat(msg, history)  -> yields chunks (live tokens)
+  run_agent(task, tools)     -> dict         (tool-calling agent loop)
+
+Why the agent loop is hand-rolled (researched):
+  The Ornith GGUFs have inconsistent tool-aware chat templates (same root
+  cause as the known repetition-loop bug when the chat template is missing).
+  So instead of relying on llama.cpp's function-calling handler, we inject
+  tool schemas Qwen/Hermes-style into the system prompt and parse
+  <tool_call>{...}</tool_call> blocks ourselves. This is template-independent
+  and works on any Ornith GGUF mirror.
+
+Colab usage:
+    !python ornith_colab.py                      # runs chat + agent demos
+  or from a cell:
+    from ornith_colab import chat, generate, run_agent
+"""
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+# --------------------------------------------------------------------------- #
+# Config
+# --------------------------------------------------------------------------- #
+GGUF_REPO = "AtomicChat/ornith-9b-GGUF"   # template-embedded mirror (avoids loop bug)
+GGUF_FILE = "*Q4_K_M*.gguf"               # ~5.5 GB; good for T4 or CPU RAM
+
+# Device: "cpu" (default — this build showcases CPU capability), "gpu", or
+# "auto". Override from a Colab cell: os.environ["ORNITH_DEVICE"] = "gpu".
+DEVICE = os.environ.get("ORNITH_DEVICE", "cpu").lower()
+
+N_CTX = 16384          # agents burn context on tool results — give them room
+TEMPERATURE = 0.6
+TOP_P = 0.95
+TOP_K = 20
+REPEAT_PENALTY = 1.05  # insurance against loops
+MAX_TOKENS = 2048
+
+# Char-code tags so raw special-token bytes never sit in source.
+_TAG = lambda *cs: "".join(chr(c) for c in cs)
+_IM_START = _TAG(60, 124, 105, 109, 95, 115, 116, 97, 114, 116, 124, 62)   # <|im_start|>
+_IM_END = _TAG(60, 124, 105, 109, 95, 101, 110, 100, 124, 62)             # <|im_end|>
+_THINK_CLOSE = _TAG(60, 47, 116, 104, 105, 110, 107, 62)                  # </think>
+
+FALLBACK_CHAT_TEMPLATE = (
+    "{%- for m in messages %}"
+    f"{_IM_START}" "{{ m['role'] }}\n{{ m['content'] }}" f"{_IM_END}" "\n"
+    "{%- endfor %}"
+    "{%- if add_generation_prompt %}"
+    f"{_IM_START}" "assistant\n"
+    "{%- endif %}"
+)
+
+_TOOL_CALL_RE = re.compile(r"<tool_call>\s*(\{.*?\})\s*</tool_call>", re.DOTALL)
+
+# End-of-turn stops (built from char codes). "<|im_end|>" is the ChatML turn
+# marker; "<|endoftext|>" is the tokenizer EOS.
+DEFAULT_STOP = [_IM_END, _TAG(60, 124, 101, 110, 100, 111, 102, 116, 101, 120, 116, 124, 62)]
+
+
+# --------------------------------------------------------------------------- #
+# Environment detection + install
+# --------------------------------------------------------------------------- #
+def _has_nvidia_gpu() -> bool:
+    if not shutil.which("nvidia-smi"):
+        return False
+    try:
+        subprocess.run(["nvidia-smi"], check=True,
+                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        return True
+    except Exception:
+        return False
+
+
+def _pip(*args: str) -> None:
+    subprocess.run([sys.executable, "-m", "pip", "install", "-q", *args], check=True)
+
+
+def _resolve_use_gpu() -> bool:
+    if DEVICE == "cpu":
+        return False
+    if DEVICE == "gpu":
+        return True
+    return _has_nvidia_gpu()   # "auto"
+
+
+def _ensure_deps(use_gpu: bool) -> None:
+    for pkg in ("huggingface_hub", "psutil"):
+        try:
+            __import__(pkg)
+        except ImportError:
+            _pip(pkg)
+
+    try:
+        import llama_cpp  # noqa: F401
+        return
+    except ImportError:
+        pass
+
+    if use_gpu:
+        for cu in ("cu124", "cu122", "cu121"):
+            try:
+                _pip("llama-cpp-python", "--extra-index-url",
+                     f"https://abetlen.github.io/llama-cpp-python/whl/{cu}")
+                import llama_cpp  # noqa: F401
+                print(f"[ornith] installed llama-cpp-python (CUDA {cu} wheel)")
+                return
+            except Exception:
+                continue
+        print("[ornith] no prebuilt CUDA wheel matched; building from source...")
+        env = dict(os.environ, CMAKE_ARGS="-DGGML_CUDA=on")
+        subprocess.run([sys.executable, "-m", "pip", "install", "-q",
+                        "--no-cache-dir", "llama-cpp-python"], check=True, env=env)
+    else:
+        _pip("llama-cpp-python")
+        print("[ornith] installed llama-cpp-python (CPU wheel)")
+
+
+# --------------------------------------------------------------------------- #
+# Model loading
+# --------------------------------------------------------------------------- #
+_LLM = None
+
+
+def load_model():
+    global _LLM
+    if _LLM is not None:
+        return _LLM
+
+    use_gpu = _resolve_use_gpu()
+    print(f"[ornith] device={DEVICE}  use_gpu={use_gpu}  ->  "
+          f"{'offloading all layers to GPU' if use_gpu else f'running on CPU ({os.cpu_count()} threads)'}")
+    _ensure_deps(use_gpu)
+
+    from llama_cpp import Llama
+
+    kwargs = dict(
+        repo_id=GGUF_REPO,
+        filename=GGUF_FILE,
+        n_ctx=N_CTX,
+        n_gpu_layers=(-1 if use_gpu else 0),
+        n_threads=(None if use_gpu else (os.cpu_count() or 2)),
+        n_batch=512,
+        flash_attn=use_gpu,       # faster KV attention on the T4
+        verbose=False,
+    )
+    print(f"[ornith] loading {GGUF_REPO} ({GGUF_FILE}) ... first run downloads ~5.5 GB")
+    try:
+        llm = Llama.from_pretrained(**kwargs)
+    except TypeError:
+        # older llama-cpp-python without flash_attn kwarg
+        kwargs.pop("flash_attn", None)
+        llm = Llama.from_pretrained(**kwargs)
+    except Exception as exc:
+        raise RuntimeError(
+            f"Failed to load GGUF from {GGUF_REPO}. Try another mirror "
+            f"(e.g. deepreinforce-ai/Ornith-1.0-9B-GGUF). Error: {exc}")
+
+    meta = getattr(llm, "metadata", {}) or {}
+    if not meta.get("tokenizer.chat_template"):
+        print("[ornith] no embedded chat template -> applying Qwen3 fallback")
+        from llama_cpp.llama_chat_format import Jinja2ChatFormatter
+        llm.chat_handler = Jinja2ChatFormatter(
+            template=FALLBACK_CHAT_TEMPLATE, eos_token=_IM_END, bos_token="",
+        ).to_chat_handler()
+
+    _LLM = llm
+    print("[ornith] model ready")
+    return _LLM
+
+
+# --------------------------------------------------------------------------- #
+# Core generation
+# --------------------------------------------------------------------------- #
+def _messages(user_msg, history):
+    msgs = list(history or [])
+    if user_msg is not None:
+        msgs.append({"role": "user", "content": user_msg})
+    return msgs
+
+
+def stream_chat(user_msg, history=None, max_tokens=MAX_TOKENS, stop=None):
+    """Yield text chunks live (includes the <think>...</think> block)."""
+    llm = load_model()
+    # Always enforce the end-of-turn stops (dedup while preserving order).
+    effective_stop = list(dict.fromkeys((stop or []) + DEFAULT_STOP))
+    stream = llm.create_chat_completion(
+        messages=_messages(user_msg, history),
+        max_tokens=max_tokens, temperature=TEMPERATURE, top_p=TOP_P,
+        top_k=TOP_K, repeat_penalty=REPEAT_PENALTY, stop=effective_stop, stream=True,
+    )
+    for chunk in stream:
+        delta = chunk["choices"][0]["delta"].get("content")
+        if delta:
+            yield delta
+
+
+def _split_think(text):
+    if _THINK_CLOSE in text:
+        reasoning, answer = text.split(_THINK_CLOSE, 1)
+        return reasoning.replace("<think>", "").strip(), answer.strip()
+    return "", text.strip()
+
+
+def generate(user_msg, history=None, max_tokens=MAX_TOKENS, stop=None):
+    """Structured, non-streaming call. Returns {reasoning, answer, raw}."""
+    raw = "".join(stream_chat(user_msg, history, max_tokens, stop))
+    reasoning, answer = _split_think(raw)
+    return {"reasoning": reasoning, "answer": answer, "raw": raw}
+
+
+def chat(user_msg, history=None, max_tokens=MAX_TOKENS) -> str:
+    """Just the final answer (reasoning stripped)."""
+    return generate(user_msg, history, max_tokens)["answer"]
+
+
+# --------------------------------------------------------------------------- #
+# Tool-calling agent loop
+# --------------------------------------------------------------------------- #
+def _tools_system_prompt(tools):
+    schemas = "\n".join(json.dumps(t["schema"]) for t in tools)
+    return (
+        "You are Ornith, an agentic assistant that can call tools to act.\n"
+        "You have access to these tools (JSON schemas):\n"
+        f"{schemas}\n\n"
+        "When you need a tool, emit EXACTLY one block per call:\n"
+        '<tool_call>{"name": "<tool_name>", "arguments": {<args>}}</tool_call>\n'
+        "You may emit multiple tool_call blocks in one turn. After you receive "
+        "the tool results, continue reasoning. When the task is fully done and "
+        "you need no more tools, reply with the final answer and NO tool_call block."
+    )
+
+
+def _parse_tool_calls(text):
+    calls = []
+    for m in _TOOL_CALL_RE.finditer(text):
+        try:
+            obj = json.loads(m.group(1))
+            calls.append({"name": obj.get("name"), "arguments": obj.get("arguments", {})})
+        except json.JSONDecodeError:
+            continue
+    return calls
+
+
+def run_agent(task, tools, max_steps=6, verbose=True):
+    """
+    Minimal tool-calling agent loop.
+
+    tools: list of {
+        "name": str,
+        "schema": {json schema shown to the model},
+        "fn": callable(**arguments) -> anything json-serializable,
+    }
+    Returns {answer, steps, transcript}.
+    """
+    registry = {t["name"]: t["fn"] for t in tools}
+    history = [{"role": "system", "content": _tools_system_prompt(tools)},
+               {"role": "user", "content": task}]
+    transcript = []
+
+    for step in range(1, max_steps + 1):
+        # Stop right after a tool_call so we can execute promptly.
+        out = generate(None, history, stop=["</tool_call>"])
+        raw = out["raw"]
+        # generate() stripped the closing tag via `stop`; restore it for parsing.
+        if "<tool_call>" in raw and "</tool_call>" not in raw:
+            raw = raw + "</tool_call>"
+        calls = _parse_tool_calls(raw)
+        history.append({"role": "assistant", "content": raw})
+
+        if verbose:
+            print(f"\n[agent step {step}] reasoning: {out['reasoning'][:200]}")
+            if calls:
+                print(f"[agent step {step}] tool calls: {calls}")
+
+        if not calls:
+            answer = out["answer"] or out["raw"].strip()
+            transcript.append({"step": step, "type": "final", "content": answer})
+            return {"answer": answer, "steps": step, "transcript": transcript}
+
+        # Execute every requested tool and feed results back as one user turn.
+        results = []
+        for c in calls:
+            fn = registry.get(c["name"])
+            if fn is None:
+                res = f"ERROR: unknown tool '{c['name']}'"
+            else:
+                try:
+                    res = fn(**(c["arguments"] or {}))
+                except Exception as exc:
+                    res = f"ERROR: {exc}"
+            results.append({"name": c["name"], "result": res})
+            transcript.append({"step": step, "type": "tool", "call": c, "result": res})
+            if verbose:
+                print(f"[agent step {step}] {c['name']} -> {str(res)[:200]}")
+
+        tool_msg = "\n".join(
+            f"<tool_response>{json.dumps(r, default=str)}</tool_response>" for r in results
+        )
+        history.append({"role": "user", "content": tool_msg})
+
+    return {"answer": "(stopped: max_steps reached)", "steps": max_steps,
+            "transcript": transcript}
+
+
+# --------------------------------------------------------------------------- #
+# Performance instrumentation: TPS, RAM, KV cache
+# --------------------------------------------------------------------------- #
+def _meta_int(llm, suffix):
+    """Read an int from GGUF metadata by key suffix (arch-agnostic)."""
+    for k, v in (getattr(llm, "metadata", {}) or {}).items():
+        if k.endswith(suffix):
+            try:
+                return int(v)
+            except (TypeError, ValueError):
+                pass
+    return None
+
+
+def kv_cache_report(llm):
+    """
+    Estimate KV-cache memory from the model's attention geometry.
+    KV bytes/token = n_layer * n_head_kv * (key_len + val_len) * bytes_per_elem
+    (llama.cpp defaults the KV cache to f16 = 2 bytes/element.)
+    """
+    n_layer = _meta_int(llm, ".block_count")
+    n_embd = _meta_int(llm, ".embedding_length")
+    n_head = _meta_int(llm, ".attention.head_count")
+    n_head_kv = _meta_int(llm, ".attention.head_count_kv") or n_head
+    key_len = _meta_int(llm, ".attention.key_length")
+    val_len = _meta_int(llm, ".attention.value_length")
+    head_dim = key_len or ((n_embd // n_head) if (n_embd and n_head) else None)
+    key_len = key_len or head_dim
+    val_len = val_len or head_dim
+
+    info = {"n_layer": n_layer, "n_head": n_head, "n_head_kv": n_head_kv,
+            "head_dim": head_dim, "n_ctx": llm.n_ctx()}
+    if not (n_layer and n_head_kv and key_len and val_len):
+        info["note"] = "insufficient metadata to size KV cache"
+        return info
+
+    bytes_per_tok = n_layer * n_head_kv * (key_len + val_len) * 2  # f16
+    used_tokens = int(getattr(llm, "n_tokens", 0) or 0)
+    info.update({
+        "kv_bytes_per_token": bytes_per_tok,
+        "kv_full_mb": bytes_per_tok * llm.n_ctx() / (1024 ** 2),
+        "kv_used_mb": bytes_per_tok * used_tokens / (1024 ** 2),
+        "used_tokens": used_tokens,
+    })
+    return info
+
+
+def benchmark(prompt="Write a Python function to check if a string is a palindrome, with a docstring.",
+              max_tokens=256):
+    """Run one generation on the current device and print a metrics table."""
+    import time
+    import psutil
+
+    llm = load_model()
+    proc = psutil.Process(os.getpid())
+
+    t0 = time.perf_counter()
+    t_first = None
+    text = ""
+    for piece in stream_chat(prompt, max_tokens=max_tokens):
+        if t_first is None:
+            t_first = time.perf_counter()
+        text += piece
+    t_end = time.perf_counter()
+    if t_first is None:                       # produced nothing
+        print("[bench] model produced no output"); return {}
+
+    gen_tokens = len(llm.tokenize(text.encode("utf-8"), add_bos=False))
+    used = int(getattr(llm, "n_tokens", 0) or 0)
+    prompt_tokens = max(used - gen_tokens, 0)
+
+    ttft = t_first - t0                       # includes prompt prefill
+    decode_time = max(t_end - t_first, 1e-9)
+    total_time = t_end - t0
+    decode_tps = (gen_tokens - 1) / decode_time if gen_tokens > 1 else 0.0
+    prefill_tps = prompt_tokens / ttft if (prompt_tokens and ttft > 0) else 0.0
+
+    rss_gb = proc.memory_info().rss / (1024 ** 3)
+    avail_gb = psutil.virtual_memory().available / (1024 ** 3)
+    kv = kv_cache_report(llm)
+
+    use_gpu = _resolve_use_gpu()
+    print("\n" + "=" * 70)
+    print(f"PERFORMANCE  —  device={'GPU' if use_gpu else f'CPU ({os.cpu_count()} threads)'}, "
+          f"model={GGUF_REPO} {GGUF_FILE}")
+    print("=" * 70)
+    print(f"  decode speed        : {decode_tps:6.2f} tok/s   <-- the headline TPS")
+    print(f"  prefill speed       : {prefill_tps:6.2f} tok/s   (prompt processing)")
+    print(f"  time to first token : {ttft:6.2f} s")
+    print(f"  generated tokens    : {gen_tokens}  in {decode_time:.2f}s")
+    print(f"  prompt tokens       : {prompt_tokens}")
+    print(f"  overall throughput  : {gen_tokens / total_time:6.2f} tok/s (incl. prefill)")
+    print("-" * 70)
+    print(f"  process RAM (RSS)   : {rss_gb:6.2f} GB")
+    print(f"  system RAM free     : {avail_gb:6.2f} GB")
+    print("-" * 70)
+    if "kv_full_mb" in kv:
+        pct = 100 * kv["used_tokens"] / kv["n_ctx"] if kv["n_ctx"] else 0
+        print(f"  context window      : {kv['used_tokens']} / {kv['n_ctx']} tokens ({pct:.1f}% used)")
+        print(f"  KV cache / token    : {kv['kv_bytes_per_token'] / 1024:6.2f} KB")
+        print(f"  KV cache (used)     : {kv['kv_used_mb']:6.2f} MB")
+        print(f"  KV cache (full ctx) : {kv['kv_full_mb']:6.2f} MB reserved for n_ctx={kv['n_ctx']}")
+        print(f"  attn geometry       : {kv['n_layer']} layers, "
+              f"{kv['n_head']} heads / {kv['n_head_kv']} KV heads (GQA), head_dim={kv['head_dim']}")
+    else:
+        print(f"  KV cache            : {kv.get('note', 'n/a')}")
+    print("=" * 70)
+
+    return {"decode_tps": decode_tps, "prefill_tps": prefill_tps, "ttft_s": ttft,
+            "gen_tokens": gen_tokens, "prompt_tokens": prompt_tokens,
+            "rss_gb": rss_gb, "kv": kv}
+
+
+# --------------------------------------------------------------------------- #
+# Demos
+# --------------------------------------------------------------------------- #
+def _demo_chat():
+    prompt = "Write a Python function that returns the nth Fibonacci number iteratively, with a docstring."
+    print("\n" + "=" * 70 + f"\nCHAT DEMO\nPROMPT: {prompt}\n" + "=" * 70)
+
+    mode, buf = "reasoning", ""
+    print("\n--- reasoning ---")
+    for piece in stream_chat(prompt):
+        buf += piece
+        if mode == "reasoning":
+            idx = buf.find(_THINK_CLOSE)
+            if idx != -1:                       # crossed into the answer
+                sys.stdout.write(buf[:idx])
+                print("\n\n--- answer ---")
+                sys.stdout.write(buf[idx + len(_THINK_CLOSE):])
+                mode, buf = "answer", ""
+            else:                               # hold back a tail so the tag can't split
+                keep = len(_THINK_CLOSE)
+                if len(buf) > keep:
+                    sys.stdout.write(buf[:-keep])
+                    buf = buf[-keep:]
+        else:
+            sys.stdout.write(piece)
+        sys.stdout.flush()
+    if buf:
+        sys.stdout.write(buf)
+    print("\n" + "=" * 70)
+
+
+def _demo_agent():
+    print("\n" + "=" * 70 + "\nAGENT DEMO (tool calling)\n" + "=" * 70)
+
+    def calculator(expression: str):
+        """Safely evaluate a basic arithmetic expression."""
+        if not re.fullmatch(r"[0-9+\-*/().%\s]+", expression or ""):
+            return "ERROR: only arithmetic allowed"
+        return eval(expression, {"__builtins__": {}}, {})  # sandboxed namespace
+
+    tools = [{
+        "name": "calculator",
+        "schema": {
+            "name": "calculator",
+            "description": "Evaluate a basic arithmetic expression and return the number.",
+            "parameters": {
+                "type": "object",
+                "properties": {"expression": {"type": "string",
+                               "description": "e.g. '(1234*7) + 89'"}},
+                "required": ["expression"],
+            },
+        },
+        "fn": calculator,
+    }]
+
+    result = run_agent(
+        "What is (1234 * 7) + 89, and then that result divided by 3? "
+        "Use the calculator tool for each arithmetic step.",
+        tools, max_steps=6,
+    )
+    print("\n--- FINAL ANSWER ---")
+    print(result["answer"])
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    benchmark()          # showcase device capability: TPS, RAM, KV cache
+    _demo_chat()
+    _demo_agent()
+    print("Import chat / generate / run_agent / benchmark from this file.")