Spaces:
Build error
Build error
File size: 2,756 Bytes
464c149 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | """
Thin wrapper around a small GGUF model run via llama-cpp-python for CPU inference.
Two calls are used:
- generate_sql(question) -> raw SQL string (unsafe until passed through guardrail.py)
- synthesize_answer(...) -> plain-language answer from query results
Model: Qwen2.5-3B-Instruct-GGUF (Q4_K_M). Swap MODEL_REPO/MODEL_FILE for the
1.5B build if the 3B feels slow on the Spaces free CPU tier.
"""
import os
from llama_cpp import Llama
MODEL_REPO = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF")
MODEL_FILE = os.environ.get("MODEL_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf")
_llm = None
def get_llm():
global _llm
if _llm is None:
# from_pretrained downloads the GGUF from the Hub on first run and
# caches it locally (works out of the box on HF Spaces).
_llm = Llama.from_pretrained(
repo_id=MODEL_REPO,
filename=MODEL_FILE,
n_ctx=4096,
n_threads=os.cpu_count(),
verbose=False,
)
return _llm
def _chat(system_prompt: str, user_prompt: str, max_tokens: int = 300, temperature: float = 0.1) -> str:
llm = get_llm()
resp = llm.create_chat_completion(
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
max_tokens=max_tokens,
temperature=temperature,
)
return resp["choices"][0]["message"]["content"].strip()
def generate_sql(prompt: str) -> str:
"""`prompt` should already be the full schema+few-shot+question text
from llm/prompt_template.py:build_prompt(). Returns raw SQL text."""
raw = _chat(
system_prompt="You write SQLite SELECT queries. Output only the SQL, nothing else.",
user_prompt=prompt,
max_tokens=200,
temperature=0.0,
)
# models sometimes wrap output in ```sql fences despite instructions — strip them
raw = raw.replace("```sql", "").replace("```", "").strip()
return raw
def synthesize_answer(question: str, columns: list, rows: list) -> str:
if not rows:
return "No matching results found."
preview = rows[:20]
table_text = " | ".join(columns) + "\n" + "\n".join(
" | ".join(str(v) for v in r) for r in preview
)
user_prompt = (
f"Question: {question}\n\n"
f"Query result ({len(rows)} row(s), showing up to 20):\n{table_text}\n\n"
"Answer the question in one or two plain sentences, referencing the "
"actual numbers/names from the result. Do not mention SQL."
)
return _chat(
system_prompt="You are a concise network operations assistant.",
user_prompt=user_prompt,
max_tokens=200,
temperature=0.2,
)
|