Spaces:
Build error
Build error
Upload 10 files
Browse files- .gitattributes +1 -0
- README.md +85 -7
- app.py +86 -0
- guardrail.py +76 -0
- infer.py +81 -0
- network.db +3 -0
- prompt_template.py +96 -0
- requirements.txt +2 -0
- schema.sql +60 -0
- seed_from_csv.py +200 -0
- simulator.py +77 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
network.db filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,13 +1,91 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Network State Assistant
|
| 3 |
+
emoji: 🌐
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 4.44.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# Network State Assistant — demo
|
| 13 |
+
|
| 14 |
+
Text-to-SQL chatbot over a simulated network state database. Ask questions
|
| 15 |
+
in plain English about device status, traffic, alerts, and performance.
|
| 16 |
+
|
| 17 |
+
All files sit in one flat directory — no subfolders — so this uploads
|
| 18 |
+
cleanly to a Hugging Face Space via drag-and-drop with no folder handling.
|
| 19 |
+
|
| 20 |
+
## What's in here
|
| 21 |
+
|
| 22 |
+
```
|
| 23 |
+
schema.sql table DDL (devices, interfaces, traffic_flows, alerts, metrics_timeseries)
|
| 24 |
+
seed_from_csv.py seeds network.db — synthetic data by default, or a real NetFlow CSV
|
| 25 |
+
network.db the seeded SQLite database (commit this so data exists on boot)
|
| 26 |
+
prompt_template.py schema context + few-shot examples for the text-to-SQL prompt
|
| 27 |
+
infer.py llama-cpp-python wrapper around a small GGUF model (Qwen2.5-3B-Instruct)
|
| 28 |
+
guardrail.py validates generated SQL is a safe, whitelisted SELECT before it runs
|
| 29 |
+
simulator.py background thread that inserts new rows to fake a live feed
|
| 30 |
+
app.py Gradio chat app tying it all together
|
| 31 |
+
requirements.txt gradio, llama-cpp-python
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## Run locally
|
| 35 |
+
|
| 36 |
+
```bash
|
| 37 |
+
pip install -r requirements.txt
|
| 38 |
+
python seed_from_csv.py # creates network.db with synthetic data
|
| 39 |
+
python app.py
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
First run downloads the quantized model from the Hugging Face Hub (a few GB)
|
| 43 |
+
and caches it — expect the first question to be slow, subsequent ones fast.
|
| 44 |
+
|
| 45 |
+
To seed from a real NetFlow dataset instead of synthetic data (e.g.
|
| 46 |
+
NF-UNSW-NB15-v2 from https://staff.itee.uq.edu.au/marius/NIDS_datasets/):
|
| 47 |
+
|
| 48 |
+
```bash
|
| 49 |
+
python seed_from_csv.py --csv path/to/nf_unsw_nb15_v2.csv --limit 20000
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
## Deploy to Hugging Face Spaces
|
| 53 |
+
|
| 54 |
+
1. Create a new Space → SDK: **Gradio**, hardware: **CPU basic**.
|
| 55 |
+
2. On the Space's **Files** tab → **Add file → Upload files**, drag in every
|
| 56 |
+
file listed above — they're all flat, no folders to worry about.
|
| 57 |
+
3. Commit. The Space builds automatically from `app.py` + `requirements.txt`.
|
| 58 |
+
4. The model downloads from the Hub on first launch via
|
| 59 |
+
`Llama.from_pretrained(...)` in `infer.py` — no manual upload needed.
|
| 60 |
+
|
| 61 |
+
## Swapping the model
|
| 62 |
+
|
| 63 |
+
`infer.py` reads `MODEL_REPO` / `MODEL_FILE` from environment variables (set
|
| 64 |
+
them in the Space's settings), defaulting to:
|
| 65 |
+
|
| 66 |
+
- `Qwen/Qwen2.5-3B-Instruct-GGUF`, `qwen2.5-3b-instruct-q4_k_m.gguf`
|
| 67 |
+
|
| 68 |
+
If the 3B model feels slow on the free CPU tier, drop to
|
| 69 |
+
`Qwen/Qwen2.5-1.5B-Instruct-GGUF` — noticeably faster, somewhat weaker on
|
| 70 |
+
complex joins/aggregations.
|
| 71 |
+
|
| 72 |
+
## Safety notes (demo-grade, not production)
|
| 73 |
+
|
| 74 |
+
- The DB connection in `app.py` opens SQLite in read-only URI mode
|
| 75 |
+
(`mode=ro`) as a second line of defense behind the guardrail.
|
| 76 |
+
- `guardrail.py` rejects anything that isn't a single whitelisted-table
|
| 77 |
+
SELECT statement — no INSERT/UPDATE/DELETE/DROP/etc., no multiple
|
| 78 |
+
statements.
|
| 79 |
+
- This is enough for a live demo. Before any production use: move to
|
| 80 |
+
Postgres with a dedicated read-only role, add query timeouts, and add
|
| 81 |
+
logging/eval on generated SQL accuracy.
|
| 82 |
+
|
| 83 |
+
## Example questions
|
| 84 |
+
|
| 85 |
+
- "Which devices are currently down?"
|
| 86 |
+
- "What are the top 5 talkers by bandwidth in the last hour?"
|
| 87 |
+
- "Any unresolved critical alerts right now?"
|
| 88 |
+
- "Which interfaces have packet loss above 2% right now?"
|
| 89 |
+
- "How many devices are in a degraded state right now?"
|
| 90 |
+
- "List all alerts on the core switch in the last 24 hours"
|
| 91 |
+
- "Show total traffic volume by protocol today"
|
app.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Network text-to-SQL chat demo.
|
| 3 |
+
|
| 4 |
+
Run locally:
|
| 5 |
+
python db/seed_from_csv.py # one-time: create db/network.db
|
| 6 |
+
python app.py
|
| 7 |
+
|
| 8 |
+
On Hugging Face Spaces: set the Space SDK to Gradio, push this repo, and it
|
| 9 |
+
will build automatically (app.py is the default entrypoint).
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import sqlite3
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import gradio as gr
|
| 16 |
+
|
| 17 |
+
import simulator
|
| 18 |
+
from guardrail import validate_and_sanitize, SQLGuardrailError
|
| 19 |
+
from infer import generate_sql, synthesize_answer
|
| 20 |
+
from prompt_template import build_prompt
|
| 21 |
+
|
| 22 |
+
DB_PATH = Path(__file__).parent / "network.db"
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def get_connection():
|
| 26 |
+
# read-only connection: SQLite URI mode blocks writes at the driver level,
|
| 27 |
+
# a second line of defense behind the guardrail.
|
| 28 |
+
return sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, check_same_thread=False)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def answer_question(question: str, history):
|
| 32 |
+
if not DB_PATH.exists():
|
| 33 |
+
return "Database not found. Run `python seed_from_csv.py` first.", ""
|
| 34 |
+
|
| 35 |
+
prompt = build_prompt(question)
|
| 36 |
+
raw_sql = generate_sql(prompt)
|
| 37 |
+
|
| 38 |
+
try:
|
| 39 |
+
safe_sql = validate_and_sanitize(raw_sql)
|
| 40 |
+
except SQLGuardrailError as e:
|
| 41 |
+
return f"I couldn't safely run that query ({e}). Try rephrasing.", raw_sql
|
| 42 |
+
|
| 43 |
+
try:
|
| 44 |
+
conn = get_connection()
|
| 45 |
+
cur = conn.cursor()
|
| 46 |
+
cur.execute(safe_sql)
|
| 47 |
+
rows = cur.fetchall()
|
| 48 |
+
columns = [d[0] for d in cur.description]
|
| 49 |
+
conn.close()
|
| 50 |
+
except sqlite3.Error as e:
|
| 51 |
+
return f"The query failed to execute ({e}).", safe_sql
|
| 52 |
+
|
| 53 |
+
nl_answer = synthesize_answer(question, columns, rows)
|
| 54 |
+
return nl_answer, safe_sql
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def chat_fn(message, history):
|
| 58 |
+
answer, sql = answer_question(message, history)
|
| 59 |
+
shown = answer
|
| 60 |
+
if sql:
|
| 61 |
+
shown += f"\n\n<details><summary>Show SQL</summary>\n\n```sql\n{sql}\n```\n</details>"
|
| 62 |
+
return shown
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
EXAMPLE_QUESTIONS = [
|
| 66 |
+
"Which devices are currently down?",
|
| 67 |
+
"What are the top 5 talkers by bandwidth in the last hour?",
|
| 68 |
+
"Any unresolved critical alerts right now?",
|
| 69 |
+
"Which interfaces have packet loss above 2% right now?",
|
| 70 |
+
"How many devices are in a degraded state right now?",
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
with gr.Blocks(title="Network State Assistant") as demo:
|
| 74 |
+
gr.Markdown(
|
| 75 |
+
"# Network State Assistant (demo)\n"
|
| 76 |
+
"Ask about device status, traffic, alerts, and performance in plain English."
|
| 77 |
+
)
|
| 78 |
+
chat = gr.ChatInterface(
|
| 79 |
+
fn=chat_fn,
|
| 80 |
+
examples=EXAMPLE_QUESTIONS,
|
| 81 |
+
title=None,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
if __name__ == "__main__":
|
| 85 |
+
simulator.start(interval_seconds=5)
|
| 86 |
+
demo.launch()
|
guardrail.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Validates LLM-generated SQL before it touches the database.
|
| 3 |
+
|
| 4 |
+
Rules enforced:
|
| 5 |
+
- Exactly one statement, and it must be a SELECT.
|
| 6 |
+
- No DDL/DML keywords anywhere (belt and suspenders alongside the read-only
|
| 7 |
+
DB connection used in app.py).
|
| 8 |
+
- Every table referenced must be in the whitelist.
|
| 9 |
+
- A LIMIT is added if the query doesn't already have one and isn't an
|
| 10 |
+
aggregate-only query.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import re
|
| 14 |
+
|
| 15 |
+
ALLOWED_TABLES = {
|
| 16 |
+
"devices",
|
| 17 |
+
"interfaces",
|
| 18 |
+
"traffic_flows",
|
| 19 |
+
"alerts",
|
| 20 |
+
"metrics_timeseries",
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
FORBIDDEN_KEYWORDS = {
|
| 24 |
+
"insert", "update", "delete", "drop", "alter", "create",
|
| 25 |
+
"truncate", "replace", "attach", "detach", "pragma", "vacuum",
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
DEFAULT_LIMIT = 50
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class SQLGuardrailError(Exception):
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _extract_tables(sql: str) -> set:
|
| 36 |
+
# crude but effective for this schema: look for tokens after FROM/JOIN
|
| 37 |
+
tables = set()
|
| 38 |
+
for m in re.finditer(r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)", sql, re.IGNORECASE):
|
| 39 |
+
tables.add(m.group(1).lower())
|
| 40 |
+
return tables
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def validate_and_sanitize(sql: str) -> str:
|
| 44 |
+
"""Raises SQLGuardrailError on anything unsafe. Returns a safe SQL string."""
|
| 45 |
+
if sql is None:
|
| 46 |
+
raise SQLGuardrailError("Model returned no SQL.")
|
| 47 |
+
|
| 48 |
+
cleaned = sql.strip().strip("`").strip()
|
| 49 |
+
# strip a leading "sql" language tag if the model added a fenced block anyway
|
| 50 |
+
cleaned = re.sub(r"^sql\s*", "", cleaned, flags=re.IGNORECASE)
|
| 51 |
+
cleaned = cleaned.rstrip(";").strip()
|
| 52 |
+
|
| 53 |
+
if not cleaned:
|
| 54 |
+
raise SQLGuardrailError("Empty SQL after cleaning.")
|
| 55 |
+
|
| 56 |
+
if ";" in cleaned:
|
| 57 |
+
raise SQLGuardrailError("Multiple statements are not allowed.")
|
| 58 |
+
|
| 59 |
+
lowered = cleaned.lower()
|
| 60 |
+
if not lowered.startswith("select"):
|
| 61 |
+
raise SQLGuardrailError("Only SELECT statements are allowed.")
|
| 62 |
+
|
| 63 |
+
tokens = set(re.findall(r"[a-zA-Z_]+", lowered))
|
| 64 |
+
forbidden_hit = tokens & FORBIDDEN_KEYWORDS
|
| 65 |
+
if forbidden_hit:
|
| 66 |
+
raise SQLGuardrailError(f"Forbidden keyword(s) detected: {forbidden_hit}")
|
| 67 |
+
|
| 68 |
+
referenced = _extract_tables(cleaned)
|
| 69 |
+
unknown = referenced - ALLOWED_TABLES
|
| 70 |
+
if unknown:
|
| 71 |
+
raise SQLGuardrailError(f"Unknown table(s) referenced: {unknown}")
|
| 72 |
+
|
| 73 |
+
if "limit" not in lowered:
|
| 74 |
+
cleaned = f"{cleaned} LIMIT {DEFAULT_LIMIT}"
|
| 75 |
+
|
| 76 |
+
return cleaned + ";"
|
infer.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Thin wrapper around a small GGUF model run via llama-cpp-python for CPU inference.
|
| 3 |
+
|
| 4 |
+
Two calls are used:
|
| 5 |
+
- generate_sql(question) -> raw SQL string (unsafe until passed through guardrail.py)
|
| 6 |
+
- synthesize_answer(...) -> plain-language answer from query results
|
| 7 |
+
|
| 8 |
+
Model: Qwen2.5-3B-Instruct-GGUF (Q4_K_M). Swap MODEL_REPO/MODEL_FILE for the
|
| 9 |
+
1.5B build if the 3B feels slow on the Spaces free CPU tier.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import os
|
| 13 |
+
from llama_cpp import Llama
|
| 14 |
+
|
| 15 |
+
MODEL_REPO = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF")
|
| 16 |
+
MODEL_FILE = os.environ.get("MODEL_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf")
|
| 17 |
+
|
| 18 |
+
_llm = None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def get_llm():
|
| 22 |
+
global _llm
|
| 23 |
+
if _llm is None:
|
| 24 |
+
# from_pretrained downloads the GGUF from the Hub on first run and
|
| 25 |
+
# caches it locally (works out of the box on HF Spaces).
|
| 26 |
+
_llm = Llama.from_pretrained(
|
| 27 |
+
repo_id=MODEL_REPO,
|
| 28 |
+
filename=MODEL_FILE,
|
| 29 |
+
n_ctx=4096,
|
| 30 |
+
n_threads=os.cpu_count(),
|
| 31 |
+
verbose=False,
|
| 32 |
+
)
|
| 33 |
+
return _llm
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _chat(system_prompt: str, user_prompt: str, max_tokens: int = 300, temperature: float = 0.1) -> str:
|
| 37 |
+
llm = get_llm()
|
| 38 |
+
resp = llm.create_chat_completion(
|
| 39 |
+
messages=[
|
| 40 |
+
{"role": "system", "content": system_prompt},
|
| 41 |
+
{"role": "user", "content": user_prompt},
|
| 42 |
+
],
|
| 43 |
+
max_tokens=max_tokens,
|
| 44 |
+
temperature=temperature,
|
| 45 |
+
)
|
| 46 |
+
return resp["choices"][0]["message"]["content"].strip()
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def generate_sql(prompt: str) -> str:
|
| 50 |
+
"""`prompt` should already be the full schema+few-shot+question text
|
| 51 |
+
from llm/prompt_template.py:build_prompt(). Returns raw SQL text."""
|
| 52 |
+
raw = _chat(
|
| 53 |
+
system_prompt="You write SQLite SELECT queries. Output only the SQL, nothing else.",
|
| 54 |
+
user_prompt=prompt,
|
| 55 |
+
max_tokens=200,
|
| 56 |
+
temperature=0.0,
|
| 57 |
+
)
|
| 58 |
+
# models sometimes wrap output in ```sql fences despite instructions — strip them
|
| 59 |
+
raw = raw.replace("```sql", "").replace("```", "").strip()
|
| 60 |
+
return raw
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def synthesize_answer(question: str, columns: list, rows: list) -> str:
|
| 64 |
+
if not rows:
|
| 65 |
+
return "No matching results found."
|
| 66 |
+
preview = rows[:20]
|
| 67 |
+
table_text = " | ".join(columns) + "\n" + "\n".join(
|
| 68 |
+
" | ".join(str(v) for v in r) for r in preview
|
| 69 |
+
)
|
| 70 |
+
user_prompt = (
|
| 71 |
+
f"Question: {question}\n\n"
|
| 72 |
+
f"Query result ({len(rows)} row(s), showing up to 20):\n{table_text}\n\n"
|
| 73 |
+
"Answer the question in one or two plain sentences, referencing the "
|
| 74 |
+
"actual numbers/names from the result. Do not mention SQL."
|
| 75 |
+
)
|
| 76 |
+
return _chat(
|
| 77 |
+
system_prompt="You are a concise network operations assistant.",
|
| 78 |
+
user_prompt=user_prompt,
|
| 79 |
+
max_tokens=200,
|
| 80 |
+
temperature=0.2,
|
| 81 |
+
)
|
network.db
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:92c0d5332c2d241d6dfa0ad344887de13730c1071cb055db7aee85ea8066b359
|
| 3 |
+
size 770048
|
prompt_template.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Schema context + few-shot examples fed to the text-to-SQL LLM."""
|
| 2 |
+
|
| 3 |
+
SCHEMA_CONTEXT = """
|
| 4 |
+
You are a network operations assistant that writes SQLite SELECT queries
|
| 5 |
+
against the following schema. Only these tables and columns exist.
|
| 6 |
+
|
| 7 |
+
devices(device_id, hostname, ip_address, device_type, vendor, location, status, last_seen)
|
| 8 |
+
-- status is one of: up, down, degraded
|
| 9 |
+
|
| 10 |
+
interfaces(interface_id, device_id, name, status, speed_mbps)
|
| 11 |
+
-- status is one of: up, down
|
| 12 |
+
|
| 13 |
+
traffic_flows(flow_id, ts, src_ip, dst_ip, src_port, dst_port, protocol,
|
| 14 |
+
bytes, packets, duration_ms, device_id, label)
|
| 15 |
+
-- label is 'benign' or an attack type string
|
| 16 |
+
|
| 17 |
+
alerts(alert_id, ts, device_id, severity, alert_type, description, resolved)
|
| 18 |
+
-- severity is one of: info, warning, critical
|
| 19 |
+
-- resolved is 0 or 1
|
| 20 |
+
|
| 21 |
+
metrics_timeseries(metric_id, ts, device_id, interface_id, metric_name, value)
|
| 22 |
+
-- metric_name is one of: latency_ms, packet_loss_pct, bandwidth_util_pct
|
| 23 |
+
|
| 24 |
+
All timestamps (ts, last_seen) are ISO8601 strings, comparable with datetime().
|
| 25 |
+
'now' in SQLite gives the current UTC time.
|
| 26 |
+
|
| 27 |
+
Rules:
|
| 28 |
+
- Output ONLY a single SQLite SELECT statement. No prose, no markdown fences.
|
| 29 |
+
- Never write INSERT, UPDATE, DELETE, DROP, ALTER, or any statement other than SELECT.
|
| 30 |
+
- Always include a LIMIT (default 50) unless the question asks for a count/aggregate.
|
| 31 |
+
- Prefer JOINs to devices when a question asks about a device by name.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
FEW_SHOT_EXAMPLES = [
|
| 35 |
+
(
|
| 36 |
+
"Which devices are currently down?",
|
| 37 |
+
"SELECT hostname, ip_address, location FROM devices WHERE status = 'down' LIMIT 50;",
|
| 38 |
+
),
|
| 39 |
+
(
|
| 40 |
+
"Show me all interfaces on router-03 and their status",
|
| 41 |
+
"SELECT i.name, i.status, i.speed_mbps FROM interfaces i "
|
| 42 |
+
"JOIN devices d ON d.device_id = i.device_id "
|
| 43 |
+
"WHERE d.hostname = 'router-03' LIMIT 50;",
|
| 44 |
+
),
|
| 45 |
+
(
|
| 46 |
+
"What are the top 5 talkers by bandwidth in the last hour?",
|
| 47 |
+
"SELECT d.hostname, SUM(f.bytes) AS total_bytes FROM traffic_flows f "
|
| 48 |
+
"JOIN devices d ON d.device_id = f.device_id "
|
| 49 |
+
"WHERE f.ts >= datetime('now', '-1 hour') "
|
| 50 |
+
"GROUP BY d.hostname ORDER BY total_bytes DESC LIMIT 5;",
|
| 51 |
+
),
|
| 52 |
+
(
|
| 53 |
+
"Any unresolved critical alerts right now?",
|
| 54 |
+
"SELECT a.ts, d.hostname, a.alert_type, a.description FROM alerts a "
|
| 55 |
+
"JOIN devices d ON d.device_id = a.device_id "
|
| 56 |
+
"WHERE a.severity = 'critical' AND a.resolved = 0 "
|
| 57 |
+
"ORDER BY a.ts DESC LIMIT 50;",
|
| 58 |
+
),
|
| 59 |
+
(
|
| 60 |
+
"Which interfaces have packet loss above 2% right now?",
|
| 61 |
+
"SELECT d.hostname, i.name, m.value FROM metrics_timeseries m "
|
| 62 |
+
"JOIN interfaces i ON i.interface_id = m.interface_id "
|
| 63 |
+
"JOIN devices d ON d.device_id = m.device_id "
|
| 64 |
+
"WHERE m.metric_name = 'packet_loss_pct' AND m.value > 2 "
|
| 65 |
+
"AND m.ts >= datetime('now', '-15 minutes') "
|
| 66 |
+
"ORDER BY m.value DESC LIMIT 50;",
|
| 67 |
+
),
|
| 68 |
+
(
|
| 69 |
+
"How many devices are in a degraded state right now?",
|
| 70 |
+
"SELECT COUNT(*) AS degraded_count FROM devices WHERE status = 'degraded';",
|
| 71 |
+
),
|
| 72 |
+
(
|
| 73 |
+
"List all alerts on the core switch in the last 24 hours",
|
| 74 |
+
"SELECT a.ts, a.severity, a.alert_type, a.description FROM alerts a "
|
| 75 |
+
"JOIN devices d ON d.device_id = a.device_id "
|
| 76 |
+
"WHERE d.hostname LIKE '%switch%' AND a.ts >= datetime('now', '-1 day') "
|
| 77 |
+
"ORDER BY a.ts DESC LIMIT 50;",
|
| 78 |
+
),
|
| 79 |
+
(
|
| 80 |
+
"Show total traffic volume by protocol today",
|
| 81 |
+
"SELECT protocol, SUM(bytes) AS total_bytes FROM traffic_flows "
|
| 82 |
+
"WHERE ts >= datetime('now', 'start of day') "
|
| 83 |
+
"GROUP BY protocol ORDER BY total_bytes DESC LIMIT 50;",
|
| 84 |
+
),
|
| 85 |
+
]
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def build_prompt(user_question: str) -> str:
|
| 89 |
+
examples_text = "\n".join(
|
| 90 |
+
f"Q: {q}\nSQL: {sql}" for q, sql in FEW_SHOT_EXAMPLES
|
| 91 |
+
)
|
| 92 |
+
return (
|
| 93 |
+
f"{SCHEMA_CONTEXT}\n"
|
| 94 |
+
f"Examples:\n{examples_text}\n\n"
|
| 95 |
+
f"Q: {user_question}\nSQL:"
|
| 96 |
+
)
|
requirements.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=4.44
|
| 2 |
+
llama-cpp-python>=0.2.90
|
schema.sql
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- Network state schema (SQLite dialect, demo)
|
| 2 |
+
-- Migrates cleanly to Postgres later: SERIAL -> INTEGER PRIMARY KEY AUTOINCREMENT already
|
| 3 |
+
-- SQLite-compatible; swap to Postgres types when you move off the demo DB.
|
| 4 |
+
|
| 5 |
+
CREATE TABLE IF NOT EXISTS devices (
|
| 6 |
+
device_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 7 |
+
hostname TEXT,
|
| 8 |
+
ip_address TEXT UNIQUE,
|
| 9 |
+
device_type TEXT, -- router, switch, firewall, host
|
| 10 |
+
vendor TEXT,
|
| 11 |
+
location TEXT,
|
| 12 |
+
status TEXT, -- up, down, degraded
|
| 13 |
+
last_seen TEXT -- ISO8601 timestamp
|
| 14 |
+
);
|
| 15 |
+
|
| 16 |
+
CREATE TABLE IF NOT EXISTS interfaces (
|
| 17 |
+
interface_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 18 |
+
device_id INTEGER REFERENCES devices(device_id),
|
| 19 |
+
name TEXT, -- eth0, Gi0/1, etc.
|
| 20 |
+
status TEXT, -- up, down
|
| 21 |
+
speed_mbps INTEGER
|
| 22 |
+
);
|
| 23 |
+
|
| 24 |
+
CREATE TABLE IF NOT EXISTS traffic_flows (
|
| 25 |
+
flow_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 26 |
+
ts TEXT, -- ISO8601 timestamp
|
| 27 |
+
src_ip TEXT,
|
| 28 |
+
dst_ip TEXT,
|
| 29 |
+
src_port INTEGER,
|
| 30 |
+
dst_port INTEGER,
|
| 31 |
+
protocol TEXT,
|
| 32 |
+
bytes INTEGER,
|
| 33 |
+
packets INTEGER,
|
| 34 |
+
duration_ms INTEGER,
|
| 35 |
+
device_id INTEGER REFERENCES devices(device_id),
|
| 36 |
+
label TEXT -- benign / attack type
|
| 37 |
+
);
|
| 38 |
+
|
| 39 |
+
CREATE TABLE IF NOT EXISTS alerts (
|
| 40 |
+
alert_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 41 |
+
ts TEXT,
|
| 42 |
+
device_id INTEGER REFERENCES devices(device_id),
|
| 43 |
+
severity TEXT, -- info, warning, critical
|
| 44 |
+
alert_type TEXT, -- port_scan, dos, malware, link_down
|
| 45 |
+
description TEXT,
|
| 46 |
+
resolved INTEGER DEFAULT 0 -- 0/1 boolean
|
| 47 |
+
);
|
| 48 |
+
|
| 49 |
+
CREATE TABLE IF NOT EXISTS metrics_timeseries (
|
| 50 |
+
metric_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 51 |
+
ts TEXT,
|
| 52 |
+
device_id INTEGER REFERENCES devices(device_id),
|
| 53 |
+
interface_id INTEGER REFERENCES interfaces(interface_id),
|
| 54 |
+
metric_name TEXT, -- latency_ms, packet_loss_pct, bandwidth_util_pct
|
| 55 |
+
value REAL
|
| 56 |
+
);
|
| 57 |
+
|
| 58 |
+
CREATE INDEX IF NOT EXISTS idx_flows_device_ts ON traffic_flows(device_id, ts);
|
| 59 |
+
CREATE INDEX IF NOT EXISTS idx_alerts_device_ts ON alerts(device_id, ts);
|
| 60 |
+
CREATE INDEX IF NOT EXISTS idx_metrics_device_ts ON metrics_timeseries(device_id, ts);
|
seed_from_csv.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Seeds network.db with device/interface/flow/alert/metric data.
|
| 3 |
+
|
| 4 |
+
Two modes:
|
| 5 |
+
1. Real data: point --csv at a NetFlow-style CSV (e.g. NF-UNSW-NB15-v2 from
|
| 6 |
+
https://staff.itee.uq.edu.au/marius/NIDS_datasets/) with columns similar to
|
| 7 |
+
IPV4_SRC_ADDR, IPV4_DST_ADDR, L4_SRC_PORT, L4_DST_PORT, PROTOCOL, IN_BYTES,
|
| 8 |
+
IN_PKTS, FLOW_DURATION_MILLISECONDS, Label, Attack. Devices/interfaces/
|
| 9 |
+
alerts are derived from the distinct IPs in the flow data.
|
| 10 |
+
2. Synthetic fallback (default, no --csv given): generates a realistic set of
|
| 11 |
+
devices, interfaces, flows, alerts, and metrics from scratch so the demo
|
| 12 |
+
works with zero external downloads.
|
| 13 |
+
|
| 14 |
+
Usage:
|
| 15 |
+
python seed_from_csv.py # synthetic data
|
| 16 |
+
python seed_from_csv.py --csv path/to/nf_unsw_nb15_v2.csv --limit 20000
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import argparse
|
| 20 |
+
import ipaddress
|
| 21 |
+
import random
|
| 22 |
+
import sqlite3
|
| 23 |
+
from datetime import datetime, timedelta
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
DB_PATH = Path(__file__).parent / "network.db"
|
| 27 |
+
SCHEMA_PATH = Path(__file__).parent / "schema.sql"
|
| 28 |
+
|
| 29 |
+
DEVICE_TYPES = ["router", "switch", "firewall", "host", "access_point"]
|
| 30 |
+
VENDORS = ["Cisco", "Juniper", "Arista", "MikroTik", "Ubiquiti", "Fortinet"]
|
| 31 |
+
LOCATIONS = ["HQ-Floor1", "HQ-Floor2", "DataCenter-A", "Branch-Lagos", "Branch-Abuja", "Cloud-Edge"]
|
| 32 |
+
PROTOCOLS = ["TCP", "UDP", "ICMP"]
|
| 33 |
+
ALERT_TYPES = ["port_scan", "dos", "malware", "link_down", "auth_failure", "config_drift"]
|
| 34 |
+
SEVERITIES = ["info", "warning", "critical"]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def reset_db():
|
| 38 |
+
if DB_PATH.exists():
|
| 39 |
+
DB_PATH.unlink()
|
| 40 |
+
conn = sqlite3.connect(DB_PATH)
|
| 41 |
+
conn.executescript(SCHEMA_PATH.read_text())
|
| 42 |
+
conn.commit()
|
| 43 |
+
return conn
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def now_minus(seconds):
|
| 47 |
+
return (datetime.utcnow() - timedelta(seconds=seconds)).isoformat(timespec="seconds")
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def seed_synthetic(conn, n_devices=18, n_flows=4000, n_alerts=40, n_metrics=3000):
|
| 51 |
+
cur = conn.cursor()
|
| 52 |
+
|
| 53 |
+
device_ids = []
|
| 54 |
+
for i in range(n_devices):
|
| 55 |
+
dtype = random.choice(DEVICE_TYPES)
|
| 56 |
+
status = random.choices(["up", "degraded", "down"], weights=[0.82, 0.13, 0.05])[0]
|
| 57 |
+
cur.execute(
|
| 58 |
+
"INSERT INTO devices (hostname, ip_address, device_type, vendor, location, status, last_seen) "
|
| 59 |
+
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
| 60 |
+
(
|
| 61 |
+
f"{dtype}-{i+1:02d}",
|
| 62 |
+
str(ipaddress.IPv4Address(0x0A000000 + i + 1)), # 10.0.0.x
|
| 63 |
+
dtype,
|
| 64 |
+
random.choice(VENDORS),
|
| 65 |
+
random.choice(LOCATIONS),
|
| 66 |
+
status,
|
| 67 |
+
now_minus(random.randint(0, 600)),
|
| 68 |
+
),
|
| 69 |
+
)
|
| 70 |
+
device_ids.append(cur.lastrowid)
|
| 71 |
+
|
| 72 |
+
interface_ids = []
|
| 73 |
+
for did in device_ids:
|
| 74 |
+
for j in range(random.randint(1, 4)):
|
| 75 |
+
cur.execute(
|
| 76 |
+
"INSERT INTO interfaces (device_id, name, status, speed_mbps) VALUES (?, ?, ?, ?)",
|
| 77 |
+
(
|
| 78 |
+
did,
|
| 79 |
+
f"Gi0/{j}",
|
| 80 |
+
random.choices(["up", "down"], weights=[0.9, 0.1])[0],
|
| 81 |
+
random.choice([100, 1000, 10000]),
|
| 82 |
+
),
|
| 83 |
+
)
|
| 84 |
+
interface_ids.append((cur.lastrowid, did))
|
| 85 |
+
|
| 86 |
+
for _ in range(n_flows):
|
| 87 |
+
did = random.choice(device_ids)
|
| 88 |
+
is_attack = random.random() < 0.06
|
| 89 |
+
cur.execute(
|
| 90 |
+
"INSERT INTO traffic_flows (ts, src_ip, dst_ip, src_port, dst_port, protocol, "
|
| 91 |
+
"bytes, packets, duration_ms, device_id, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
| 92 |
+
(
|
| 93 |
+
now_minus(random.randint(0, 86400)),
|
| 94 |
+
f"10.0.0.{random.randint(1, 254)}",
|
| 95 |
+
f"{random.choice(['172.16', '192.168', '203.0'])}.{random.randint(0,255)}.{random.randint(1,254)}",
|
| 96 |
+
random.randint(1024, 65535),
|
| 97 |
+
random.choice([80, 443, 22, 53, 3389, 8080, 3306]),
|
| 98 |
+
random.choice(PROTOCOLS),
|
| 99 |
+
random.randint(200, 5_000_000),
|
| 100 |
+
random.randint(1, 4000),
|
| 101 |
+
random.randint(1, 60000),
|
| 102 |
+
did,
|
| 103 |
+
random.choice(ALERT_TYPES) if is_attack else "benign",
|
| 104 |
+
),
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
for _ in range(n_alerts):
|
| 108 |
+
did = random.choice(device_ids)
|
| 109 |
+
cur.execute(
|
| 110 |
+
"INSERT INTO alerts (ts, device_id, severity, alert_type, description, resolved) "
|
| 111 |
+
"VALUES (?, ?, ?, ?, ?, ?)",
|
| 112 |
+
(
|
| 113 |
+
now_minus(random.randint(0, 172800)),
|
| 114 |
+
did,
|
| 115 |
+
random.choices(SEVERITIES, weights=[0.5, 0.35, 0.15])[0],
|
| 116 |
+
random.choice(ALERT_TYPES),
|
| 117 |
+
"Auto-generated demo alert",
|
| 118 |
+
random.choices([0, 1], weights=[0.4, 0.6])[0],
|
| 119 |
+
),
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
for _ in range(n_metrics):
|
| 123 |
+
iface_id, did = random.choice(interface_ids)
|
| 124 |
+
metric_name = random.choice(["latency_ms", "packet_loss_pct", "bandwidth_util_pct"])
|
| 125 |
+
value = {
|
| 126 |
+
"latency_ms": round(random.uniform(1, 180), 2),
|
| 127 |
+
"packet_loss_pct": round(random.uniform(0, 6), 2),
|
| 128 |
+
"bandwidth_util_pct": round(random.uniform(2, 98), 2),
|
| 129 |
+
}[metric_name]
|
| 130 |
+
cur.execute(
|
| 131 |
+
"INSERT INTO metrics_timeseries (ts, device_id, interface_id, metric_name, value) "
|
| 132 |
+
"VALUES (?, ?, ?, ?, ?)",
|
| 133 |
+
(now_minus(random.randint(0, 3600)), did, iface_id, metric_name, value),
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
conn.commit()
|
| 137 |
+
print(f"Synthetic seed complete: {len(device_ids)} devices, {n_flows} flows, "
|
| 138 |
+
f"{n_alerts} alerts, {n_metrics} metrics.")
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def seed_from_real_csv(conn, csv_path, limit):
|
| 142 |
+
import csv as csv_mod
|
| 143 |
+
|
| 144 |
+
cur = conn.cursor()
|
| 145 |
+
ip_to_device = {}
|
| 146 |
+
|
| 147 |
+
def get_or_create_device(ip):
|
| 148 |
+
if ip in ip_to_device:
|
| 149 |
+
return ip_to_device[ip]
|
| 150 |
+
cur.execute(
|
| 151 |
+
"INSERT INTO devices (hostname, ip_address, device_type, vendor, location, status, last_seen) "
|
| 152 |
+
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
| 153 |
+
(f"host-{ip}", ip, "host", random.choice(VENDORS), random.choice(LOCATIONS), "up", now_minus(0)),
|
| 154 |
+
)
|
| 155 |
+
ip_to_device[ip] = cur.lastrowid
|
| 156 |
+
return cur.lastrowid
|
| 157 |
+
|
| 158 |
+
with open(csv_path, newline="") as f:
|
| 159 |
+
reader = csv_mod.DictReader(f)
|
| 160 |
+
for i, row in enumerate(reader):
|
| 161 |
+
if i >= limit:
|
| 162 |
+
break
|
| 163 |
+
src_ip = row.get("IPV4_SRC_ADDR") or row.get("src_ip") or "0.0.0.0"
|
| 164 |
+
dst_ip = row.get("IPV4_DST_ADDR") or row.get("dst_ip") or "0.0.0.0"
|
| 165 |
+
device_id = get_or_create_device(src_ip)
|
| 166 |
+
label = row.get("Attack") or row.get("label") or ("benign" if row.get("Label", "0") == "0" else "attack")
|
| 167 |
+
cur.execute(
|
| 168 |
+
"INSERT INTO traffic_flows (ts, src_ip, dst_ip, src_port, dst_port, protocol, "
|
| 169 |
+
"bytes, packets, duration_ms, device_id, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
| 170 |
+
(
|
| 171 |
+
now_minus(random.randint(0, 86400)),
|
| 172 |
+
src_ip,
|
| 173 |
+
dst_ip,
|
| 174 |
+
int(row.get("L4_SRC_PORT", 0) or 0),
|
| 175 |
+
int(row.get("L4_DST_PORT", 0) or 0),
|
| 176 |
+
row.get("PROTOCOL", "TCP"),
|
| 177 |
+
int(row.get("IN_BYTES", 0) or 0),
|
| 178 |
+
int(row.get("IN_PKTS", 0) or 0),
|
| 179 |
+
int(float(row.get("FLOW_DURATION_MILLISECONDS", 0) or 0)),
|
| 180 |
+
device_id,
|
| 181 |
+
label,
|
| 182 |
+
),
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
conn.commit()
|
| 186 |
+
print(f"Loaded {i+1} flows from {csv_path}, derived {len(ip_to_device)} devices.")
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
if __name__ == "__main__":
|
| 190 |
+
parser = argparse.ArgumentParser()
|
| 191 |
+
parser.add_argument("--csv", type=str, default=None, help="Path to a real NetFlow CSV")
|
| 192 |
+
parser.add_argument("--limit", type=int, default=20000, help="Max rows to load from CSV")
|
| 193 |
+
args = parser.parse_args()
|
| 194 |
+
|
| 195 |
+
conn = reset_db()
|
| 196 |
+
if args.csv:
|
| 197 |
+
seed_from_real_csv(conn, args.csv, args.limit)
|
| 198 |
+
else:
|
| 199 |
+
seed_synthetic(conn)
|
| 200 |
+
conn.close()
|
simulator.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Background thread that periodically inserts new flow/metric rows so the
|
| 2 |
+
demo DB feels like it's reflecting a live network, without needing a real
|
| 3 |
+
ingestion pipeline."""
|
| 4 |
+
|
| 5 |
+
import random
|
| 6 |
+
import sqlite3
|
| 7 |
+
import threading
|
| 8 |
+
import time
|
| 9 |
+
from datetime import datetime
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
|
| 12 |
+
DB_PATH = Path(__file__).parent / "network.db"
|
| 13 |
+
PROTOCOLS = ["TCP", "UDP", "ICMP"]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _tick(conn):
|
| 17 |
+
cur = conn.cursor()
|
| 18 |
+
cur.execute("SELECT device_id FROM devices")
|
| 19 |
+
device_ids = [r[0] for r in cur.fetchall()]
|
| 20 |
+
if not device_ids:
|
| 21 |
+
return
|
| 22 |
+
|
| 23 |
+
now = datetime.utcnow().isoformat(timespec="seconds")
|
| 24 |
+
did = random.choice(device_ids)
|
| 25 |
+
|
| 26 |
+
cur.execute(
|
| 27 |
+
"INSERT INTO traffic_flows (ts, src_ip, dst_ip, src_port, dst_port, protocol, "
|
| 28 |
+
"bytes, packets, duration_ms, device_id, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
| 29 |
+
(
|
| 30 |
+
now,
|
| 31 |
+
f"10.0.0.{random.randint(1, 254)}",
|
| 32 |
+
f"192.168.{random.randint(0,255)}.{random.randint(1,254)}",
|
| 33 |
+
random.randint(1024, 65535),
|
| 34 |
+
random.choice([80, 443, 22, 53]),
|
| 35 |
+
random.choice(PROTOCOLS),
|
| 36 |
+
random.randint(200, 2_000_000),
|
| 37 |
+
random.randint(1, 3000),
|
| 38 |
+
random.randint(1, 5000),
|
| 39 |
+
did,
|
| 40 |
+
"benign",
|
| 41 |
+
),
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
cur.execute("SELECT interface_id, device_id FROM interfaces WHERE device_id = ?", (did,))
|
| 45 |
+
ifaces = cur.fetchall()
|
| 46 |
+
if ifaces:
|
| 47 |
+
iface_id, iface_device_id = random.choice(ifaces)
|
| 48 |
+
metric_name = random.choice(["latency_ms", "packet_loss_pct", "bandwidth_util_pct"])
|
| 49 |
+
value = {
|
| 50 |
+
"latency_ms": round(random.uniform(1, 180), 2),
|
| 51 |
+
"packet_loss_pct": round(random.uniform(0, 6), 2),
|
| 52 |
+
"bandwidth_util_pct": round(random.uniform(2, 98), 2),
|
| 53 |
+
}[metric_name]
|
| 54 |
+
cur.execute(
|
| 55 |
+
"INSERT INTO metrics_timeseries (ts, device_id, interface_id, metric_name, value) "
|
| 56 |
+
"VALUES (?, ?, ?, ?, ?)",
|
| 57 |
+
(now, iface_device_id, iface_id, metric_name, value),
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
conn.commit()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _loop(interval_seconds):
|
| 64 |
+
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
|
| 65 |
+
while True:
|
| 66 |
+
try:
|
| 67 |
+
_tick(conn)
|
| 68 |
+
except Exception as e:
|
| 69 |
+
print(f"[simulator] tick failed: {e}")
|
| 70 |
+
time.sleep(interval_seconds)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def start(interval_seconds: int = 5):
|
| 74 |
+
"""Starts the simulator in a daemon thread. Call once at app startup."""
|
| 75 |
+
t = threading.Thread(target=_loop, args=(interval_seconds,), daemon=True)
|
| 76 |
+
t.start()
|
| 77 |
+
return t
|