EYEDOL commited on
Commit
464c149
·
verified ·
1 Parent(s): ffeada1

Upload 10 files

Browse files
Files changed (11) hide show
  1. .gitattributes +1 -0
  2. README.md +85 -7
  3. app.py +86 -0
  4. guardrail.py +76 -0
  5. infer.py +81 -0
  6. network.db +3 -0
  7. prompt_template.py +96 -0
  8. requirements.txt +2 -0
  9. schema.sql +60 -0
  10. seed_from_csv.py +200 -0
  11. simulator.py +77 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ network.db filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,13 +1,91 @@
1
  ---
2
- title: Asknetworkstate
3
- emoji: 👀
4
- colorFrom: green
5
- colorTo: blue
6
  sdk: gradio
7
- sdk_version: 6.26.0
8
- python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Network State Assistant
3
+ emoji: 🌐
4
+ colorFrom: blue
5
+ colorTo: green
6
  sdk: gradio
7
+ sdk_version: 4.44.0
 
8
  app_file: app.py
9
  pinned: false
10
  ---
11
 
12
+ # Network State Assistant — demo
13
+
14
+ Text-to-SQL chatbot over a simulated network state database. Ask questions
15
+ in plain English about device status, traffic, alerts, and performance.
16
+
17
+ All files sit in one flat directory — no subfolders — so this uploads
18
+ cleanly to a Hugging Face Space via drag-and-drop with no folder handling.
19
+
20
+ ## What's in here
21
+
22
+ ```
23
+ schema.sql table DDL (devices, interfaces, traffic_flows, alerts, metrics_timeseries)
24
+ seed_from_csv.py seeds network.db — synthetic data by default, or a real NetFlow CSV
25
+ network.db the seeded SQLite database (commit this so data exists on boot)
26
+ prompt_template.py schema context + few-shot examples for the text-to-SQL prompt
27
+ infer.py llama-cpp-python wrapper around a small GGUF model (Qwen2.5-3B-Instruct)
28
+ guardrail.py validates generated SQL is a safe, whitelisted SELECT before it runs
29
+ simulator.py background thread that inserts new rows to fake a live feed
30
+ app.py Gradio chat app tying it all together
31
+ requirements.txt gradio, llama-cpp-python
32
+ ```
33
+
34
+ ## Run locally
35
+
36
+ ```bash
37
+ pip install -r requirements.txt
38
+ python seed_from_csv.py # creates network.db with synthetic data
39
+ python app.py
40
+ ```
41
+
42
+ First run downloads the quantized model from the Hugging Face Hub (a few GB)
43
+ and caches it — expect the first question to be slow, subsequent ones fast.
44
+
45
+ To seed from a real NetFlow dataset instead of synthetic data (e.g.
46
+ NF-UNSW-NB15-v2 from https://staff.itee.uq.edu.au/marius/NIDS_datasets/):
47
+
48
+ ```bash
49
+ python seed_from_csv.py --csv path/to/nf_unsw_nb15_v2.csv --limit 20000
50
+ ```
51
+
52
+ ## Deploy to Hugging Face Spaces
53
+
54
+ 1. Create a new Space → SDK: **Gradio**, hardware: **CPU basic**.
55
+ 2. On the Space's **Files** tab → **Add file → Upload files**, drag in every
56
+ file listed above — they're all flat, no folders to worry about.
57
+ 3. Commit. The Space builds automatically from `app.py` + `requirements.txt`.
58
+ 4. The model downloads from the Hub on first launch via
59
+ `Llama.from_pretrained(...)` in `infer.py` — no manual upload needed.
60
+
61
+ ## Swapping the model
62
+
63
+ `infer.py` reads `MODEL_REPO` / `MODEL_FILE` from environment variables (set
64
+ them in the Space's settings), defaulting to:
65
+
66
+ - `Qwen/Qwen2.5-3B-Instruct-GGUF`, `qwen2.5-3b-instruct-q4_k_m.gguf`
67
+
68
+ If the 3B model feels slow on the free CPU tier, drop to
69
+ `Qwen/Qwen2.5-1.5B-Instruct-GGUF` — noticeably faster, somewhat weaker on
70
+ complex joins/aggregations.
71
+
72
+ ## Safety notes (demo-grade, not production)
73
+
74
+ - The DB connection in `app.py` opens SQLite in read-only URI mode
75
+ (`mode=ro`) as a second line of defense behind the guardrail.
76
+ - `guardrail.py` rejects anything that isn't a single whitelisted-table
77
+ SELECT statement — no INSERT/UPDATE/DELETE/DROP/etc., no multiple
78
+ statements.
79
+ - This is enough for a live demo. Before any production use: move to
80
+ Postgres with a dedicated read-only role, add query timeouts, and add
81
+ logging/eval on generated SQL accuracy.
82
+
83
+ ## Example questions
84
+
85
+ - "Which devices are currently down?"
86
+ - "What are the top 5 talkers by bandwidth in the last hour?"
87
+ - "Any unresolved critical alerts right now?"
88
+ - "Which interfaces have packet loss above 2% right now?"
89
+ - "How many devices are in a degraded state right now?"
90
+ - "List all alerts on the core switch in the last 24 hours"
91
+ - "Show total traffic volume by protocol today"
app.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Network text-to-SQL chat demo.
3
+
4
+ Run locally:
5
+ python db/seed_from_csv.py # one-time: create db/network.db
6
+ python app.py
7
+
8
+ On Hugging Face Spaces: set the Space SDK to Gradio, push this repo, and it
9
+ will build automatically (app.py is the default entrypoint).
10
+ """
11
+
12
+ import sqlite3
13
+ from pathlib import Path
14
+
15
+ import gradio as gr
16
+
17
+ import simulator
18
+ from guardrail import validate_and_sanitize, SQLGuardrailError
19
+ from infer import generate_sql, synthesize_answer
20
+ from prompt_template import build_prompt
21
+
22
+ DB_PATH = Path(__file__).parent / "network.db"
23
+
24
+
25
+ def get_connection():
26
+ # read-only connection: SQLite URI mode blocks writes at the driver level,
27
+ # a second line of defense behind the guardrail.
28
+ return sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, check_same_thread=False)
29
+
30
+
31
+ def answer_question(question: str, history):
32
+ if not DB_PATH.exists():
33
+ return "Database not found. Run `python seed_from_csv.py` first.", ""
34
+
35
+ prompt = build_prompt(question)
36
+ raw_sql = generate_sql(prompt)
37
+
38
+ try:
39
+ safe_sql = validate_and_sanitize(raw_sql)
40
+ except SQLGuardrailError as e:
41
+ return f"I couldn't safely run that query ({e}). Try rephrasing.", raw_sql
42
+
43
+ try:
44
+ conn = get_connection()
45
+ cur = conn.cursor()
46
+ cur.execute(safe_sql)
47
+ rows = cur.fetchall()
48
+ columns = [d[0] for d in cur.description]
49
+ conn.close()
50
+ except sqlite3.Error as e:
51
+ return f"The query failed to execute ({e}).", safe_sql
52
+
53
+ nl_answer = synthesize_answer(question, columns, rows)
54
+ return nl_answer, safe_sql
55
+
56
+
57
+ def chat_fn(message, history):
58
+ answer, sql = answer_question(message, history)
59
+ shown = answer
60
+ if sql:
61
+ shown += f"\n\n<details><summary>Show SQL</summary>\n\n```sql\n{sql}\n```\n</details>"
62
+ return shown
63
+
64
+
65
+ EXAMPLE_QUESTIONS = [
66
+ "Which devices are currently down?",
67
+ "What are the top 5 talkers by bandwidth in the last hour?",
68
+ "Any unresolved critical alerts right now?",
69
+ "Which interfaces have packet loss above 2% right now?",
70
+ "How many devices are in a degraded state right now?",
71
+ ]
72
+
73
+ with gr.Blocks(title="Network State Assistant") as demo:
74
+ gr.Markdown(
75
+ "# Network State Assistant (demo)\n"
76
+ "Ask about device status, traffic, alerts, and performance in plain English."
77
+ )
78
+ chat = gr.ChatInterface(
79
+ fn=chat_fn,
80
+ examples=EXAMPLE_QUESTIONS,
81
+ title=None,
82
+ )
83
+
84
+ if __name__ == "__main__":
85
+ simulator.start(interval_seconds=5)
86
+ demo.launch()
guardrail.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Validates LLM-generated SQL before it touches the database.
3
+
4
+ Rules enforced:
5
+ - Exactly one statement, and it must be a SELECT.
6
+ - No DDL/DML keywords anywhere (belt and suspenders alongside the read-only
7
+ DB connection used in app.py).
8
+ - Every table referenced must be in the whitelist.
9
+ - A LIMIT is added if the query doesn't already have one and isn't an
10
+ aggregate-only query.
11
+ """
12
+
13
+ import re
14
+
15
+ ALLOWED_TABLES = {
16
+ "devices",
17
+ "interfaces",
18
+ "traffic_flows",
19
+ "alerts",
20
+ "metrics_timeseries",
21
+ }
22
+
23
+ FORBIDDEN_KEYWORDS = {
24
+ "insert", "update", "delete", "drop", "alter", "create",
25
+ "truncate", "replace", "attach", "detach", "pragma", "vacuum",
26
+ }
27
+
28
+ DEFAULT_LIMIT = 50
29
+
30
+
31
+ class SQLGuardrailError(Exception):
32
+ pass
33
+
34
+
35
+ def _extract_tables(sql: str) -> set:
36
+ # crude but effective for this schema: look for tokens after FROM/JOIN
37
+ tables = set()
38
+ for m in re.finditer(r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)", sql, re.IGNORECASE):
39
+ tables.add(m.group(1).lower())
40
+ return tables
41
+
42
+
43
+ def validate_and_sanitize(sql: str) -> str:
44
+ """Raises SQLGuardrailError on anything unsafe. Returns a safe SQL string."""
45
+ if sql is None:
46
+ raise SQLGuardrailError("Model returned no SQL.")
47
+
48
+ cleaned = sql.strip().strip("`").strip()
49
+ # strip a leading "sql" language tag if the model added a fenced block anyway
50
+ cleaned = re.sub(r"^sql\s*", "", cleaned, flags=re.IGNORECASE)
51
+ cleaned = cleaned.rstrip(";").strip()
52
+
53
+ if not cleaned:
54
+ raise SQLGuardrailError("Empty SQL after cleaning.")
55
+
56
+ if ";" in cleaned:
57
+ raise SQLGuardrailError("Multiple statements are not allowed.")
58
+
59
+ lowered = cleaned.lower()
60
+ if not lowered.startswith("select"):
61
+ raise SQLGuardrailError("Only SELECT statements are allowed.")
62
+
63
+ tokens = set(re.findall(r"[a-zA-Z_]+", lowered))
64
+ forbidden_hit = tokens & FORBIDDEN_KEYWORDS
65
+ if forbidden_hit:
66
+ raise SQLGuardrailError(f"Forbidden keyword(s) detected: {forbidden_hit}")
67
+
68
+ referenced = _extract_tables(cleaned)
69
+ unknown = referenced - ALLOWED_TABLES
70
+ if unknown:
71
+ raise SQLGuardrailError(f"Unknown table(s) referenced: {unknown}")
72
+
73
+ if "limit" not in lowered:
74
+ cleaned = f"{cleaned} LIMIT {DEFAULT_LIMIT}"
75
+
76
+ return cleaned + ";"
infer.py ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Thin wrapper around a small GGUF model run via llama-cpp-python for CPU inference.
3
+
4
+ Two calls are used:
5
+ - generate_sql(question) -> raw SQL string (unsafe until passed through guardrail.py)
6
+ - synthesize_answer(...) -> plain-language answer from query results
7
+
8
+ Model: Qwen2.5-3B-Instruct-GGUF (Q4_K_M). Swap MODEL_REPO/MODEL_FILE for the
9
+ 1.5B build if the 3B feels slow on the Spaces free CPU tier.
10
+ """
11
+
12
+ import os
13
+ from llama_cpp import Llama
14
+
15
+ MODEL_REPO = os.environ.get("MODEL_REPO", "Qwen/Qwen2.5-3B-Instruct-GGUF")
16
+ MODEL_FILE = os.environ.get("MODEL_FILE", "qwen2.5-3b-instruct-q4_k_m.gguf")
17
+
18
+ _llm = None
19
+
20
+
21
+ def get_llm():
22
+ global _llm
23
+ if _llm is None:
24
+ # from_pretrained downloads the GGUF from the Hub on first run and
25
+ # caches it locally (works out of the box on HF Spaces).
26
+ _llm = Llama.from_pretrained(
27
+ repo_id=MODEL_REPO,
28
+ filename=MODEL_FILE,
29
+ n_ctx=4096,
30
+ n_threads=os.cpu_count(),
31
+ verbose=False,
32
+ )
33
+ return _llm
34
+
35
+
36
+ def _chat(system_prompt: str, user_prompt: str, max_tokens: int = 300, temperature: float = 0.1) -> str:
37
+ llm = get_llm()
38
+ resp = llm.create_chat_completion(
39
+ messages=[
40
+ {"role": "system", "content": system_prompt},
41
+ {"role": "user", "content": user_prompt},
42
+ ],
43
+ max_tokens=max_tokens,
44
+ temperature=temperature,
45
+ )
46
+ return resp["choices"][0]["message"]["content"].strip()
47
+
48
+
49
+ def generate_sql(prompt: str) -> str:
50
+ """`prompt` should already be the full schema+few-shot+question text
51
+ from llm/prompt_template.py:build_prompt(). Returns raw SQL text."""
52
+ raw = _chat(
53
+ system_prompt="You write SQLite SELECT queries. Output only the SQL, nothing else.",
54
+ user_prompt=prompt,
55
+ max_tokens=200,
56
+ temperature=0.0,
57
+ )
58
+ # models sometimes wrap output in ```sql fences despite instructions — strip them
59
+ raw = raw.replace("```sql", "").replace("```", "").strip()
60
+ return raw
61
+
62
+
63
+ def synthesize_answer(question: str, columns: list, rows: list) -> str:
64
+ if not rows:
65
+ return "No matching results found."
66
+ preview = rows[:20]
67
+ table_text = " | ".join(columns) + "\n" + "\n".join(
68
+ " | ".join(str(v) for v in r) for r in preview
69
+ )
70
+ user_prompt = (
71
+ f"Question: {question}\n\n"
72
+ f"Query result ({len(rows)} row(s), showing up to 20):\n{table_text}\n\n"
73
+ "Answer the question in one or two plain sentences, referencing the "
74
+ "actual numbers/names from the result. Do not mention SQL."
75
+ )
76
+ return _chat(
77
+ system_prompt="You are a concise network operations assistant.",
78
+ user_prompt=user_prompt,
79
+ max_tokens=200,
80
+ temperature=0.2,
81
+ )
network.db ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92c0d5332c2d241d6dfa0ad344887de13730c1071cb055db7aee85ea8066b359
3
+ size 770048
prompt_template.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Schema context + few-shot examples fed to the text-to-SQL LLM."""
2
+
3
+ SCHEMA_CONTEXT = """
4
+ You are a network operations assistant that writes SQLite SELECT queries
5
+ against the following schema. Only these tables and columns exist.
6
+
7
+ devices(device_id, hostname, ip_address, device_type, vendor, location, status, last_seen)
8
+ -- status is one of: up, down, degraded
9
+
10
+ interfaces(interface_id, device_id, name, status, speed_mbps)
11
+ -- status is one of: up, down
12
+
13
+ traffic_flows(flow_id, ts, src_ip, dst_ip, src_port, dst_port, protocol,
14
+ bytes, packets, duration_ms, device_id, label)
15
+ -- label is 'benign' or an attack type string
16
+
17
+ alerts(alert_id, ts, device_id, severity, alert_type, description, resolved)
18
+ -- severity is one of: info, warning, critical
19
+ -- resolved is 0 or 1
20
+
21
+ metrics_timeseries(metric_id, ts, device_id, interface_id, metric_name, value)
22
+ -- metric_name is one of: latency_ms, packet_loss_pct, bandwidth_util_pct
23
+
24
+ All timestamps (ts, last_seen) are ISO8601 strings, comparable with datetime().
25
+ 'now' in SQLite gives the current UTC time.
26
+
27
+ Rules:
28
+ - Output ONLY a single SQLite SELECT statement. No prose, no markdown fences.
29
+ - Never write INSERT, UPDATE, DELETE, DROP, ALTER, or any statement other than SELECT.
30
+ - Always include a LIMIT (default 50) unless the question asks for a count/aggregate.
31
+ - Prefer JOINs to devices when a question asks about a device by name.
32
+ """
33
+
34
+ FEW_SHOT_EXAMPLES = [
35
+ (
36
+ "Which devices are currently down?",
37
+ "SELECT hostname, ip_address, location FROM devices WHERE status = 'down' LIMIT 50;",
38
+ ),
39
+ (
40
+ "Show me all interfaces on router-03 and their status",
41
+ "SELECT i.name, i.status, i.speed_mbps FROM interfaces i "
42
+ "JOIN devices d ON d.device_id = i.device_id "
43
+ "WHERE d.hostname = 'router-03' LIMIT 50;",
44
+ ),
45
+ (
46
+ "What are the top 5 talkers by bandwidth in the last hour?",
47
+ "SELECT d.hostname, SUM(f.bytes) AS total_bytes FROM traffic_flows f "
48
+ "JOIN devices d ON d.device_id = f.device_id "
49
+ "WHERE f.ts >= datetime('now', '-1 hour') "
50
+ "GROUP BY d.hostname ORDER BY total_bytes DESC LIMIT 5;",
51
+ ),
52
+ (
53
+ "Any unresolved critical alerts right now?",
54
+ "SELECT a.ts, d.hostname, a.alert_type, a.description FROM alerts a "
55
+ "JOIN devices d ON d.device_id = a.device_id "
56
+ "WHERE a.severity = 'critical' AND a.resolved = 0 "
57
+ "ORDER BY a.ts DESC LIMIT 50;",
58
+ ),
59
+ (
60
+ "Which interfaces have packet loss above 2% right now?",
61
+ "SELECT d.hostname, i.name, m.value FROM metrics_timeseries m "
62
+ "JOIN interfaces i ON i.interface_id = m.interface_id "
63
+ "JOIN devices d ON d.device_id = m.device_id "
64
+ "WHERE m.metric_name = 'packet_loss_pct' AND m.value > 2 "
65
+ "AND m.ts >= datetime('now', '-15 minutes') "
66
+ "ORDER BY m.value DESC LIMIT 50;",
67
+ ),
68
+ (
69
+ "How many devices are in a degraded state right now?",
70
+ "SELECT COUNT(*) AS degraded_count FROM devices WHERE status = 'degraded';",
71
+ ),
72
+ (
73
+ "List all alerts on the core switch in the last 24 hours",
74
+ "SELECT a.ts, a.severity, a.alert_type, a.description FROM alerts a "
75
+ "JOIN devices d ON d.device_id = a.device_id "
76
+ "WHERE d.hostname LIKE '%switch%' AND a.ts >= datetime('now', '-1 day') "
77
+ "ORDER BY a.ts DESC LIMIT 50;",
78
+ ),
79
+ (
80
+ "Show total traffic volume by protocol today",
81
+ "SELECT protocol, SUM(bytes) AS total_bytes FROM traffic_flows "
82
+ "WHERE ts >= datetime('now', 'start of day') "
83
+ "GROUP BY protocol ORDER BY total_bytes DESC LIMIT 50;",
84
+ ),
85
+ ]
86
+
87
+
88
+ def build_prompt(user_question: str) -> str:
89
+ examples_text = "\n".join(
90
+ f"Q: {q}\nSQL: {sql}" for q, sql in FEW_SHOT_EXAMPLES
91
+ )
92
+ return (
93
+ f"{SCHEMA_CONTEXT}\n"
94
+ f"Examples:\n{examples_text}\n\n"
95
+ f"Q: {user_question}\nSQL:"
96
+ )
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ gradio>=4.44
2
+ llama-cpp-python>=0.2.90
schema.sql ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ -- Network state schema (SQLite dialect, demo)
2
+ -- Migrates cleanly to Postgres later: SERIAL -> INTEGER PRIMARY KEY AUTOINCREMENT already
3
+ -- SQLite-compatible; swap to Postgres types when you move off the demo DB.
4
+
5
+ CREATE TABLE IF NOT EXISTS devices (
6
+ device_id INTEGER PRIMARY KEY AUTOINCREMENT,
7
+ hostname TEXT,
8
+ ip_address TEXT UNIQUE,
9
+ device_type TEXT, -- router, switch, firewall, host
10
+ vendor TEXT,
11
+ location TEXT,
12
+ status TEXT, -- up, down, degraded
13
+ last_seen TEXT -- ISO8601 timestamp
14
+ );
15
+
16
+ CREATE TABLE IF NOT EXISTS interfaces (
17
+ interface_id INTEGER PRIMARY KEY AUTOINCREMENT,
18
+ device_id INTEGER REFERENCES devices(device_id),
19
+ name TEXT, -- eth0, Gi0/1, etc.
20
+ status TEXT, -- up, down
21
+ speed_mbps INTEGER
22
+ );
23
+
24
+ CREATE TABLE IF NOT EXISTS traffic_flows (
25
+ flow_id INTEGER PRIMARY KEY AUTOINCREMENT,
26
+ ts TEXT, -- ISO8601 timestamp
27
+ src_ip TEXT,
28
+ dst_ip TEXT,
29
+ src_port INTEGER,
30
+ dst_port INTEGER,
31
+ protocol TEXT,
32
+ bytes INTEGER,
33
+ packets INTEGER,
34
+ duration_ms INTEGER,
35
+ device_id INTEGER REFERENCES devices(device_id),
36
+ label TEXT -- benign / attack type
37
+ );
38
+
39
+ CREATE TABLE IF NOT EXISTS alerts (
40
+ alert_id INTEGER PRIMARY KEY AUTOINCREMENT,
41
+ ts TEXT,
42
+ device_id INTEGER REFERENCES devices(device_id),
43
+ severity TEXT, -- info, warning, critical
44
+ alert_type TEXT, -- port_scan, dos, malware, link_down
45
+ description TEXT,
46
+ resolved INTEGER DEFAULT 0 -- 0/1 boolean
47
+ );
48
+
49
+ CREATE TABLE IF NOT EXISTS metrics_timeseries (
50
+ metric_id INTEGER PRIMARY KEY AUTOINCREMENT,
51
+ ts TEXT,
52
+ device_id INTEGER REFERENCES devices(device_id),
53
+ interface_id INTEGER REFERENCES interfaces(interface_id),
54
+ metric_name TEXT, -- latency_ms, packet_loss_pct, bandwidth_util_pct
55
+ value REAL
56
+ );
57
+
58
+ CREATE INDEX IF NOT EXISTS idx_flows_device_ts ON traffic_flows(device_id, ts);
59
+ CREATE INDEX IF NOT EXISTS idx_alerts_device_ts ON alerts(device_id, ts);
60
+ CREATE INDEX IF NOT EXISTS idx_metrics_device_ts ON metrics_timeseries(device_id, ts);
seed_from_csv.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Seeds network.db with device/interface/flow/alert/metric data.
3
+
4
+ Two modes:
5
+ 1. Real data: point --csv at a NetFlow-style CSV (e.g. NF-UNSW-NB15-v2 from
6
+ https://staff.itee.uq.edu.au/marius/NIDS_datasets/) with columns similar to
7
+ IPV4_SRC_ADDR, IPV4_DST_ADDR, L4_SRC_PORT, L4_DST_PORT, PROTOCOL, IN_BYTES,
8
+ IN_PKTS, FLOW_DURATION_MILLISECONDS, Label, Attack. Devices/interfaces/
9
+ alerts are derived from the distinct IPs in the flow data.
10
+ 2. Synthetic fallback (default, no --csv given): generates a realistic set of
11
+ devices, interfaces, flows, alerts, and metrics from scratch so the demo
12
+ works with zero external downloads.
13
+
14
+ Usage:
15
+ python seed_from_csv.py # synthetic data
16
+ python seed_from_csv.py --csv path/to/nf_unsw_nb15_v2.csv --limit 20000
17
+ """
18
+
19
+ import argparse
20
+ import ipaddress
21
+ import random
22
+ import sqlite3
23
+ from datetime import datetime, timedelta
24
+ from pathlib import Path
25
+
26
+ DB_PATH = Path(__file__).parent / "network.db"
27
+ SCHEMA_PATH = Path(__file__).parent / "schema.sql"
28
+
29
+ DEVICE_TYPES = ["router", "switch", "firewall", "host", "access_point"]
30
+ VENDORS = ["Cisco", "Juniper", "Arista", "MikroTik", "Ubiquiti", "Fortinet"]
31
+ LOCATIONS = ["HQ-Floor1", "HQ-Floor2", "DataCenter-A", "Branch-Lagos", "Branch-Abuja", "Cloud-Edge"]
32
+ PROTOCOLS = ["TCP", "UDP", "ICMP"]
33
+ ALERT_TYPES = ["port_scan", "dos", "malware", "link_down", "auth_failure", "config_drift"]
34
+ SEVERITIES = ["info", "warning", "critical"]
35
+
36
+
37
+ def reset_db():
38
+ if DB_PATH.exists():
39
+ DB_PATH.unlink()
40
+ conn = sqlite3.connect(DB_PATH)
41
+ conn.executescript(SCHEMA_PATH.read_text())
42
+ conn.commit()
43
+ return conn
44
+
45
+
46
+ def now_minus(seconds):
47
+ return (datetime.utcnow() - timedelta(seconds=seconds)).isoformat(timespec="seconds")
48
+
49
+
50
+ def seed_synthetic(conn, n_devices=18, n_flows=4000, n_alerts=40, n_metrics=3000):
51
+ cur = conn.cursor()
52
+
53
+ device_ids = []
54
+ for i in range(n_devices):
55
+ dtype = random.choice(DEVICE_TYPES)
56
+ status = random.choices(["up", "degraded", "down"], weights=[0.82, 0.13, 0.05])[0]
57
+ cur.execute(
58
+ "INSERT INTO devices (hostname, ip_address, device_type, vendor, location, status, last_seen) "
59
+ "VALUES (?, ?, ?, ?, ?, ?, ?)",
60
+ (
61
+ f"{dtype}-{i+1:02d}",
62
+ str(ipaddress.IPv4Address(0x0A000000 + i + 1)), # 10.0.0.x
63
+ dtype,
64
+ random.choice(VENDORS),
65
+ random.choice(LOCATIONS),
66
+ status,
67
+ now_minus(random.randint(0, 600)),
68
+ ),
69
+ )
70
+ device_ids.append(cur.lastrowid)
71
+
72
+ interface_ids = []
73
+ for did in device_ids:
74
+ for j in range(random.randint(1, 4)):
75
+ cur.execute(
76
+ "INSERT INTO interfaces (device_id, name, status, speed_mbps) VALUES (?, ?, ?, ?)",
77
+ (
78
+ did,
79
+ f"Gi0/{j}",
80
+ random.choices(["up", "down"], weights=[0.9, 0.1])[0],
81
+ random.choice([100, 1000, 10000]),
82
+ ),
83
+ )
84
+ interface_ids.append((cur.lastrowid, did))
85
+
86
+ for _ in range(n_flows):
87
+ did = random.choice(device_ids)
88
+ is_attack = random.random() < 0.06
89
+ cur.execute(
90
+ "INSERT INTO traffic_flows (ts, src_ip, dst_ip, src_port, dst_port, protocol, "
91
+ "bytes, packets, duration_ms, device_id, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
92
+ (
93
+ now_minus(random.randint(0, 86400)),
94
+ f"10.0.0.{random.randint(1, 254)}",
95
+ f"{random.choice(['172.16', '192.168', '203.0'])}.{random.randint(0,255)}.{random.randint(1,254)}",
96
+ random.randint(1024, 65535),
97
+ random.choice([80, 443, 22, 53, 3389, 8080, 3306]),
98
+ random.choice(PROTOCOLS),
99
+ random.randint(200, 5_000_000),
100
+ random.randint(1, 4000),
101
+ random.randint(1, 60000),
102
+ did,
103
+ random.choice(ALERT_TYPES) if is_attack else "benign",
104
+ ),
105
+ )
106
+
107
+ for _ in range(n_alerts):
108
+ did = random.choice(device_ids)
109
+ cur.execute(
110
+ "INSERT INTO alerts (ts, device_id, severity, alert_type, description, resolved) "
111
+ "VALUES (?, ?, ?, ?, ?, ?)",
112
+ (
113
+ now_minus(random.randint(0, 172800)),
114
+ did,
115
+ random.choices(SEVERITIES, weights=[0.5, 0.35, 0.15])[0],
116
+ random.choice(ALERT_TYPES),
117
+ "Auto-generated demo alert",
118
+ random.choices([0, 1], weights=[0.4, 0.6])[0],
119
+ ),
120
+ )
121
+
122
+ for _ in range(n_metrics):
123
+ iface_id, did = random.choice(interface_ids)
124
+ metric_name = random.choice(["latency_ms", "packet_loss_pct", "bandwidth_util_pct"])
125
+ value = {
126
+ "latency_ms": round(random.uniform(1, 180), 2),
127
+ "packet_loss_pct": round(random.uniform(0, 6), 2),
128
+ "bandwidth_util_pct": round(random.uniform(2, 98), 2),
129
+ }[metric_name]
130
+ cur.execute(
131
+ "INSERT INTO metrics_timeseries (ts, device_id, interface_id, metric_name, value) "
132
+ "VALUES (?, ?, ?, ?, ?)",
133
+ (now_minus(random.randint(0, 3600)), did, iface_id, metric_name, value),
134
+ )
135
+
136
+ conn.commit()
137
+ print(f"Synthetic seed complete: {len(device_ids)} devices, {n_flows} flows, "
138
+ f"{n_alerts} alerts, {n_metrics} metrics.")
139
+
140
+
141
+ def seed_from_real_csv(conn, csv_path, limit):
142
+ import csv as csv_mod
143
+
144
+ cur = conn.cursor()
145
+ ip_to_device = {}
146
+
147
+ def get_or_create_device(ip):
148
+ if ip in ip_to_device:
149
+ return ip_to_device[ip]
150
+ cur.execute(
151
+ "INSERT INTO devices (hostname, ip_address, device_type, vendor, location, status, last_seen) "
152
+ "VALUES (?, ?, ?, ?, ?, ?, ?)",
153
+ (f"host-{ip}", ip, "host", random.choice(VENDORS), random.choice(LOCATIONS), "up", now_minus(0)),
154
+ )
155
+ ip_to_device[ip] = cur.lastrowid
156
+ return cur.lastrowid
157
+
158
+ with open(csv_path, newline="") as f:
159
+ reader = csv_mod.DictReader(f)
160
+ for i, row in enumerate(reader):
161
+ if i >= limit:
162
+ break
163
+ src_ip = row.get("IPV4_SRC_ADDR") or row.get("src_ip") or "0.0.0.0"
164
+ dst_ip = row.get("IPV4_DST_ADDR") or row.get("dst_ip") or "0.0.0.0"
165
+ device_id = get_or_create_device(src_ip)
166
+ label = row.get("Attack") or row.get("label") or ("benign" if row.get("Label", "0") == "0" else "attack")
167
+ cur.execute(
168
+ "INSERT INTO traffic_flows (ts, src_ip, dst_ip, src_port, dst_port, protocol, "
169
+ "bytes, packets, duration_ms, device_id, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
170
+ (
171
+ now_minus(random.randint(0, 86400)),
172
+ src_ip,
173
+ dst_ip,
174
+ int(row.get("L4_SRC_PORT", 0) or 0),
175
+ int(row.get("L4_DST_PORT", 0) or 0),
176
+ row.get("PROTOCOL", "TCP"),
177
+ int(row.get("IN_BYTES", 0) or 0),
178
+ int(row.get("IN_PKTS", 0) or 0),
179
+ int(float(row.get("FLOW_DURATION_MILLISECONDS", 0) or 0)),
180
+ device_id,
181
+ label,
182
+ ),
183
+ )
184
+
185
+ conn.commit()
186
+ print(f"Loaded {i+1} flows from {csv_path}, derived {len(ip_to_device)} devices.")
187
+
188
+
189
+ if __name__ == "__main__":
190
+ parser = argparse.ArgumentParser()
191
+ parser.add_argument("--csv", type=str, default=None, help="Path to a real NetFlow CSV")
192
+ parser.add_argument("--limit", type=int, default=20000, help="Max rows to load from CSV")
193
+ args = parser.parse_args()
194
+
195
+ conn = reset_db()
196
+ if args.csv:
197
+ seed_from_real_csv(conn, args.csv, args.limit)
198
+ else:
199
+ seed_synthetic(conn)
200
+ conn.close()
simulator.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Background thread that periodically inserts new flow/metric rows so the
2
+ demo DB feels like it's reflecting a live network, without needing a real
3
+ ingestion pipeline."""
4
+
5
+ import random
6
+ import sqlite3
7
+ import threading
8
+ import time
9
+ from datetime import datetime
10
+ from pathlib import Path
11
+
12
+ DB_PATH = Path(__file__).parent / "network.db"
13
+ PROTOCOLS = ["TCP", "UDP", "ICMP"]
14
+
15
+
16
+ def _tick(conn):
17
+ cur = conn.cursor()
18
+ cur.execute("SELECT device_id FROM devices")
19
+ device_ids = [r[0] for r in cur.fetchall()]
20
+ if not device_ids:
21
+ return
22
+
23
+ now = datetime.utcnow().isoformat(timespec="seconds")
24
+ did = random.choice(device_ids)
25
+
26
+ cur.execute(
27
+ "INSERT INTO traffic_flows (ts, src_ip, dst_ip, src_port, dst_port, protocol, "
28
+ "bytes, packets, duration_ms, device_id, label) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
29
+ (
30
+ now,
31
+ f"10.0.0.{random.randint(1, 254)}",
32
+ f"192.168.{random.randint(0,255)}.{random.randint(1,254)}",
33
+ random.randint(1024, 65535),
34
+ random.choice([80, 443, 22, 53]),
35
+ random.choice(PROTOCOLS),
36
+ random.randint(200, 2_000_000),
37
+ random.randint(1, 3000),
38
+ random.randint(1, 5000),
39
+ did,
40
+ "benign",
41
+ ),
42
+ )
43
+
44
+ cur.execute("SELECT interface_id, device_id FROM interfaces WHERE device_id = ?", (did,))
45
+ ifaces = cur.fetchall()
46
+ if ifaces:
47
+ iface_id, iface_device_id = random.choice(ifaces)
48
+ metric_name = random.choice(["latency_ms", "packet_loss_pct", "bandwidth_util_pct"])
49
+ value = {
50
+ "latency_ms": round(random.uniform(1, 180), 2),
51
+ "packet_loss_pct": round(random.uniform(0, 6), 2),
52
+ "bandwidth_util_pct": round(random.uniform(2, 98), 2),
53
+ }[metric_name]
54
+ cur.execute(
55
+ "INSERT INTO metrics_timeseries (ts, device_id, interface_id, metric_name, value) "
56
+ "VALUES (?, ?, ?, ?, ?)",
57
+ (now, iface_device_id, iface_id, metric_name, value),
58
+ )
59
+
60
+ conn.commit()
61
+
62
+
63
+ def _loop(interval_seconds):
64
+ conn = sqlite3.connect(DB_PATH, check_same_thread=False)
65
+ while True:
66
+ try:
67
+ _tick(conn)
68
+ except Exception as e:
69
+ print(f"[simulator] tick failed: {e}")
70
+ time.sleep(interval_seconds)
71
+
72
+
73
+ def start(interval_seconds: int = 5):
74
+ """Starts the simulator in a daemon thread. Call once at app startup."""
75
+ t = threading.Thread(target=_loop, args=(interval_seconds,), daemon=True)
76
+ t.start()
77
+ return t