app/renderer/knowledge.js

// Chat grounding: search the Devseis knowledge base (library/knowledge.json) and build the chat prompt.
// Used by the web chat, the desktop app and (through training/chat-contexts.mjs) the training-data builder,
// so the model is trained on exactly the prompts it gets at runtime. Pure functions, no DOM or network.

export const CHAT_SYSTEM_PROMPT = 'You are the Devseis security assistant. You help people secure their computers and phones and '
  + 'understand ISO 27001, GDPR and the EU AI Act. Answer in plain language that a non-technical person can follow, using only '
  + 'the CONTEXT and MY AUDIT below, and cite the passages you use as [1], [2]. If the CONTEXT does not answer the question, say '
  + 'you do not know rather than guessing. If the question is about the user\'s own device and MY AUDIT is "none", explain that '
  + 'you cannot see their device and suggest running an audit. Never invent settings, versions or numbers.';

const STOP = new Set('a an and are as at be by can do does for from has have how i if in is it its me my of on or should so that the their them this to was what when where which who why will with you your'.split(' '));
// Words people use -> words the knowledge base uses.
const SYNONYMS = {
  filevault: 'encryption', bitlocker: 'encryption', luks: 'encryption', encrypt: 'encryption', encrypted: 'encryption',
  antivirus: 'malware antivirus', virus: 'malware', defender: 'malware antivirus defender', xprotect: 'malware', clamav: 'malware',
  passcode: 'passcode screen lock', pin: 'passcode pin screen lock', autolock: 'auto lock screen timeout', lock: 'lock screen',
  password: 'password', admin: 'administrator', administrator: 'administrator', sudo: 'administrator', root: 'administrator rooted',
  jailbreak: 'jailbroken', jailbroken: 'jailbroken', rooted: 'rooted', update: 'updates update', updates: 'updates update', patch: 'updates security patch',
  backup: 'backup', 'time machine': 'backup', icloud: 'icloud backup', usb: 'usb storage', 'usb stick': 'usb storage', thumb: 'usb storage',
  firewall: 'firewall', vpn: 'remote access', rdp: 'remote access', teamviewer: 'remote access teamviewer', anydesk: 'remote access anydesk',
  cve: 'vulnerabilities vulnerability', vulnerability: 'vulnerabilities vulnerability', vulnerable: 'vulnerabilities', ai: 'ai', chatgpt: 'chatgpt ai',
  gpt: 'chatgpt ai', copilot: 'copilot ai', gemini: 'gemini ai', iphone: 'iphone ios', ipad: 'iphone ios ipados', mac: 'macos', macbook: 'macos',
  osx: 'macos', windows: 'windows', win: 'windows', ubuntu: 'linux ubuntu', debian: 'linux debian', fedora: 'linux', android: 'android',
  galaxy: 'android', pixel: 'android', samsung: 'android', gdpr: 'gdpr', iso: 'iso 27001', '27001': 'iso 27001', 'ai act': 'ai act',
  install: 'install', download: 'install download', privacy: 'privacy', private: 'privacy', notifications: 'notification lock screen previews',
};
const OS_WORDS = { macos: ['macos', 'mac', 'macbook', 'osx', 'filevault'], windows: ['windows', 'win', 'bitlocker', 'defender'],
  linux: ['linux', 'ubuntu', 'debian', 'fedora', 'luks'], ios: ['iphone', 'ipad', 'ios'], android: ['android', 'galaxy', 'pixel', 'samsung'] };

function stem(word) {
  if (word.length > 5 && word.endsWith('ing')) return word.slice(0, -3);
  if (word.length > 4 && word.endsWith('ed')) return word.slice(0, -2);
  if (word.length > 4 && word.endsWith('es') && !word.endsWith('ses')) return word.slice(0, -2);
  if (word.length > 3 && word.endsWith('s') && !word.endsWith('ss')) return word.slice(0, -1);
  return word;
}
export function tokens(text, expand = false) {
  let t = String(text).toLowerCase();
  if (expand) for (const [k, v] of Object.entries(SYNONYMS)) if (k.includes(' ') && t.includes(k)) t += ' ' + v;
  const words = t.split(/[^a-z0-9.]+/).map(w => w.replace(/^\.+|\.+$/g, '')).filter(w => w && !STOP.has(w));
  const out = [];
  for (const w of words) {
    out.push(stem(w));
    if (expand && SYNONYMS[w]) for (const s of SYNONYMS[w].split(' ')) out.push(stem(s));
  }
  return out;
}

// The operating system a question is about, if it names one.
export function osInQuestion(question) {
  const words = new Set(String(question).toLowerCase().split(/[^a-z0-9]+/));
  for (const [os, list] of Object.entries(OS_WORDS)) if (list.some(w => words.has(w))) return os;
  return null;
}

export class Knowledge {
  constructor(data) {
    this.chunks = data.chunks;
    this.docs = this.chunks.map(c => {
      const tf = new Map();
      for (const t of [...tokens(c.title), ...tokens(c.title), ...tokens(c.text)]) tf.set(t, (tf.get(t) || 0) + 1);
      let len = 0; for (const v of tf.values()) len += v;
      return { tf, len };
    });
    this.avg = this.docs.reduce((s, d) => s + d.len, 0) / this.docs.length;
    this.df = new Map();
    for (const d of this.docs) for (const t of d.tf.keys()) this.df.set(t, (this.df.get(t) || 0) + 1);
    this.byId = new Map(this.chunks.map((c, i) => [c.id, i]));
  }

  // Top-k passages (BM25), preferring the OS the question names, else the device's OS.
  search(question, { k = 4, os = null } = {}) {
    const target = osInQuestion(question) || os;
    const wantsOn = /\b(turn on|switch on|enable|activate|set up|setup)\b/i.test(question);
    const q = tokens(question, true);
    const N = this.docs.length, k1 = 1.4, b = 0.75;
    const scores = this.docs.map((d, i) => {
      let s = 0;
      for (const t of new Set(q)) {
        const f = d.tf.get(t); if (!f) continue;
        const idf = Math.log(1 + (N - this.df.get(t) + 0.5) / (this.df.get(t) + 0.5));
        s += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * d.len / this.avg));
      }
      const c = this.chunks[i];
      if (target && c.os) s *= c.os === target ? 1.6 : 0.45;
      if (wantsOn && c.check && /\.enabled$/.test(c.check)) s *= 1.35;  // "turn on X" -> the X-is-on check
      return { i, s };
    });
    return scores.filter(x => x.s > 0).sort((a, b) => b.s - a.s || a.i - b.i).slice(0, k).map(x => this.chunks[x.i]);
  }

  get(id) { const i = this.byId.get(id); return i === undefined ? null : this.chunks[i]; }
}

// Short, model-friendly summary of an audit the user ran (evidence JSON), or "none".
export function auditBrief(evidence, maxGaps = 8) {
  if (!evidence) return 'none';
  const h = evidence.host, s = evidence.summary;
  const assessed = s.Compliant + s['Non-Compliant'];
  const lines = [`${h.os_name} ${h.os_version} (${h.os_family}), audited ${String(evidence.audit.started_at).slice(0, 10)}; `
    + `${s.Compliant} of ${assessed} assessed checks compliant; ${s['Non-Compliant']} gaps, ${s.Pending} pending, ${s.Error} not read.`];
  const gaps = evidence.results.filter(r => r.status === 'Non-Compliant').slice(0, maxGaps);
  if (gaps.length) lines.push('Gaps: ' + gaps.map(r => `${r.name}: ${r.actual}`).join('; ') + '.');
  return lines.join(' ');
}

// Messages for one chat turn. history: earlier [{role, content}] turns (already answered).
// device: what the browser saw about the device the chat runs on (deviceBrief in the former chat/device.js), or null.
export function chatMessages(question, passages, audit = 'none', history = [], device = null) {
  const context = passages.length ? passages.map((p, n) => `[${n + 1}] ${p.title}: ${p.text}`).join('\n') : '(no matching passages)';
  const user = `CONTEXT:\n${context}\n\nMY AUDIT: ${audit}\n\n${device ? `MY DEVICE: ${device}\n\n` : ''}QUESTION: ${question}`;
  return [{ role: 'system', content: CHAT_SYSTEM_PROMPT }, ...history.slice(-4), { role: 'user', content: user }];
}

// Degenerate output from a broken engine ("QUITQUITQUIT…", "warmth warmth warmth…") has no numbers or citations to catch.
export function repeatedText(answer) {
  const words = answer.match(/\S+/g) || [];
  return /(.{2,30}?)\1{9,}/s.test(answer) || (words.length >= 30 && new Set(words).size / words.length < 0.2);
}

// A scrap instead of an answer ("Answer to 1:4:30"): under 6 words, no citation. Real short replies (greetings,
// "I can't see that") are longer than this.
export function tooShort(answer) {
  return !/\[\d+\]/.test(answer) && (answer.match(/[A-Za-z]{2,}/g) || []).length < 6;
}

// Grounding rules for a chat answer (same as check_chat() in training/build_chat.py).
const NUMBER = /(?<![\w.])\d+(?:\.\d+)?(?!\w)/g;
export function chatProblems(userPrompt, answer) {
  const problems = [];
  if (repeatedText(answer)) problems.push('repeated text');
  const passages = (userPrompt.split('MY AUDIT:')[0].match(/^\[\d+\] /gm) || []).length;
  for (const m of answer.matchAll(/\[(\d+)\]/g)) if (+m[1] < 1 || +m[1] > passages) problems.push(`citation [${m[1]}] out of range`);
  const allowed = new Set(userPrompt.match(NUMBER) || []);
  for (const n of answer.match(NUMBER) || []) if (!allowed.has(n) && !(/^\d+$/.test(n) && +n <= 12)) problems.push(`number ${n} not in sources`);
  return problems;
}

// With an audit attached: the fix guide for the gap the question names, else the top gap, is always among the
// passages (as in the training data), so "what should I fix first?" is answered from the right guide.
export function searchWithAudit(kb, question, evidence, { k = 4, os = null } = {}) {
  let passages = kb.search(question, { k, os: evidence ? evidence.host.os_family : os });
  if (!evidence) return passages;
  const gaps = evidence.results.filter(r => r.status === 'Non-Compliant').slice(0, 8);
  if (!gaps.length) return passages;
  const q = String(question).toLowerCase();
  const named = gaps.find(g => q.includes(g.name.toLowerCase()));
  const gap = named || gaps[0];
  const gold = kb.get(`fix:${gap.id}:${evidence.host.os_family}`);
  if (gold && !passages.some(p => p.id === gold.id)) passages = [...passages.slice(0, k - 1), gold];
  return passages;
}