Blame for src/core/context.ts

/**
 * The context files behind the Windows menu, in the formats bankml's Savante UI
 * and mindX already use, so files move between them unchanged:
 *
 *   .history  JSONL, one exchange per line          (bankml savante.history)
 *   .memory   JSONL, one note per line              (bankml savante.memory)
 *   .prompt   plain text: the system prompt         (bankml/mindX *.prompt)
 *   .persona  JSON: mindX .persona v1 or boardroom  (mindX personas)
 *   .model    YAML: an inference policy             (cryptoAGI *.model)
 */

// ---- .history --------------------------------------------------------------

/** One exchange. Extra fields from other tools are kept as they are. */
export interface HistoryRecord {
  /** Seconds since the epoch, as bankml writes it. */
  ts: number;
  session?: string;
  /** The output field the exchange happened in (UIF addition). */
  output?: string;
  user: string;
  assistant: string;
  /** Where the system prompt came from, e.g. "persona.system_prompt". */
  prompt?: string;
  model?: string;
  [extra: string]: unknown;
}

// ---- .memory ---------------------------------------------------------------

export interface MemoryNote {
  ts: number;
  /** ISO time. */
  at: string;
  text: string;
  /** sha256 of `text`, hex. */
  sha256: string;
  /** typed and response notes reach the model; a delivery record (how an answer was delivered) is kept, never sent. */
  source: { kind: 'typed' | 'response' | 'delivery'; [extra: string]: unknown };
}

/** Header bankml puts above the notes in the system prompt. */
export const MEMORY_HEADER =
  '\n\nMEMORY — notes the operator kept from earlier conversations. They are context, not evidence: cite them as the operator\'s notes, never as findings.\n';
export const MEMORY_BLOCK_CHARS = 2400;

/** The notes as they reach the model: newest first, `- ` lines, capped like bankml. */
export function memoryBlock(notes: MemoryNote[], cap = MEMORY_BLOCK_CHARS): string {
  if (!notes.some(n => n.source.kind !== 'delivery')) return '';
  const lines: string[] = [];
  let used = 0;
  for (const n of [...notes].filter(n => n.source.kind !== 'delivery').sort((a, b) => b.ts - a.ts)) {
    const line = `- ${n.text.replace(/\s+/g, ' ').trim()}`;
    if (used + line.length + 1 > cap) break;
    lines.push(line);
    used += line.length + 1;
  }
  return lines.length ? MEMORY_HEADER + lines.join('\n') : '';
}

/**
 * A delivery record as one .memory line: the note fields the UIF reads (ts, at, text, sha256, source.kind
 * "delivery") beside the engine's own record. A record in the sagi .memory shape (id, date, question, answer,
 * carrier, deliveries, finish, limit, usage, timings, response_sha256) is kept field for field, so the same line
 * can be read by Savante's bind/skill_history.py. `text` is its one-line summary.
 */
export async function deliveryLine(record: Record<string, unknown>, text: string, now = Date.now() / 1000): Promise<string> {
  const ts = typeof record.ts === 'number' ? record.ts : now;
  const summary = text.replace(/\s+/g, ' ').trim();
  return JSON.stringify({ ...record, ts, at: new Date(ts * 1000).toISOString(), text: summary, sha256: await sha256Hex(summary), source: { kind: 'delivery' } });
}

// ---- .persona --------------------------------------------------------------

/**
 * A persona. Two shapes are understood: mindX ".persona v1" (`system_prompt`,
 * `mantra`, `voice_examples`, …) and the boardroom shape (`role`,
 * `behavioral_traits`, `beliefs`, `desires`, …). Unknown keys are kept.
 */
export interface PersonaDoc {
  name?: string;
  persona?: string;
  role?: string;
  description?: string;
  system_prompt?: string;
  communication_style?: string;
  behavioral_traits?: string[];
  beliefs?: Record<string, boolean>;
  desires?: Record<string, unknown>;
  [extra: string]: unknown;
}

export function parsePersona(text: string): PersonaDoc {
  let data: unknown;
  try {
    data = JSON.parse(text);
  } catch {
    throw new Error('not a valid .persona file: it is not JSON');
  }
  if (!data || typeof data !== 'object' || Array.isArray(data)) throw new Error('not a valid .persona file: expected a JSON object');
  const p = data as PersonaDoc;
  if (p.system_prompt !== undefined && typeof p.system_prompt !== 'string') throw new Error('not a valid .persona file: "system_prompt" must be text');
  if (p.behavioral_traits !== undefined && !(Array.isArray(p.behavioral_traits) && p.behavioral_traits.every(t => typeof t === 'string')))
    throw new Error('not a valid .persona file: "behavioral_traits" must be a list of text');
  return p;
}

export function personaName(p: PersonaDoc | null): string | undefined {
  return p ? (p.name ?? p.persona ?? undefined) : undefined;
}

const HIGH = new Set(['high', 'critical']);

/** The persona's sourced facts about itself (`awareness.facts[].fact`, as in bankml.persona). */
export function awarenessFacts(p: PersonaDoc): string[] {
  const a = p.awareness;
  const facts = a && typeof a === 'object' ? (a as { facts?: unknown }).facts : undefined;
  if (!Array.isArray(facts)) return [];
  return facts
    .map(f => (typeof f === 'string' ? f : f && typeof f === 'object' ? (f as { fact?: unknown }).fact : undefined))
    .filter((f): f is string => typeof f === 'string' && !!f.trim())
    .map(f => f.trim());
}

/**
 * Lines appended after the prompt: boardroom traits, true beliefs and high
 * priorities, then the persona's awareness facts.
 */
export function personaLines(p: PersonaDoc): string[] {
  const lines: string[] = [];
  if (p.behavioral_traits?.length) lines.push(`Behavioural traits: ${p.behavioral_traits.join(', ')}`);
  const beliefs = Object.entries(p.beliefs ?? {}).filter(([, v]) => v === true).map(([k]) => k.replace(/_/g, ' '));
  if (beliefs.length) lines.push(`Operating beliefs: ${beliefs.join(', ')}`);
  const priorities = Object.entries(p.desires ?? {})
    .filter(([, v]) => {
      const level = typeof v === 'string' ? v : v && typeof v === 'object' ? (v as { priority?: unknown }).priority : undefined;
      return typeof level === 'string' && HIGH.has(level.toLowerCase());
    })
    .map(([k]) => k.replace(/_/g, ' '));
  if (priorities.length) lines.push(`Priorities: ${priorities.join(', ')}`);
  const facts = awarenessFacts(p);
  if (facts.length) lines.push(`What I know about myself:\n${facts.map(f => `- ${f}`).join('\n')}`);
  return lines;
}

// ---- .model ----------------------------------------------------------------

/** An inference policy, not a pinned weight file. Other keys are kept in `extra`. */
export interface ModelSpec {
  model_facet?: string;
  task_class?: string;
  logical_model?: string;
  pinned?: boolean;
  [extra: string]: unknown;
}

export const TASK_CLASSES = ['general', 'reasoning', 'code_generation', 'simple_chat'];
export const DEFAULT_MODEL_TEXT = `model_facet: uif.default
task_class: general          # general | reasoning | code_generation | simple_chat
logical_model: auto
pinned: false
`;

function scalar(raw: string): unknown {
  const v = raw.trim();
  if (v === 'true') return true;
  if (v === 'false') return false;
  if (v === 'null' || v === '~' || v === '') return null;
  if (/^-?\d+(\.\d+)?$/.test(v)) return Number(v);
  const q = /^(['"])(.*)\1$/.exec(v);
  return q ? q[2] : v;
}

/**
 * Read the top-level `key: value` lines of a .model file. Comments are
 * dropped; nested blocks (lists, maps) are kept as their raw text so the
 * file round-trips through the original text, not through this object.
 */
export function parseModel(text: string): ModelSpec {
  const out: ModelSpec = {};
  let block: string | null = null;
  let blockLines: string[] = [];
  const flush = () => {
    if (block) out[block] = blockLines.join('\n');
    block = null;
    blockLines = [];
  };
  for (const line of text.split(/\r?\n/)) {
    if (/^\s*(#.*)?$/.test(line)) continue;
    if (/^\s/.test(line)) {
      if (block) blockLines.push(line);
      continue;
    }
    flush();
    const m = /^([A-Za-z_][\w.-]*)\s*:\s*(.*)$/.exec(line);
    if (!m) throw new Error(`not a valid .model file: cannot read "${line.trim()}"`);
    const value = m[2].replace(/\s+#.*$/, '');
    if (value.trim() === '') block = m[1];
    else out[m[1]] = scalar(value);
  }
  flush();
  return out;
}

/** Change one top-level scalar in the .model text, keeping comments and everything else. */
export function setModelField(text: string, key: string, value: string | boolean): string {
  const rendered = String(value);
  const re = new RegExp(`^(${key.replace(/[.]/g, '\\.')}\\s*:\\s*)([^#\\n]*?)(\\s*#.*)?$`, 'm');
  if (re.test(text)) return text.replace(re, (_m, head: string, _old: string, comment?: string) => `${head}${rendered}${comment ?? ''}`);
  return `${text.replace(/\s*$/, '\n')}${key}: ${rendered}\n`;
}

/** A short label for the model a message went to. */
export function modelLabel(spec: ModelSpec): string {
  const name = typeof spec.logical_model === 'string' && spec.logical_model ? spec.logical_model : 'auto';
  return spec.pinned ? `${name} (pinned)` : `${name} · ${spec.task_class ?? 'general'}`;
}

// ---- JSONL -----------------------------------------------------------------

export function toJSONL(records: unknown[]): string {
  return records.map(r => JSON.stringify(r)).join('\n') + (records.length ? '\n' : '');
}

/** Parse JSONL; `check` validates each record. Errors name the line. */
export function fromJSONL<T>(text: string, kind: string, check: (r: Record<string, unknown>) => T | null): T[] {
  const out: T[] = [];
  text.split(/\r?\n/).forEach((line, i) => {
    if (!line.trim()) return;
    let r: unknown;
    try {
      r = JSON.parse(line);
    } catch {
      throw new Error(`not a valid ${kind} file: line ${i + 1} is not JSON`);
    }
    const ok = r && typeof r === 'object' && !Array.isArray(r) ? check(r as Record<string, unknown>) : null;
    if (!ok) throw new Error(`not a valid ${kind} file: line ${i + 1} is missing required fields`);
    out.push(ok);
  });
  return out;
}

export const readHistoryRecord = (r: Record<string, unknown>): HistoryRecord | null =>
  typeof r.user === 'string' && typeof r.assistant === 'string'
    ? ({ ...r, ts: typeof r.ts === 'number' ? r.ts : Date.now() / 1000 } as HistoryRecord)
    : null;

export const readMemoryNote = (r: Record<string, unknown>): MemoryNote | null => {
  if (typeof r.text !== 'string') return null;
  const ts = typeof r.ts === 'number' ? r.ts : Date.now() / 1000;
  const source = r.source && typeof r.source === 'object' ? (r.source as MemoryNote['source']) : { kind: 'typed' as const };
  return {
    ts,
    at: typeof r.at === 'string' ? r.at : new Date(ts * 1000).toISOString(),
    text: r.text,
    sha256: typeof r.sha256 === 'string' ? r.sha256 : '',
    source: source.kind === 'response' || source.kind === 'delivery' ? source : { ...source, kind: 'typed' },
  };
};

// ---- hashing and commitments (rfc6962-sha256, as bankml) --------------------

const enc = new TextEncoder();

export async function sha256Hex(data: string | Uint8Array): Promise<string> {
  const bytes = typeof data === 'string' ? enc.encode(data) : data;
  const digest = await crypto.subtle.digest('SHA-256', bytes as BufferSource);
  return [...new Uint8Array(digest)].map(b => b.toString(16).padStart(2, '0')).join('');
}

async function sha256Bytes(...parts: Uint8Array[]): Promise<Uint8Array> {
  const all = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
  let o = 0;
  for (const p of parts) {
    all.set(p, o);
    o += p.length;
  }
  return new Uint8Array(await crypto.subtle.digest('SHA-256', all));
}

export const MERKLE_SCHEME = 'rfc6962-sha256';

/**
 * Merkle root over JSONL lines, as bankml's `merkle_root`: leaf =
 * sha256(0x00 || exact line, without its newline), node =
 * sha256(0x01 || left || right), split at the largest power of two below n.
 * Null for no records.
 */
export async function merkleRoot(lines: string[]): Promise<string | null> {
  const hex = (b: Uint8Array) => [...b].map(x => x.toString(16).padStart(2, '0')).join('');
  const kept = lines.filter(l => l.trim());
  if (!kept.length) return null;
  const leaves = await Promise.all(kept.map(l => sha256Bytes(new Uint8Array([0]), enc.encode(l))));
  const build = async (nodes: Uint8Array[]): Promise<Uint8Array> => {
    if (nodes.length === 1) return nodes[0];
    let k = 1;
    while (k * 2 < nodes.length) k *= 2;
    const [l, r] = await Promise.all([build(nodes.slice(0, k)), build(nodes.slice(k))]);
    return sha256Bytes(new Uint8Array([1]), l, r);
  };
  return hex(await build(leaves));
}

/** bankml's shareable commitment: it reveals the shape of the file, not its content. */
export async function commitment(file: string, lines: string[]) {
  const kept = lines.filter(l => l.trim());
  return {
    file,
    records: kept.length,
    merkle_root: await merkleRoot(kept),
    scheme: MERKLE_SCHEME,
    file_sha256: kept.length ? await sha256Hex(kept.map(l => l + '\n').join('')) : null,
  };
}

// ---- what reaches the model ------------------------------------------------

export type PromptSource = 'persona' | 'prompt';

export interface ChatMessage {
  role: 'system' | 'user' | 'assistant';
  content: string;
}

/** Everything the host needs to call a model, sent with every chat message. */
export interface SendContext {
  /** The composed system prompt: prompt (or persona) + persona lines + memory block. */
  system: string;
  /** Ready for a chat API: system, then recent exchanges, then this message. */
  messages: ChatMessage[];
  /** Where the system prompt came from, e.g. "persona.system_prompt" or ".prompt". */
  promptSource: string;
  prompt: string;
  persona: PersonaDoc | null;
  memory: MemoryNote[];
  model: ModelSpec;
  /** The raw .model text. */
  modelText: string;
  /** The recent exchanges that were included in `messages`. */
  history: HistoryRecord[];
}

export const HISTORY_WINDOW = 12;
export const KEEP_CHARS = 4000;

const cut = (s: string, n = KEEP_CHARS) => (s.length > n ? s.slice(0, n) + '…' : s);

/** Choose and assemble the system prompt the way bankml and the boardroom do. */
export function composeSystem(opts: {
  prompt: string;
  persona: PersonaDoc | null;
  source: PromptSource;
  memory: MemoryNote[];
}): { system: string; promptSource: string } {
  const { prompt, persona, source, memory } = opts;
  const personaPrompt = persona?.system_prompt?.trim();
  let base: string;
  let from: string;
  if (source === 'persona' && personaPrompt) {
    base = personaPrompt;
    from = 'persona.system_prompt';
  } else if (prompt.trim()) {
    base = prompt.trim();
    from = '.prompt';
  } else if (personaPrompt) {
    base = personaPrompt;
    from = 'persona.system_prompt';
  } else {
    base = '';
    from = 'none';
  }
  const lines = persona ? personaLines(persona) : [];
  const system = [base, ...lines].filter(Boolean).join('\n') + memoryBlock(memory);
  return { system: system.trim(), promptSource: from };
}

/** System message, then the last exchanges as user/assistant pairs, then the new message. */
export function buildMessages(system: string, history: HistoryRecord[], text: string, window = HISTORY_WINDOW): ChatMessage[] {
  const msgs: ChatMessage[] = [];
  if (system) msgs.push({ role: 'system', content: system });
  for (const h of history.slice(-window)) {
    msgs.push({ role: 'user', content: cut(h.user) }, { role: 'assistant', content: cut(h.assistant) });
  }
  msgs.push({ role: 'user', content: text });
  return msgs;
}