add query expansion module with lex/vec/hyde parsing

Implements expandQuery() which runs the expansion model and
parseExpansionOutput() which extracts structured lex, vec, and hyde
variants from the model's text output, with fallback to raw text.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/pipeline/expansion.test.ts

@@ -0,0 +1,94 @@
+import { describe, it, expect } from "vitest";
+import { parseExpansionOutput } from "./expansion";
+
+// ---------------------------------------------------------------------------
+// parseExpansionOutput
+// ---------------------------------------------------------------------------
+describe("parseExpansionOutput", () => {
+  it("parses all three variants from well-formed output", () => {
+    const text = [
+      "lex: sqlite, fts5, full-text search",
+      "vec: How to perform full-text search using SQLite FTS5",
+      "hyde: This document explains the SQLite FTS5 extension for full-text search indexing.",
+    ].join("\n");
+
+    const result = parseExpansionOutput(text);
+    expect(result.lex).toBe("sqlite, fts5, full-text search");
+    expect(result.vec).toEqual([
+      "How to perform full-text search using SQLite FTS5",
+    ]);
+    expect(result.hyde).toBe(
+      "This document explains the SQLite FTS5 extension for full-text search indexing.",
+    );
+  });
+
+  it("handles multiple vec lines", () => {
+    const text = [
+      "lex: embeddings, vector search",
+      "vec: Generating vector embeddings for semantic search",
+      "vec: Using cosine similarity to find related documents",
+      "hyde: This guide covers the creation of vector embeddings.",
+    ].join("\n");
+
+    const result = parseExpansionOutput(text);
+    expect(result.vec).toEqual([
+      "Generating vector embeddings for semantic search",
+      "Using cosine similarity to find related documents",
+    ]);
+  });
+
+  it("is case-insensitive for prefixes", () => {
+    const text = [
+      "LEX: uppercase keywords",
+      "Vec: Mixed case sentence",
+      "HYDE: All caps hypothetical document",
+    ].join("\n");
+
+    const result = parseExpansionOutput(text);
+    expect(result.lex).toBe("uppercase keywords");
+    expect(result.vec).toEqual(["Mixed case sentence"]);
+    expect(result.hyde).toBe("All caps hypothetical document");
+  });
+
+  it("falls back to raw text when no prefixes are found", () => {
+    const raw = "some unstructured model output with no prefixes";
+    const result = parseExpansionOutput(raw);
+    expect(result.lex).toBe(raw);
+    expect(result.vec).toEqual([raw]);
+    expect(result.hyde).toBe(raw);
+  });
+
+  it("uses raw text for missing variants", () => {
+    const text = "lex: only keywords here";
+    const result = parseExpansionOutput(text);
+    expect(result.lex).toBe("only keywords here");
+    // vec and hyde fall back to original text
+    expect(result.vec).toEqual([text]);
+    expect(result.hyde).toBe(text);
+  });
+
+  it("handles extra whitespace and blank lines", () => {
+    const text = [
+      "",
+      "  lex:  spaced keywords  ",
+      "",
+      "  vec:  spaced sentence  ",
+      "  hyde:  spaced hypothetical  ",
+      "",
+    ].join("\n");
+
+    const result = parseExpansionOutput(text);
+    expect(result.lex).toBe("spaced keywords");
+    expect(result.vec).toEqual(["spaced sentence"]);
+    expect(result.hyde).toBe("spaced hypothetical");
+  });
+
+  it("handles only vec present", () => {
+    const text = "vec: just a vector sentence";
+    const result = parseExpansionOutput(text);
+    expect(result.vec).toEqual(["just a vector sentence"]);
+    // lex and hyde fall back to original text
+    expect(result.lex).toBe(text);
+    expect(result.hyde).toBe(text);
+  });
+});

src/pipeline/expansion.ts

@@ -0,0 +1,68 @@
+import { getExpansionPipeline } from "./models";
+import type { ExpandedQuery } from "../types";
+
+// The expansion model prompt format (from QMD)
+const EXPANSION_PROMPT = (query: string) =>
+  `/no_think Expand this search query: ${query}`;
+
+// Parse the model's output text into structured ExpandedQuery
+function parseExpansionOutput(text: string): ExpandedQuery {
+  // The model outputs lines like:
+  //   lex: keyword1, keyword2, keyword3
+  //   vec: A semantically rich sentence about the topic
+  //   hyde: This document discusses the implementation of...
+  //
+  // Parse each line by prefix. Handle multiple vec lines.
+  // Fallback: if parsing fails, use the raw text as all three variants.
+
+  const lines = text.trim().split("\n");
+  let lex = "";
+  const vec: string[] = [];
+  let hyde = "";
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.toLowerCase().startsWith("lex:")) {
+      lex = trimmed.slice(4).trim();
+    } else if (trimmed.toLowerCase().startsWith("vec:")) {
+      vec.push(trimmed.slice(4).trim());
+    } else if (trimmed.toLowerCase().startsWith("hyde:")) {
+      hyde = trimmed.slice(5).trim();
+    }
+  }
+
+  // Fallback: if we couldn't parse any variant, use raw text
+  if (!lex && vec.length === 0 && !hyde) {
+    return { lex: text, vec: [text], hyde: text };
+  }
+
+  return {
+    lex: lex || text,
+    vec: vec.length > 0 ? vec : [text],
+    hyde: hyde || text,
+  };
+}
+
+// Run query expansion using the loaded model
+export async function expandQuery(query: string): Promise<ExpandedQuery> {
+  const pipe = getExpansionPipeline();
+  if (!pipe) throw new Error("Expansion model not loaded");
+
+  const prompt = EXPANSION_PROMPT(query);
+  const output = await pipe(prompt, {
+    max_new_tokens: 256,
+    do_sample: false,
+  });
+
+  // pipeline("text-generation") returns array of { generated_text: string }
+  const generated = (output as Array<{ generated_text: string }>)[0]
+    .generated_text;
+
+  // The generated text includes the prompt - strip it
+  const responseText =
+    typeof generated === "string" ? generated.slice(prompt.length) : generated;
+
+  return parseExpansionOutput(String(responseText));
+}
+
+export { parseExpansionOutput }; // export for testing