mike-backend / backend /src /lib /chatTools.commentBatch.test.ts
bardd's picture
Deploy Mike backend-only Space
06671aa verified
Raw History Blame Contribute Delete
33.9 kB
// @ts-ignore vitest is not installed locally but provided by npx vitest at test time
import { describe, it, expect } from "vitest";
import {
buildDocxEditSurface,
renderDocxEditSurfaceForModel,
} from "./docxTrackedChanges";
import {
TOOLS,
makeCommentKey,
verifyCommentBatchPostWrite,
validateCommentPlanIntent,
validatePlannedFinding,
validateFindingTargetEvidence,
buildWriterAnchorEligibilityFailures,
buildCandidateContext,
isPassItem,
buildIgnoredPassReport,
buildApplySummary,
type CommentPlanIntent,
} from "./chatTools";
// Helper to build a minimal DOCX with a table using the docx package
async function buildTestDocx(): Promise<Buffer> {
const { Document, Paragraph, Table, TableRow, TableCell, Packer } = await import("docx");
const doc = new Document({
sections: [{
children: [
new Paragraph({ text: "First body paragraph." }),
new Table({
rows: [
new TableRow({
children: [
new TableCell({ children: [new Paragraph({ text: "Cell one text." })] }),
new TableCell({ children: [new Paragraph({ text: "Cell two text." })] }),
],
}),
],
}),
new Paragraph({ text: "Last body paragraph." }),
],
}],
});
return Buffer.from(await Packer.toBuffer(doc));
}
// Helper to build a DOCX that differs in body text
async function buildModifiedDocx(): Promise<Buffer> {
const { Document, Paragraph, Packer } = await import("docx");
const doc = new Document({
sections: [{
children: [
new Paragraph({ text: "Modified first body paragraph." }),
new Paragraph({ text: "Last body paragraph." }),
],
}],
});
return Buffer.from(await Packer.toBuffer(doc));
}
// Helper to inject a comments.xml into a docx buffer
async function injectCommentsXml(docxBytes: Buffer, commentCount: number): Promise<Buffer> {
const JSZip = (await import("jszip")).default;
const zip = await JSZip.loadAsync(docxBytes);
const commentsXml = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:comments xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
${Array.from({ length: commentCount }, (_, i) =>
`<w:comment w:id="${i}" w:author="Mike"><w:p><w:r><w:t>Comment ${i}</w:t></w:r></w:p></w:comment>`
).join("\n")}
</w:comments>`;
zip.file("word/comments.xml", commentsXml);
const [contentTypes] = await Promise.all([
zip.file("[Content_Types].xml")?.async("string") ?? "",
]);
let newContentTypes = contentTypes;
if (!contentTypes.includes("comments.xml")) {
newContentTypes = contentTypes.replace(
"</Types>",
'<Override PartName="/word/comments.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml"/></Types>'
);
zip.file("[Content_Types].xml", newContentTypes);
}
return Buffer.from(await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }));
}
// Build a synthetic anchor manifest from a surface for testing validateCommentPlanIntent
function buildTestManifest(surface: Awaited<ReturnType<typeof buildDocxEditSurface>>) {
const anchorMap = new Map();
for (const a of surface.anchors) anchorMap.set(a.block_id, a);
return {
docLabel: "doc-0",
documentId: "doc-uuid",
docRevisionId: surface.surfaceId,
surfaceId: surface.surfaceId,
readSerial: 1,
anchors: anchorMap,
};
}
function completeIntent(
overrides: Partial<CommentPlanIntent> = {},
): CommentPlanIntent {
return {
block_id: "P_0001",
comment:
"[RISK] Issue: Delivery wording is unclear. | Zamil Impact: Delay exposure. | Recommended Action: Clarify the trigger.",
comment_key: "delivery-trigger",
status: "RISK",
reasoning: {
clause_section: "Section 6 – Delivery",
violated_standard: "Delivery trigger must be objective.",
zamil_exposure: "Delivery timing may become disputed.",
},
target_text: "First body paragraph.",
type: "COMMENT",
...overrides,
};
}
describe("Comment batch pipeline — best-of-both-worlds refactor", () => {
describe("Phase 1-2: Document surface and table-cell support", () => {
it("includes table-cell paragraphs in the edit surface", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
// Should have 4 paragraphs: first body + 2 table cells + last body
expect(surface.blocks.length).toBe(4);
expect(surface.anchors.length).toBe(4);
});
it("exposes container=table_cell in the model surface render", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const rendered = renderDocxEditSurfaceForModel(surface);
expect(rendered).toContain("container=table_cell");
const tableCellBlocks = surface.anchors.filter((a) => a.container === "table_cell");
expect(tableCellBlocks.length).toBe(2);
for (const anchor of tableCellBlocks) {
expect(rendered).toContain(`[[${anchor.block_id} type=paragraph container=table_cell`);
}
});
it("does not skip paragraphs inside tables", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const tableTexts = surface.blocks
.filter((_, i) => surface.anchors[i]?.container === "table_cell")
.map((b) => b.text);
expect(tableTexts).toContain("Cell one text.");
expect(tableTexts).toContain("Cell two text.");
});
});
describe("Phase 3-6: Anchor validation and comment keys", () => {
it("uses explicit comment_key when provided", () => {
const key = makeCommentKey({
doc_id: "doc-0",
block_id: "P_0001",
target_text: "",
anchor_mode: "paragraph",
comment: "Some issue",
comment_key: "delivery-timeline-issue",
});
expect(key).toBe("doc-0:delivery-timeline-issue");
});
it("falls back to doc_id:block_id:comment digest when comment_key is missing", () => {
const key = makeCommentKey({
doc_id: "doc-0",
block_id: "P_0001",
target_text: "",
anchor_mode: "paragraph",
comment: "Some issue here",
});
expect(key).toBe("doc-0:P_0001:Some issue here");
});
});
describe("Phase 3: validateCommentPlanIntent against manifest", () => {
it("passes when block_id exists in manifest and hashes match", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const manifest = buildTestManifest(surface);
const result = validateCommentPlanIntent({
intent: { block_id: "P_0001", comment: "Test comment" },
manifest: manifest as any,
currentSurface: surface,
});
expect(result.ok).toBe(true);
if (result.ok) {
expect(result.anchor.block_id).toBe("P_0001");
}
});
it("fails with anchor_not_found when block_id is not in manifest", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const manifest = buildTestManifest(surface);
const result = validateCommentPlanIntent({
intent: { block_id: "P_9999", comment: "Test comment" },
manifest: manifest as any,
currentSurface: surface,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("anchor_not_found");
}
});
it("fails with anchor_revision_stale when manifest surfaceId mismatches currentSurface", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const manifest = buildTestManifest(surface);
// Mutate manifest surfaceId to simulate stale read
const staleManifest = { ...manifest, docRevisionId: "stale:surface", surfaceId: "stale:surface" };
const result = validateCommentPlanIntent({
intent: { block_id: "P_0001", comment: "Test comment" },
manifest: staleManifest as any,
currentSurface: surface,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("anchor_revision_stale");
}
});
it("fails with anchor_stale_requires_document_refresh when currentSurface is missing the block", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const manifest = buildTestManifest(surface);
// Create a currentSurface with the SAME surfaceId but missing the target anchor
// (simulates a case where the manifest says the block exists but live doc doesn't)
const tamperedSurface = {
...surface,
anchors: surface.anchors.filter((a) => a.block_id !== "P_0001"),
};
const result = validateCommentPlanIntent({
intent: { block_id: "P_0001", comment: "Test comment" },
manifest: manifest as any,
currentSurface: tamperedSurface as any,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("anchor_stale_requires_document_refresh");
}
});
it("fails with anchor_identity_mismatch when native_para_id differs", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const manifest = buildTestManifest(surface);
// Mutate the currentSurface anchor to have a different native_para_id
const tamperedSurface = {
...surface,
anchors: surface.anchors.map((a, i) =>
i === 0 ? { ...a, native_para_id: "FAKE1234" } : a
),
};
const result = validateCommentPlanIntent({
intent: { block_id: "P_0001", comment: "Test comment" },
manifest: manifest as any,
currentSurface: tamperedSurface as any,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("anchor_identity_mismatch");
}
});
it("fails with ANCHOR_NOT_ELIGIBLE for a navigation anchor", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const ineligibleSurface = {
...surface,
anchors: surface.anchors.map((anchor, index) =>
index === 0
? {
...anchor,
role: "navigation" as const,
anchor_eligible: false,
anchor_rejection_reason: "table_of_contents" as const,
navigation_score: 1,
}
: anchor,
),
navigationRegions: [
{
startBlockId: "P_0001",
endBlockId: "P_0001",
startIndex: 0,
endIndex: 0,
reason: "table_of_contents" as const,
},
],
};
const manifest = buildTestManifest(ineligibleSurface);
const result = validateCommentPlanIntent({
intent: {
block_id: "P_0001",
comment: "Test comment",
target_text: "First body paragraph.",
},
manifest: manifest as any,
currentSurface: ineligibleSurface,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("ANCHOR_NOT_ELIGIBLE");
expect(result.excludedNavigationRegion).toMatchObject({
start_block_id: "P_0001",
end_block_id: "P_0001",
});
expect(result.candidates?.every((candidate) => candidate.block_id !== "P_0001")).toBe(true);
}
});
it("rejects a tracked edit planned on a navigation block", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const ineligibleSurface = {
...surface,
anchors: surface.anchors.map((anchor, index) =>
index === 0
? {
...anchor,
role: "navigation" as const,
anchor_eligible: false,
anchor_rejection_reason: "table_of_contents" as const,
navigation_label: "First body paragraph",
}
: anchor,
),
navigationRegions: [
{
startBlockId: "P_0001",
endBlockId: "P_0001",
startIndex: 0,
endIndex: 0,
reason: "table_of_contents" as const,
},
],
};
const result = validatePlannedFinding({
intent: completeIntent({
target_text: undefined,
type: "REWRITE_PARAGRAPH",
new_text: "Replacement wording.",
}),
manifest: buildTestManifest(ineligibleSurface) as any,
currentSurface: ineligibleSurface,
allowCommentFallback: true,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("ANCHOR_NOT_ELIGIBLE");
}
});
});
describe("Phase 3: supporting evidence and deterministic redirect", () => {
it("requires meaningful target_text for non-MISSING findings", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const result = validateFindingTargetEvidence({
intent: completeIntent({ target_text: undefined }),
surface,
anchor: surface.anchors[0],
});
expect(result).toMatchObject({
ok: false,
failureCode: "schema_missing_target_text",
});
});
it("rejects target_text that is not in the selected block", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const result = validateFindingTargetEvidence({
intent: completeIntent({
target_text: "Supplier shall deliver within thirty days.",
}),
surface,
anchor: surface.anchors[0],
});
expect(result).toMatchObject({
ok: false,
failureCode: "target_text_not_in_block",
});
});
it("allows MISSING findings without target_text", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const result = validateFindingTargetEvidence({
intent: completeIntent({
status: "MISSING",
target_text: undefined,
}),
surface,
anchor: surface.anchors[0],
});
expect(result).toEqual({ ok: true });
});
it("rejects heading-only evidence when substantive text follows", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const headingSurface = {
...surface,
blocks: surface.blocks.map((block, index) =>
index === 0
? { ...block, text: "Section 6 – Delivery", length: 20 }
: index === 1
? {
...block,
text:
"The Supplier shall deliver all materials within thirty days after receiving the approved purchase order and all required drawings.",
length: 124,
}
: block,
),
anchors: surface.anchors.map((anchor, index) =>
index === 0
? {
...anchor,
style_id: "Heading1",
style_name: "Heading 1",
text_preview: "Section 6 – Delivery",
}
: anchor,
),
};
const result = validateFindingTargetEvidence({
intent: completeIntent({
target_text: "Section 6 – Delivery",
}),
surface: headingSurface,
anchor: headingSurface.anchors[0],
});
expect(result).toMatchObject({
ok: false,
failureCode: "target_text_heading_only",
});
});
it("auto-redirects only a unique eligible title and quote match", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const clauseText =
"Section 6 Delivery Supplier shall deliver the materials within thirty days after receiving the approved purchase order.";
const redirectSurface = {
...surface,
blocks: surface.blocks.map((block, index) =>
index === 0
? { ...block, text: "Section 6 Delivery 12" }
: index === 1
? { ...block, text: clauseText }
: block,
),
anchors: surface.anchors.map((anchor, index) =>
index === 0
? {
...anchor,
role: "navigation" as const,
anchor_eligible: false,
anchor_rejection_reason: "manual_index" as const,
navigation_label: "Section 6 Delivery",
text_preview: "Section 6 Delivery 12",
}
: index === 1
? { ...anchor, text_preview: clauseText }
: anchor,
),
navigationRegions: [
{
startBlockId: "P_0001",
endBlockId: "P_0001",
startIndex: 0,
endIndex: 0,
reason: "manual_index" as const,
},
],
};
const result = validatePlannedFinding({
intent: completeIntent({
block_id: "P_0001",
target_text:
"Supplier shall deliver the materials within thirty days",
}),
manifest: buildTestManifest(redirectSurface) as any,
currentSurface: redirectSurface,
allowCommentFallback: true,
});
expect(result.ok).toBe(true);
if (result.ok) {
expect(result.item.block_id).toBe("P_0002");
expect(result.item.anchorRedirectedFrom).toBe("P_0001");
expect(result.item.anchorRedirectedTo).toBe("P_0002");
}
});
it("does not auto-redirect when two eligible blocks match", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const clauseText =
"Section 6 Delivery Supplier shall deliver the materials within thirty days after receiving the approved purchase order.";
const ambiguousSurface = {
...surface,
blocks: surface.blocks.map((block, index) =>
index === 0
? { ...block, text: "Section 6 Delivery 12" }
: index === 1 || index === 2
? { ...block, text: clauseText }
: block,
),
anchors: surface.anchors.map((anchor, index) =>
index === 0
? {
...anchor,
role: "navigation" as const,
anchor_eligible: false,
anchor_rejection_reason: "manual_index" as const,
navigation_label: "Section 6 Delivery",
text_preview: "Section 6 Delivery 12",
}
: index === 1 || index === 2
? { ...anchor, text_preview: clauseText }
: anchor,
),
navigationRegions: [
{
startBlockId: "P_0001",
endBlockId: "P_0001",
startIndex: 0,
endIndex: 0,
reason: "manual_index" as const,
},
],
};
const result = validatePlannedFinding({
intent: completeIntent({
block_id: "P_0001",
target_text:
"Supplier shall deliver the materials within thirty days",
}),
manifest: buildTestManifest(ambiguousSurface) as any,
currentSurface: ambiguousSurface,
allowCommentFallback: true,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.failureCode).toBe("ANCHOR_NOT_ELIGIBLE");
}
});
});
describe("Phase 5: Candidate context for corrections", () => {
it("returns nearby anchors around the target block_id", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const candidates = buildCandidateContext({
surface,
targetBlockId: "P_0002",
windowSize: 2,
});
// Should include P_0001, P_0002, P_0003 (window of 2 around index 1)
expect(candidates.length).toBeGreaterThan(0);
expect(candidates.some((c) => c.block_id === "P_0002")).toBe(true);
});
it("never returns navigation anchors as correction context", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const ineligibleSurface = {
...surface,
anchors: surface.anchors.map((anchor, index) =>
index === 0
? {
...anchor,
role: "navigation" as const,
anchor_eligible: false,
anchor_rejection_reason: "table_of_contents" as const,
}
: anchor,
),
};
const candidates = buildCandidateContext({
surface: ineligibleSurface,
targetBlockId: "P_0001",
windowSize: 3,
});
expect(candidates.some((candidate) => candidate.block_id === "P_0001")).toBe(false);
});
});
describe("New tools exist in TOOLS array", () => {
it("includes plan_comments_on_document", () => {
const names = TOOLS.map((t: any) => t.function.name);
expect(names).toContain("plan_comments_on_document");
});
it("includes correct_comment_anchors", () => {
const names = TOOLS.map((t: any) => t.function.name);
expect(names).toContain("correct_comment_anchors");
});
it("still includes comment_on_document for backward compatibility", () => {
const names = TOOLS.map((t: any) => t.function.name);
expect(names).toContain("comment_on_document");
});
});
describe("Writer-boundary defense", () => {
it("rejects ineligible anchors before the Python writer", async () => {
const bytes = await buildTestDocx();
const surface = await buildDocxEditSurface(bytes);
const anchor = {
...surface.anchors[0],
role: "navigation" as const,
anchor_eligible: false,
anchor_rejection_reason: "table_of_contents" as const,
};
const failures = buildWriterAnchorEligibilityFailures([
{
id: "comment-1",
args: {
block_id: anchor.block_id,
comment_key: "delivery-trigger",
type: "COMMENT",
},
anchor,
},
]);
expect(failures).toHaveLength(1);
expect(failures[0]).toMatchObject({
failure_code: "ANCHOR_NOT_ELIGIBLE",
model_block_id: "P_0001",
anchor_rejection_reason: "table_of_contents",
});
});
});
describe("Phase 9-10: Post-write verification and manifest refresh", () => {
it("passes verification when body text is unchanged and comments exist in xml", async () => {
const original = await buildTestDocx();
const commented = await injectCommentsXml(original, 2);
const verification = await verifyCommentBatchPostWrite({
originalBytes: original,
commentedBytes: commented,
expectedAppliedCount: 2,
appliedReports: [
{ status: "applied", model_block_id: "P_0001" },
{ status: "applied", model_block_id: "P_0002" },
],
});
expect(verification.ok).toBe(true);
expect(verification.body_text_unchanged).toBe(true);
expect(verification.comments_in_xml).toBe(2);
expect(verification.missing_from_xml).toBe(0);
expect(verification.original_block_count).toBe(verification.commented_block_count);
});
it("fails verification when body text changed after write", async () => {
const original = await buildTestDocx();
const modified = await buildModifiedDocx();
const commented = await injectCommentsXml(modified, 1);
const verification = await verifyCommentBatchPostWrite({
originalBytes: original,
commentedBytes: commented,
expectedAppliedCount: 1,
appliedReports: [{ status: "applied", model_block_id: "P_0001" }],
});
expect(verification.ok).toBe(false);
expect(verification.body_text_unchanged).toBe(false);
});
it("detects missing comments in comments.xml", async () => {
const original = await buildTestDocx();
// Inject fewer comments than expected
const commented = await injectCommentsXml(original, 1);
const verification = await verifyCommentBatchPostWrite({
originalBytes: original,
commentedBytes: commented,
expectedAppliedCount: 3,
appliedReports: [
{ status: "applied", model_block_id: "P_0001" },
{ status: "applied", model_block_id: "P_0002" },
{ status: "applied", model_block_id: "P_0003" },
],
});
expect(verification.ok).toBe(false);
expect(verification.missing_from_xml).toBeGreaterThan(0);
});
it("counts applied comments correctly in summary", async () => {
const original = await buildTestDocx();
const commented = await injectCommentsXml(original, 2);
const verification = await verifyCommentBatchPostWrite({
originalBytes: original,
commentedBytes: commented,
expectedAppliedCount: 2,
appliedReports: [
{ status: "applied", model_block_id: "P_0001" },
{ status: "applied", model_block_id: "P_0002" },
{ status: "not_applied", model_block_id: "P_0003", failure_code: "anchor_not_found" },
],
});
expect(verification.ok).toBe(true);
expect(verification.expected_applied).toBe(2);
});
});
describe("Acceptance criteria: surface stability", () => {
it("preserves paragraph count after comment-only write", async () => {
const original = await buildTestDocx();
const commented = await injectCommentsXml(original, 1);
const verification = await verifyCommentBatchPostWrite({
originalBytes: original,
commentedBytes: commented,
expectedAppliedCount: 1,
appliedReports: [{ status: "applied", model_block_id: "P_0001" }],
});
expect(verification.original_block_count).toBe(4);
expect(verification.commented_block_count).toBe(4);
});
});
describe("PASS filtering and reasoning", () => {
it("detects PASS by status", () => {
expect(
isPassItem({ block_id: "P_0001", comment: "ok", status: "PASS" })
).toBe(true);
expect(
isPassItem({ block_id: "P_0001", comment: "ok", status: "pass" })
).toBe(true);
expect(
isPassItem({ block_id: "P_0001", comment: "ok", status: "FAIL" })
).toBe(false);
});
it("detects PASS by comment prefix when status is missing", () => {
expect(
isPassItem({ block_id: "P_0001", comment: "[PASS] Looks fine." })
).toBe(true);
expect(
isPassItem({ block_id: "P_0001", comment: "[pass] Looks fine." })
).toBe(true);
expect(
isPassItem({ block_id: "P_0001", comment: "[FAIL] Issue." })
).toBe(false);
});
it("builds ignored_pass report shape", () => {
const report = buildIgnoredPassReport(
{ block_id: "P_0001", comment: "[PASS] ok", comment_key: "ok-1", reasoning: { clause_section: "c1", violated_standard: "v1", zamil_exposure: "z1" } },
3
);
expect(report.status).toBe("ignored_pass");
expect(report.ignored_reason).toBe("pass_status");
expect(report.comment_key).toBe("ok-1");
expect(report.reasoning).toEqual({ clause_section: "c1", violated_standard: "v1", zamil_exposure: "z1" });
expect(report.failure_code).toBeNull();
});
it("excludes ignored_pass from apply summary failed count", () => {
const summary = buildApplySummary([
{ status: "applied", id: "a" },
{ status: "ignored_pass", id: "b" },
{ status: "not_applied", id: "c", failure_code: "anchor_not_found" },
])!;
expect(summary.total).toBe(2);
expect(summary.applied).toBe(1);
expect(summary.failed).toBe(1);
expect(summary.failures.length).toBe(1);
expect(summary.failures[0].id).toBe("c");
});
it("counts ignored_pass as zero actionable denominator impact", () => {
const summary = buildApplySummary([
{ status: "ignored_pass", id: "b" },
{ status: "ignored_pass", id: "c" },
])!;
expect(summary.total).toBe(0);
expect(summary.applied).toBe(0);
expect(summary.failed).toBe(0);
expect(summary.failures.length).toBe(0);
});
});
});