scripts/bench-test-changed.mts

// Benchmarks `pnpm test:changed` planning/runtime behavior across repeated runs.
import { spawnSync } from "node:child_process";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { parseFlagArgs, stringFlag } from "./lib/arg-utils.mts";
import { formatMs } from "./lib/vitest-report-cli-utils.mts";

type BenchOptions = {
  cwd: string;
  maxWorkers?: number;
  mode: "ref" | "worktree";
  ref: string;
  rss: boolean;
};

type BenchResult = ReturnType<typeof runBenchCommand>;

type BenchCommandParams = Pick<BenchOptions, "cwd" | "maxWorkers" | "rss"> & {
  command: [string, ...string[]];
  label: string;
};

function parsePositiveInteger(raw: string | undefined, label: string) {
  const value = raw?.trim();
  if (!value) {
    throw new Error(`${label} requires a value`);
  }
  if (!/^\d+$/u.test(value)) {
    throw new Error(`${label} must be a positive integer`);
  }
  const parsed = Number(value);
  if (!Number.isSafeInteger(parsed) || parsed < 1) {
    throw new Error(`${label} must be a safe positive integer`);
  }
  return parsed;
}

function positiveIntegerFlag(flag: string, key: "maxWorkers") {
  return {
    consume(argv: readonly string[], index: number) {
      if (argv[index] !== flag) {
        return null;
      }
      const rawValue = argv[index + 1];
      if (!rawValue || rawValue.startsWith("--")) {
        throw new Error(`${flag} requires a value`);
      }
      return {
        flag,
        nextIndex: index + 1,
        repeatable: false,
        apply(target: BenchOptions) {
          target[key] = parsePositiveInteger(rawValue, flag);
        },
      };
    },
  };
}

export function parseArgs(argv: string[]): BenchOptions {
  const args = parseFlagArgs(
    argv,
    {
      cwd: process.cwd(),
      ref: "origin/main",
      rss: process.platform === "darwin",
      mode: "ref",
    },
    [
      stringFlag("--cwd", "cwd"),
      stringFlag("--ref", "ref"),
      positiveIntegerFlag("--max-workers", "maxWorkers"),
    ],
    {
      onUnhandledArg(arg: string, target: BenchOptions) {
        if (arg === "--no-rss") {
          target.rss = false;
          return "handled";
        }
        if (arg === "--worktree") {
          target.mode = "worktree";
          return "handled";
        }
        return undefined;
      },
    },
  );
  return {
    cwd: path.resolve(args.cwd),
    mode: args.mode,
    ref: args.ref,
    rss: args.rss,
    ...(typeof args.maxWorkers === "number" ? { maxWorkers: args.maxWorkers } : {}),
  };
}

function quoteArg(arg: string) {
  return /[^A-Za-z0-9_./:-]/.test(arg) ? JSON.stringify(arg) : arg;
}

function runGitList(args: string[], cwd: string) {
  const result = spawnSync("git", args, {
    cwd,
    encoding: "utf8",
  });
  if (result.status !== 0) {
    throw new Error(result.stderr || result.stdout || `git ${args.join(" ")} failed`);
  }
  return result.stdout
    .split("\n")
    .map((line) => line.trim())
    .filter((line) => line.length > 0);
}

function listChangedPaths(opts: BenchOptions) {
  if (opts.mode === "worktree") {
    return [
      ...new Set([
        ...runGitList(["diff", "--name-only", "--relative", "HEAD", "--"], opts.cwd),
        ...runGitList(["ls-files", "--others", "--exclude-standard"], opts.cwd),
      ]),
    ].toSorted((left, right) => left.localeCompare(right));
  }
  return runGitList(["diff", "--name-only", `${opts.ref}...HEAD`], opts.cwd);
}

export function parseMaxRssBytes(output: string) {
  const match = output.match(
    /(?:^|\n)[^\S\r\n]*(\d+)[^\S\r\n]+maximum resident set size[^\S\r\n]*(?:\r?\n|$)/u,
  );
  if (!match) {
    return null;
  }
  const parsed = Number(match[1]);
  return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null;
}

export function formatRss(valueBytes: number | null) {
  if (valueBytes === null) {
    return "n/a";
  }
  return `${(valueBytes / 1024 / 1024).toFixed(1)}MB`;
}

export function resolveBenchRssResult({
  label,
  output,
  rss,
  status,
}: {
  label: string;
  output: string;
  rss: boolean;
  status: number;
}) {
  if (!rss) {
    return { maxRssBytes: null, output, status };
  }
  const maxRssBytes = parseMaxRssBytes(output);
  if (status === 0 && maxRssBytes === null) {
    return {
      maxRssBytes,
      output: `${output}${output.endsWith("\n") ? "" : "\n"}[bench-test-changed] ${label} missing maximum resident set size from /usr/bin/time -l output\n`,
      status: 1,
    };
  }
  return { maxRssBytes, output, status };
}

function runBenchCommand(params: BenchCommandParams) {
  const env = { ...process.env };
  if (typeof params.maxWorkers === "number") {
    env.OPENCLAW_VITEST_MAX_WORKERS = String(params.maxWorkers);
  }
  const startedAt = process.hrtime.bigint();
  const commandArgs = params.rss ? ["-l", ...params.command] : params.command;
  const result = spawnSync(
    params.rss ? "/usr/bin/time" : params.command[0],
    params.rss ? commandArgs : commandArgs.slice(1),
    {
      cwd: params.cwd,
      env,
      encoding: "utf8",
      maxBuffer: 1024 * 1024 * 32,
    },
  );
  const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
  const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
  const normalized = resolveBenchRssResult({
    label: params.label,
    output,
    rss: params.rss,
    status: result.status ?? 1,
  });
  return {
    elapsedMs,
    maxRssBytes: normalized.maxRssBytes,
    status: normalized.status,
    output: normalized.output,
  };
}

function printRunSummary(label: string, result: BenchResult) {
  console.log(
    `${label.padEnd(8, " ")} wall=${formatMs(result.elapsedMs).padStart(9, " ")} rss=${formatRss(
      result.maxRssBytes,
    ).padStart(9, " ")}`,
  );
}

function main() {
  let opts;
  try {
    opts = parseArgs(process.argv.slice(2));
  } catch (error) {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  }
  const changedPaths = listChangedPaths(opts);
  if (changedPaths.length === 0) {
    console.log(
      opts.mode === "worktree"
        ? "[bench-test-changed] no changed paths in worktree"
        : `[bench-test-changed] no changed paths for ${opts.ref}...HEAD`,
    );
    process.exit(0);
  }

  console.log(
    opts.mode === "worktree"
      ? "[bench-test-changed] mode=worktree"
      : `[bench-test-changed] ref=${opts.ref}`,
  );
  console.log("[bench-test-changed] changed paths:");
  for (const changedPath of changedPaths) {
    console.log(`- ${changedPath}`);
  }

  const routedCommand: [string, ...string[]] =
    opts.mode === "worktree"
      ? [process.execPath, "--import", "tsx", "scripts/test-projects.mts", ...changedPaths]
      : [process.execPath, "--import", "tsx", "scripts/test-projects.mts", "--changed", opts.ref];
  const rootCommand: [string, ...string[]] = [
    process.execPath,
    "scripts/run-vitest.mjs",
    "run",
    "--config",
    "vitest.config.ts",
    ...changedPaths,
  ];

  console.log(`[bench-test-changed] routed: ${routedCommand.map(quoteArg).join(" ")}`);
  const routed = runBenchCommand({
    command: routedCommand,
    cwd: opts.cwd,
    label: "routed",
    rss: opts.rss,
    ...(typeof opts.maxWorkers === "number" ? { maxWorkers: opts.maxWorkers } : {}),
  });
  if (routed.status !== 0) {
    process.stderr.write(routed.output);
    process.exit(routed.status);
  }

  console.log(`[bench-test-changed] root:   ${rootCommand.map(quoteArg).join(" ")}`);
  const root = runBenchCommand({
    command: rootCommand,
    cwd: opts.cwd,
    label: "root",
    rss: opts.rss,
    ...(typeof opts.maxWorkers === "number" ? { maxWorkers: opts.maxWorkers } : {}),
  });
  if (root.status !== 0) {
    process.stderr.write(root.output);
    process.exit(root.status);
  }

  printRunSummary("routed", routed);
  printRunSummary("root", root);
  console.log(
    `[bench-test-changed] delta wall=${formatMs(root.elapsedMs - routed.elapsedMs)} rss=${
      routed.maxRssBytes !== null && root.maxRssBytes !== null
        ? formatRss(root.maxRssBytes - routed.maxRssBytes)
        : "n/a"
    }`,
  );
}

const isMain =
  typeof process.argv[1] === "string" &&
  process.argv[1].length > 0 &&
  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;

if (isMain) {
  main();
}